From c5f4415f2409e8778b4edcd1c79f1ed89577e3e2 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Thu, 29 Jan 2026 20:11:52 -0500 Subject: [PATCH 1/3] ci: fetch prebuilt binaries --- .github/workflows/release.yml | 16 +++- DEVELOPER.md | 98 +++++++++++++++++++- releases.json | 82 +++++++++++++++++ scripts/fetch_releases.py | 169 ++++++++++++++++++++++++++++++++++ 4 files changed, 359 insertions(+), 6 deletions(-) create mode 100644 releases.json create mode 100644 scripts/fetch_releases.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4f83d3a..28c3572 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,6 +24,9 @@ jobs: shell: bash steps: + - name: Checkout repo + uses: actions/checkout@v5 + - name: Checkout modflow6 uses: actions/checkout@v5 with: @@ -88,6 +91,12 @@ jobs: # - name: Setup tmate session # uses: mxschmitt/action-tmate@v3 + - name: Fetch pre-built programs + run: | + ostag="${{ steps.ostag.outputs.ostag }}" + mkdir $ostag + python3 scripts/fetch_releases.py --manifest releases.json --ostag $ostag --outdir $ostag --zip $ostag.zip + - name: Build programs uses: nick-fields/retry@v3 with: @@ -95,12 +104,9 @@ jobs: timeout_minutes: 40 command: | ostag="${{ steps.ostag.outputs.ostag }}" - mkdir $ostag - pixi run --manifest-path modflow6/pixi.toml make-program : --appdir $ostag --exclude gridgen --zip $ostag.zip --verbose + fetched=$(python3 scripts/fetch_releases.py --manifest releases.json --list) + pixi run --manifest-path modflow6/pixi.toml make-program : --appdir $ostag --exclude "$fetched" --zip $ostag.zip --verbose pixi run --manifest-path modflow6/pixi.toml make-program mf2005,mflgr,mfnwt,mfusg --appdir $ostag --double --keep --zip $ostag.zip --verbose - if [[ "${{ matrix.os }}" == "macos-14" ]]; then - pixi run --manifest-path modflow6/pixi.toml make-program mf6 --appdir $ostag --keep --zip $ostag.zip --verbose --fflags='-O1' - fi pixi run --manifest-path modflow6/pixi.toml make-code-json --appdir $ostag --zip $ostag.zip --verbose - name: Show programs diff --git a/DEVELOPER.md b/DEVELOPER.md index e92d6aa..0d5f12e 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -6,6 +6,12 @@ This document provides guidance for using this repository to release USGS execut - [Overview](#overview) +- [Program sources](#program-sources) +- [Hybrid release system](#hybrid-release-system) + - [How it works](#how-it-works) + - [Adding a program](#adding-a-program) + - [Updating a version](#updating-a-version) + - [Future migration](#future-migration) - [Triggering a release](#triggering-a-release) - [GitHub UI](#github-ui) - [GitHub CLI](#github-cli) @@ -35,6 +41,96 @@ If the triggering event is `workflow_dispatch`: **Note**: version numbers don't currently follow semantic versioning conventions, but simply increment an integer for each release. +## Program sources + +The distribution includes programs from across the MODFLOW ecosystem. Each program's source code and/or pre-built binaries come from one of three places: a MODFLOW-ORG GitHub repository with platform binaries, a MODFLOW-ORG repository with source-only releases, or a USGS server. This table tracks the current state as of January 2026. + +### Fetched from pre-built GitHub releases + +These programs are downloaded as pre-built binaries via `releases.json`. Their repositories publish platform-specific archives (linux.zip, mac.zip, macarm.zip, win64.zip) as release assets. + +| Program(s) | Repository | Release tag | Notes | +|------------|-----------|-------------|-------| +| mf6, zbud6, libmf6 | [MODFLOW-ORG/modflow6](https://github.com/MODFLOW-ORG/modflow6) | 6.6.3 | Assets named `mf6.6.3_{platform}.zip`, binaries nested in `bin/` subdirectory | +| triangle | [MODFLOW-ORG/triangle](https://github.com/MODFLOW-ORG/triangle) | v1.6 | | +| gridgen | [MODFLOW-ORG/gridgen](https://github.com/MODFLOW-ORG/gridgen) | v1.0.02 | | +| zonbud3 | [MODFLOW-ORG/zonbud](https://github.com/MODFLOW-ORG/zonbud) | v3.01 | Archive contains `zonbud`, renamed to `zonbud3` | +| zonbudusg | [MODFLOW-ORG/zonbudusg](https://github.com/MODFLOW-ORG/zonbudusg) | v1.01 | | +| mfusg_gsi | [MODFLOW-ORG/mfusgt](https://github.com/MODFLOW-ORG/mfusgt) | v2.6.0 | Archive contains `mfusgt`, renamed to `mfusg_gsi` | + +### Built from source by pymake + +These programs are compiled by pymake because their repositories do not yet publish pre-built platform binaries. + +| Program(s) | Source | Notes | +|------------|--------|-------| +| mf2005, mf2005dbl | [MODFLOW-ORG/mf2005](https://github.com/MODFLOW-ORG/mf2005) | Source-only release (v.1.12.00) | +| mfusg, mfusgdbl | [MODFLOW-ORG/mfusg](https://github.com/MODFLOW-ORG/mfusg) | Source-only release (v1.5.00) | +| mt3dms | [MODFLOW-ORG/mt3dms](https://github.com/MODFLOW-ORG/mt3dms) | Source-only release (2.0) | +| mt3dusgs | [MODFLOW-ORG/mt3d-usgs](https://github.com/MODFLOW-ORG/mt3d-usgs) | Source-only release (1.1.0) | +| mflgr, mflgrdbl | [USGS](https://water.usgs.gov/ogw/modflow-lgr/) | No GitHub repo | +| mfnwt, mfnwtdbl | [USGS](https://water.usgs.gov/water-resources/software/MODFLOW-NWT/) | No GitHub repo | +| mp6 | [USGS](https://water.usgs.gov/water-resources/software/MODPATH/) | No GitHub repo | +| mp7 | [USGS](https://water.usgs.gov/water-resources/software/MODPATH/) | [MODFLOW-ORG/modpath-v7](https://github.com/MODFLOW-ORG/modpath-v7) exists but has no releases | +| crt | [USGS](https://water.usgs.gov/ogw/CRT/) | No GitHub repo | +| vs2dt | [USGS](https://water.usgs.gov/water-resources/software/VS2DI/) | No GitHub repo | +| sutra | [USGS](https://water.usgs.gov/water-resources/software/sutra/) | No GitHub repo | +| mf2000 | [USGS](https://water.usgs.gov/nrp/gwsoftware/modflow2000/) | No GitHub repo | +| swtv4 | [USGS](https://water.usgs.gov/water-resources/software/SEAWAT/) | No GitHub repo | + +To move a program from "built by pymake" to "fetched from releases", its repository needs to start publishing platform-specific binary archives as GitHub release assets, then an entry can be added to `releases.json` (see [Adding a program](#adding-a-program)). + +## Hybrid release system + +The release workflow uses a hybrid approach: some programs are downloaded as pre-built binaries from their independently managed GitHub repositories, while others are still compiled from source by pymake. This is a stopgap until the [modflow-devtools programs API](https://github.com/MODFLOW-ORG/modflow-devtools/issues/263) is ready to manage all program installations. + +### How it works + +The file `releases.json` in the repository root is a manifest listing programs to fetch from GitHub releases. Each entry specifies a source repository, release tag, platform-specific asset filenames, and a mapping of output program names to archive filenames. + +During a release build, the workflow: + +1. Runs `scripts/fetch_releases.py` to download pre-built binaries for the current platform from each repository listed in `releases.json`. +2. Runs pymake to compile the remaining programs, excluding those already fetched. +3. Combines everything into the platform zip alongside pymake-generated metadata. + +The fetch script handles platform-specific file extensions (`.exe`, `.dll`, `.dylib`, `.so`) and supports renaming programs when the archive filename differs from the distribution name (e.g., `zonbud` in the archive becomes `zonbud3` in the distribution). + +### Adding a program + +When a program repository begins publishing pre-built platform binaries as release assets, add an entry to `releases.json`: + +```json +{ + "repo": "MODFLOW-ORG/", + "tag": "", + "assets": { + "linux": "linux.zip", + "mac": "mac.zip", + "macarm": "macarm.zip", + "win64": "win64.zip" + }, + "programs": { + "": "" + } +} +``` + +- `repo`: GitHub owner/name +- `tag`: release tag to download from +- `assets`: platform-to-asset-filename mapping (must match the release asset names) +- `programs`: maps the desired output filename to the filename inside the archive. If they are the same, use the same value for both. + +The program will automatically be excluded from the pymake build. No workflow changes are needed. + +### Updating a version + +To update a program to a new release, change the `tag` field in its `releases.json` entry (and update asset filenames if they changed). The next release build will fetch the new version. + +### Future migration + +This hybrid system will be replaced by the [modflow-devtools programs API](https://github.com/MODFLOW-ORG/modflow-devtools/issues/263) once it is ready. The programs API uses a similar model (repository + tag + platform assets) but adds registry synchronization, caching, and multi-version management. At that point, `releases.json` and `scripts/fetch_releases.py` can be removed and replaced with a `programs install` command. When installation of the full suite is possible with devtools it is possible a combined distribution like this repository becomes less of a necessity but it becomes trivially maintainable. + ## Triggering a release The `workflow_dispatch` event is GitHub's mechanism for manually triggering workflows. This can be accomplished from the Actions tab in the GitHub UI, or via the [GitHub CLI](https://cli.github.com/manual/gh_workflow_run). @@ -63,4 +159,4 @@ gh workflow run .yml -R MODFLOW-ORG/executables ```shell gh workflow run .yml -R MODFLOW-ORG/executables -r master -``` \ No newline at end of file +``` diff --git a/releases.json b/releases.json new file mode 100644 index 0000000..1a78456 --- /dev/null +++ b/releases.json @@ -0,0 +1,82 @@ +[ + { + "repo": "MODFLOW-ORG/modflow6", + "tag": "6.6.3", + "assets": { + "linux": "mf6.6.3_linux.zip", + "mac": "mf6.6.3_mac.zip", + "macarm": "mf6.6.3_macarm.zip", + "win64": "mf6.6.3_win64.zip" + }, + "programs": { + "mf6": "mf6", + "zbud6": "zbud6", + "libmf6": "libmf6" + } + }, + { + "repo": "MODFLOW-ORG/triangle", + "tag": "v1.6", + "assets": { + "linux": "linux.zip", + "mac": "mac.zip", + "macarm": "macarm.zip", + "win64": "win64.zip" + }, + "programs": { + "triangle": "triangle" + } + }, + { + "repo": "MODFLOW-ORG/gridgen", + "tag": "v1.0.02", + "assets": { + "linux": "linux.zip", + "mac": "mac.zip", + "macarm": "macarm.zip", + "win64": "win64.zip" + }, + "programs": { + "gridgen": "gridgen" + } + }, + { + "repo": "MODFLOW-ORG/zonbud", + "tag": "v3.01", + "assets": { + "linux": "linux.zip", + "mac": "mac.zip", + "macarm": "macarm.zip", + "win64": "win64.zip" + }, + "programs": { + "zonbud3": "zonbud" + } + }, + { + "repo": "MODFLOW-ORG/zonbudusg", + "tag": "v1.01", + "assets": { + "linux": "linux.zip", + "mac": "mac.zip", + "macarm": "macarm.zip", + "win64": "win64.zip" + }, + "programs": { + "zonbudusg": "zonbudusg" + } + }, + { + "repo": "MODFLOW-ORG/mfusgt", + "tag": "v2.6.0", + "assets": { + "linux": "linux.zip", + "mac": "mac.zip", + "macarm": "macarm.zip", + "win64": "win64.zip" + }, + "programs": { + "mfusg_gsi": "mfusgt" + } + } +] diff --git a/scripts/fetch_releases.py b/scripts/fetch_releases.py new file mode 100644 index 0000000..1b56645 --- /dev/null +++ b/scripts/fetch_releases.py @@ -0,0 +1,169 @@ +"""Fetch pre-built executables from independent GitHub release repos. + +This script reads releases.json and either downloads pre-built binaries +for a given platform or lists program names (for pymake --exclude). + +Usage: + # List programs managed by releases.json (for pymake --exclude) + python fetch_releases.py --manifest releases.json --list + + # Download pre-built binaries for a platform + python fetch_releases.py --manifest releases.json --ostag mac --outdir mac + + # Download and add to an existing zip + python fetch_releases.py --manifest releases.json --ostag mac --outdir mac --zip mac.zip +""" + +import argparse +import json +import os +import shutil +import stat +import sys +import tempfile +import urllib.request +import zipfile +from pathlib import Path + +# platform extensions for executables and libraries +_EXE_EXT = {"win64": ".exe"} +_LIB_EXT = {"linux": ".so", "mac": ".dylib", "macarm": ".dylib", "win64": ".dll"} + +GITHUB_URL = "https://github.com/{repo}/releases/download/{tag}/{asset}" + + +def _load_manifest(path): + with open(path) as f: + return json.load(f) + + +def _all_program_names(manifest): + """Return sorted list of all output program names across all entries.""" + names = [] + for entry in manifest: + names.extend(entry["programs"].keys()) + return sorted(names) + + +def _find_in_zip(zf, basename, ostag): + """Find a file in a zip archive matching basename with platform extension. + + Searches for the basename with and without platform-specific extensions, + matching against the filename component (ignoring directory paths within + the archive). + """ + exe_ext = _EXE_EXT.get(ostag, "") + lib_ext = _LIB_EXT.get(ostag, "") + candidates = [basename + exe_ext] + if lib_ext: + candidates.append(basename + lib_ext) + # also try bare name (unix executables have no extension) + if exe_ext: + candidates.append(basename) + + for info in zf.infolist(): + if info.is_dir(): + continue + member_name = Path(info.filename).name + if member_name in candidates: + return info + return None + + +def fetch(manifest, ostag, outdir, zip_path=None): + """Download and extract pre-built programs for the given platform.""" + outdir = Path(outdir) + outdir.mkdir(parents=True, exist_ok=True) + + fetched = [] + for entry in manifest: + repo = entry["repo"] + tag = entry["tag"] + asset = entry["assets"].get(ostag) + if asset is None: + print(f" skip {repo}: no asset for {ostag}") + continue + + url = GITHUB_URL.format(repo=repo, tag=tag, asset=asset) + print(f" downloading {repo} {tag} ({asset})") + + with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as tmp: + tmp_path = tmp.name + try: + urllib.request.urlretrieve(url, tmp_path) + with zipfile.ZipFile(tmp_path) as zf: + for output_name, archive_name in entry["programs"].items(): + info = _find_in_zip(zf, archive_name, ostag) + if info is None: + print(f" warning: {archive_name} not found in {asset}") + continue + + # determine output filename with correct extension + _, ext = os.path.splitext(info.filename) + out_file = output_name + ext + out_path = outdir / out_file + + # extract to temp then move (handles nested paths in archive) + with tempfile.TemporaryDirectory() as extract_dir: + zf.extract(info, extract_dir) + extracted = Path(extract_dir) / info.filename + shutil.copy2(extracted, out_path) + + # ensure executable permission + out_path.chmod( + out_path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + ) + print(f" {out_file}") + fetched.append(out_file) + finally: + os.unlink(tmp_path) + + # add fetched programs to zip if requested + if zip_path and fetched: + mode = "a" if Path(zip_path).exists() else "w" + with zipfile.ZipFile(zip_path, mode, zipfile.ZIP_DEFLATED) as zf: + for fname in fetched: + zf.write(outdir / fname, fname) + print(f" added {len(fetched)} programs to {zip_path}") + + return fetched + + +def main(): + parser = argparse.ArgumentParser( + description="Fetch pre-built executables from GitHub releases" + ) + parser.add_argument( + "--manifest", + default="releases.json", + help="path to releases.json manifest", + ) + parser.add_argument( + "--list", + action="store_true", + help="list program names and exit (for pymake --exclude)", + ) + parser.add_argument("--ostag", help="platform tag (linux, mac, macarm, win64)") + parser.add_argument("--outdir", help="output directory for executables") + parser.add_argument("--zip", dest="zip_path", help="zip file to add programs to") + + args = parser.parse_args() + manifest = _load_manifest(args.manifest) + + if args.list: + print(",".join(_all_program_names(manifest))) + return + + if not args.ostag or not args.outdir: + parser.error("--ostag and --outdir are required for fetch mode") + + fetched = fetch(manifest, args.ostag, args.outdir, args.zip_path) + if not fetched: + print("warning: no programs fetched", file=sys.stderr) + sys.exit(1) + + print(f"fetched {len(fetched)} programs") + + +if __name__ == "__main__": + main() From 73f7ed26c19382f95a3ebe5a8624806dcd675cdd Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Thu, 29 Jan 2026 20:18:47 -0500 Subject: [PATCH 2/3] remove macos-13 jobs --- .github/workflows/integration.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 1d030fb..fa13d85 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ ubuntu-22.04, macos-13, macos-14, windows-2022 ] + os: [ ubuntu-22.04, macos-14, windows-2022 ] defaults: run: shell: bash -l {0} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 28c3572..623e731 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, macos-13, macos-14, windows-2022] + os: [ubuntu-22.04, macos-14, windows-2022] defaults: run: shell: bash From 9351f4f054023304bf5a7683a45c786de32bbad7 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Thu, 29 Jan 2026 21:07:39 -0500 Subject: [PATCH 3/3] fix --- DEVELOPER.md | 4 ++-- releases.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DEVELOPER.md b/DEVELOPER.md index 0d5f12e..479b0c2 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -54,7 +54,7 @@ These programs are downloaded as pre-built binaries via `releases.json`. Their r | mf6, zbud6, libmf6 | [MODFLOW-ORG/modflow6](https://github.com/MODFLOW-ORG/modflow6) | 6.6.3 | Assets named `mf6.6.3_{platform}.zip`, binaries nested in `bin/` subdirectory | | triangle | [MODFLOW-ORG/triangle](https://github.com/MODFLOW-ORG/triangle) | v1.6 | | | gridgen | [MODFLOW-ORG/gridgen](https://github.com/MODFLOW-ORG/gridgen) | v1.0.02 | | -| zonbud3 | [MODFLOW-ORG/zonbud](https://github.com/MODFLOW-ORG/zonbud) | v3.01 | Archive contains `zonbud`, renamed to `zonbud3` | +| zonbud | [MODFLOW-ORG/zonbud](https://github.com/MODFLOW-ORG/zonbud) | v3.01 | | | zonbudusg | [MODFLOW-ORG/zonbudusg](https://github.com/MODFLOW-ORG/zonbudusg) | v1.01 | | | mfusg_gsi | [MODFLOW-ORG/mfusgt](https://github.com/MODFLOW-ORG/mfusgt) | v2.6.0 | Archive contains `mfusgt`, renamed to `mfusg_gsi` | @@ -94,7 +94,7 @@ During a release build, the workflow: 2. Runs pymake to compile the remaining programs, excluding those already fetched. 3. Combines everything into the platform zip alongside pymake-generated metadata. -The fetch script handles platform-specific file extensions (`.exe`, `.dll`, `.dylib`, `.so`) and supports renaming programs when the archive filename differs from the distribution name (e.g., `zonbud` in the archive becomes `zonbud3` in the distribution). +The fetch script handles platform-specific file extensions (`.exe`, `.dll`, `.dylib`, `.so`) and supports renaming programs when the archive filename differs from the distribution name (e.g., `mfusgt` in the archive becomes `mfusg_gsi` in the distribution). ### Adding a program diff --git a/releases.json b/releases.json index 1a78456..1df0931 100644 --- a/releases.json +++ b/releases.json @@ -50,7 +50,7 @@ "win64": "win64.zip" }, "programs": { - "zonbud3": "zonbud" + "zonbud": "zonbud" } }, {