diff --git a/.clang-format b/.clang-format index a80c59bddd..51cf39683a 100644 --- a/.clang-format +++ b/.clang-format @@ -93,6 +93,7 @@ PenaltyBreakString: 1000 PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 60 PointerAlignment: Left +QualifierAlignment: Left ReflowComments: false SortIncludes: true # SortUsingDeclarations: true diff --git a/.clang-format-ignore b/.clang-format-ignore new file mode 100644 index 0000000000..58566f02b1 --- /dev/null +++ b/.clang-format-ignore @@ -0,0 +1,2 @@ +# ignore matlab files +**/*.m diff --git a/.clang-tidy b/.clang-tidy index 23167e93d0..6fc7fd19d9 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,5 +1,5 @@ --- -Checks: 'clang-diagnostic-*,clang-analyzer-*,performance-*,readability-*,bugprone-*,clang-analyzer-*,cppcoreguidelines-*,mpi-*,misc-*,-readability-magic-numbers,-cppcoreguidelines-avoid-magic-numbers,-misc-non-private-member-variables-in-classes,-clang-analyzer-optin.mpi*,-bugprone-exception-escape,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-readability-function-cognitive-complexity,-misc-no-recursion,-bugprone-easily-swappable-parameters,-readability-identifier-length' +Checks: 'clang-diagnostic-*,clang-analyzer-*,performance-*,readability-*,bugprone-*,clang-analyzer-*,cppcoreguidelines-*,mpi-*,misc-*,-readability-magic-numbers,-cppcoreguidelines-avoid-magic-numbers,-misc-non-private-member-variables-in-classes,-clang-analyzer-optin.mpi*,-bugprone-exception-escape,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-readability-function-cognitive-complexity,-misc-no-recursion,-bugprone-easily-swappable-parameters,-readability-identifier-length,-cppcoreguidelines-pro-bounds-avoid-unchecked-container-access' WarningsAsErrors: '' HeaderFilterRegex: '' FormatStyle: file @@ -20,7 +20,7 @@ CheckOptions: value: 'MPI_Comm' - key: misc-include-cleaner.IgnoreHeaders - value: 'adios2/.*;bits/.*' + value: 'adios2/.*;bits/.*;cpptrace/.*;petsc.*\.h' --- Disabled checks and reasons: @@ -29,6 +29,7 @@ These are all basically unavoidable in HPC numeric code: -readability-magic-numbers -cppcoreguidelines-avoid-magic-numbers -cppcoreguidelines-pro-bounds-pointer-arithmetic +-cppcoreguidelines-pro-bounds-avoid-unchecked-container-access -readability-function-cognitive-complexity -bugprone-easily-swappable-parameters -readability-identifier-length diff --git a/.clangd b/.clangd new file mode 100644 index 0000000000..421b7e10d3 --- /dev/null +++ b/.clangd @@ -0,0 +1,6 @@ +Diagnostics: + MissingIncludes: Strict + ClangTidy: + FastCheckFilter: None + Includes: + IgnoreHeader: ["adios2/.*", "bits/.*", "cpptrace/.*", "petsc.*"] diff --git a/.docker/fedora/Dockerfile b/.docker/fedora/Dockerfile index 56445f8cea..7d5c04418d 100644 --- a/.docker/fedora/Dockerfile +++ b/.docker/fedora/Dockerfile @@ -22,18 +22,26 @@ RUN git clone $URL \ && git checkout $COMMIT \ && git submodule update --init --recursive - +ENV HOME=/home/boutuser WORKDIR /home/boutuser/BOUT-dev -RUN cmake -S . -B build -DCMAKE_INSTALL_PREFIX=/opt/bout++/ \ +RUN cmake -S . -B build -DCMAKE_INSTALL_PREFIX=/usr/local/ \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DBOUT_GENERATE_FIELDOPS=OFF \ - -DBOUT_USE_PETSC=ON -DPETSc_ROOT=/opt/petsc \ + -DBOUT_USE_PETSC=ON -DPETSc_ROOT=/usr/local \ -DBOUT_ENABLE_PYTHON=ON \ -DBOUT_USE_SUNDIALS=ON -DSUNDIALS_ROOT=/usr/lib64/$MPI/ -DSUNDIALS_INCLUDE_DIR=/usr/include/$MPI-x86_64/sundials/ \ - $CMAKE_OPTIONS || (cat /home/boutuser/BOUT-dev/build/CMakeFiles/CMake{Output,Error}.log ; exit 1) + $CMAKE_OPTIONS || (cat /home/boutuser/BOUT-dev/build/CMakeFiles/CMake{Output,Error}.log ; exit 1); \ + make -C build -j 2 VERBOSE=1; \ + sudo make -C build install; \ + rm -rf build +# Add unversioned path for python +RUN sudo ln -s /usr/local/lib/python3.* /usr/local/lib/python3.x -RUN make -C build -j 2 -RUN sudo make -C build install +ENV PATH=/usr/local/bin:$PATH \ + LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH \ + PYTHONPATH=/usr/local/lib/python3.x/site-packages/:$PYTHONPATH -RUN find /opt/bout++ +# smoke test +RUN python3 -c 'import boutpp' diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 4990f2586e..43a5292e9d 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,3 +1,21 @@ # Clang-format whole repo d8f14fdddb5ca0fbb32d8e2bf5ac2960d6ac5ce6 ed2117e6d6826a98b6988e2f18c0c34e408563b6 +# CMake formatting +17ac13c28aa3b34a0e46dbe87bb3874f6b25e706 +# Added by the bot +4b010b7634aee1045743be80c268d4644522cd29 +52301380586fdbf890f620c04f689b08d89a6c34 +a71cad2dd6ace5741a754e2ca7daacd4bb094e0e +83cf77923a4c72e44303354923021acf932b4fd2 +2c2402ed59c91164eaff46dee0f79386b7347e9e +05b7c571544c3bcb153fce67d12b9ac48947fc2d +c8f38049359170a34c915e209276238ea66b9a1e +65880e4af16cb95438437bf057c4b9fdb427b142 +d1cfb8abd6aa5c76e6c1a4d7ab20929c65f8afc2 +8d5cb39e03c2644715a50684f8cd0318b4e82676 +ec69e8838be2dde140a915e50506f8e1ce3cb534 +f2bc0488a298f136294c523bc5ab4086d090059b +1b4707187a3a85126338303dc766280b8fb2dc56 +b2e2f3575e68f771be2a4341af5fd14e2870469e +64e4285ec38569f66d31e589a4610cefd16d8076 diff --git a/.github/workflows/auto-formatting.yml b/.github/workflows/auto-formatting.yml new file mode 100644 index 0000000000..8c4c34320f --- /dev/null +++ b/.github/workflows/auto-formatting.yml @@ -0,0 +1,33 @@ +name: format-command + +on: + pull_request: + +jobs: + clang-format: + # Release candidate branches tend to have big PRs which causes all sorts of problems + if: ${{ !endsWith(github.head_ref, '-rc') }} + runs-on: ubuntu-latest + steps: + # Checkout the pull request branch, also include all history + - uses: actions/checkout@v6 + with: + ref: ${{ github.head_ref }} + fetch-depth: 0 + + - name: "Set up Python" + uses: actions/setup-python@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Install dev tools + run: uv sync --only-dev + + - name: Check prek versions are up-to-date + run: uv tool run sync-with-uv --check --diff + + - name: Run everything + run: uv tool run prek run --show-diff-on-failure --from-ref origin/${{ github.base_ref }} diff --git a/.github/workflows/black-fix.yml b/.github/workflows/black-fix.yml deleted file mode 100644 index 5cddbbfac6..0000000000 --- a/.github/workflows/black-fix.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: black - -on: - push: - paths: - - '**.py' - - 'bin/**' - - '**/runtest' - - '**/run' - - 'tests/integrated/test_suite' - - 'tests/requirements/*' - - 'tools/tokamak_grids/**' - -defaults: - run: - shell: bash - -jobs: - black: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v5 - with: - ref: ${{ github.head_ref }} - - - name: Install black - run: | - sudo apt update -y - sudo apt -y install python3-pip python3-setuptools python3-wheel - pip3 install black - - - name: Version - run: | - python3 --version - $HOME/.local/bin/black --version - - - name: Run black - run: | - pwd - ls - $HOME/.local/bin/black tests/ tools/ $(grep -EIlr '^#!.*python.*$' bin/ tests/ tools/ src/ | grep -v _boutpp_build) - - - uses: stefanzweifel/git-auto-commit-action@v7 - with: - commit_message: "Apply black changes" diff --git a/.github/workflows/clang-format.yml b/.github/workflows/clang-format.yml deleted file mode 100644 index 4c1307d7c9..0000000000 --- a/.github/workflows/clang-format.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: clang-format-command - -on: - pull_request: - paths: - - '**.cxx' - - '**.hxx' - -jobs: - clang-format: - # Release candidate branches tend to have big PRs which causes all sorts of problems - if: ${{ !endsWith(github.head_ref, '-rc') }} - runs-on: ubuntu-latest - steps: - # Checkout the pull request branch, also include all history - - uses: actions/checkout@v5 - with: - ref: ${{ github.head_ref }} - fetch-depth: 0 - - # This also installs git-clang-format - - name: Install clang-format - run: sudo apt install clang-format - - - name: Run clang-format - id: format - run: - while ! git clang-format origin/${{ github.base_ref }} ; do git add . ; done - - - name: Commit to the PR branch - uses: stefanzweifel/git-auto-commit-action@v7 - with: - commit_message: "Apply clang-format changes" diff --git a/.github/workflows/clang-tidy-review.yml b/.github/workflows/clang-tidy-review.yml index 3f20db608f..360a162456 100644 --- a/.github/workflows/clang-tidy-review.yml +++ b/.github/workflows/clang-tidy-review.yml @@ -16,12 +16,12 @@ jobs: if: ${{ !endsWith(github.head_ref, '-rc') }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: true - name: Run clang-tidy - uses: ZedThree/clang-tidy-review@v0.21.0 + uses: ZedThree/clang-tidy-review@v0.23.1 id: review with: build_dir: build @@ -46,4 +46,4 @@ jobs: -DBOUT_UPDATE_GIT_SUBMODULE=OFF - name: Upload clang-tidy fixes - uses: ZedThree/clang-tidy-review/upload@v0.21.0 + uses: ZedThree/clang-tidy-review/upload@v0.23.1 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1380d0ea8e..809f59cf9c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,7 +6,7 @@ on: - master - next # Add your branch here if you want containers for it - - db-WIP + - fix3121 - docker-ci env: @@ -26,11 +26,11 @@ jobs: matrix: mpi: [mpich] metric3d: - - name: "With OpenMP" + - name: "With 3D Metrics" cmake: ON - base_prefix: "openmp-" + base_prefix: "" tag_prefix: "3d-" - - name: "Without OpenMP" + - name: "Without 3D Metrics" cmake: OFF base_prefix: "" tag_prefix: "" @@ -52,7 +52,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Log in to the Container registry uses: docker/login-action@master @@ -75,7 +75,7 @@ jobs: build-args: | BASE=${{ matrix.mpi }}-${{ matrix.metric3d.base_prefix }}${{ matrix.config.base_postfix }}-main MPI=${{ matrix.mpi }} - CMAKE_OPTIONS=${{ matrix.config.options }} -DBOUT_ENABLE_METRIC_3D=${{ matrix.metric3d.cmake }} -DBOUT_ENABLE_OPENMP=${{ matrix.metric3d.cmake }} + CMAKE_OPTIONS=${{ matrix.config.options }} -DBOUT_ENABLE_METRIC_3D=${{ matrix.metric3d.cmake }} COMMIT=${{ github.sha }} URL=${{ github.server_url }}/${{ github.repository }} context: .docker/fedora/ diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index fc5e84662c..915265e850 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,7 +17,7 @@ jobs: if: always() steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 submodules: true @@ -57,7 +57,7 @@ jobs: if: always() steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 submodules: true @@ -106,7 +106,7 @@ jobs: if: always() steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 submodules: true diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 314ee196ae..02f018b873 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,6 +30,7 @@ jobs: OMPI_MCA_rmaps_base_oversubscribe: yes PRTE_MCA_rmaps_default_mapping_policy: ":oversubscribe" MPIRUN: mpiexec -np + BOUT_TEST_DOWNLOAD_FLAGS: --retry-on-http-error=502,503,504 --tries 3 strategy: fail-fast: true matrix: @@ -39,7 +40,7 @@ jobs: is_cron: - ${{ github.event_name == 'cron' }} config: - - name: "CMake, PETSc unreleased, ADIOS2" + - name: "PETSc unreleased, ADIOS2, 3D metrics" os: ubuntu-24.04 cmake_options: "-DBUILD_SHARED_LIBS=ON -DBOUT_ENABLE_METRIC_3D=ON @@ -120,22 +121,6 @@ jobs: omp_num_threads: 2 on_cron: false - - name: "CMake, new PETSc" - os: ubuntu-latest - cmake_options: "-DBUILD_SHARED_LIBS=ON - -DBOUT_ENABLE_METRIC_3D=ON - -DBOUT_ENABLE_OPENMP=ON - -DBOUT_USE_PETSC=ON - -DBOUT_USE_SLEPC=ON - -DBOUT_USE_SUNDIALS=ON - -DBOUT_USE_HYPRE=OFF - -DBOUT_ENABLE_PYTHON=ON - -DSUNDIALS_ROOT=/home/runner/local - -DPETSC_DIR=/home/runner/local/petsc - -DSLEPC_DIR=/home/runner/local/slepc" - build_petsc: -petsc - on_cron: false - exclude: - is_cron: true config: @@ -168,21 +153,29 @@ jobs: libparpack2-dev libhypre-dev - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: true - uses: actions/setup-python@v6 with: python-version: '3.x' + cache: 'pip' - name: Install pip packages run: | python -m pip install --upgrade pip setuptools python -m pip install -r requirements.txt + - name: Cache Zenodo test data + uses: actions/cache@v5 + with: + path: build/tests/integrated/test-fci-mpi/grid.fci.nc + # If we update the test, invalidate the cache + key: zenodo-data-${{ hashFiles('tests/integrated/test-fci-mpi/CMakeLists.txt') }} + - name: Cache SUNDIALS build - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: /home/runner/local key: bout-sundials-${{ matrix.config.os }}${{ matrix.config.build_petsc }} @@ -204,39 +197,22 @@ jobs: # standard_tests timeout-minutes: 120 runs-on: ubuntu-latest + env: + BOUT_TEST_DOWNLOAD_FLAGS: --retry-on-http-error=502,503,504 --tries 3 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: true + + - name: Cache Zenodo test data + uses: actions/cache@v5 + with: + path: build/tests/integrated/test-fci-mpi/grid.fci.nc + # If we update the test, invalidate the cache + key: zenodo-data-${{ hashFiles('tests/integrated/test-fci-mpi/CMakeLists.txt') }} + - name: Build Fedora run: ./.ci_fedora.sh setup openmpi shell: bash env: TRAVIS_BUILD_DIR: ${{ github.workspace }} - CUDA: - timeout-minutes: 60 - runs-on: ubuntu-latest - container: ghcr.io/ggeorgakoudis/boutdev-cuda:latest - - steps: - - uses: actions/checkout@v5 - with: - submodules: true - - name: Build minimal CUDA 12.2 @ GCC9.4.0 @ Ubuntu 20.04 - run: | - . /spack/share/spack/setup-env.sh - spack env activate -p /spack-env - git config --global --add safe.directory $GITHUB_WORKSPACE - rm -rf build - cmake -S $GITHUB_WORKSPACE -B build \ - -DCMAKE_C_COMPILER=gcc \ - -DCMAKE_CXX_COMPILER=g++ \ - -DBOUT_ENABLE_RAJA=on \ - -DBOUT_ENABLE_UMPIRE=on \ - -DBOUT_ENABLE_CUDA=on \ - -DCMAKE_CUDA_ARCHITECTURES=80 \ - -DCUDA_ARCH=compute_80,code=sm_80 \ - -DBOUT_ENABLE_WARNINGS=off \ - -DBOUT_USE_SYSTEM_FMT=on - cd build - make -j 4 diff --git a/.gitmodules b/.gitmodules index 4b33b3b615..2bdbd2a57b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "externalpackages/boutdata"] path = externalpackages/boutdata url = https://github.com/boutproject/boutdata.git +[submodule "externalpackages/cpptrace"] + path = externalpackages/cpptrace + url = https://github.com/ZedThree/cpptrace.git diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..84a440847b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,62 @@ +# This file requires prek rather than pre-commit +exclude: + glob: + - "tools/**/*.pro" + - "tools/**/*.m" + - "tools/**/*.[ch]" + +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: builtin + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-added-large-files + +# This would be nice to use, but it is a significant slowdown: + +# # Sync versions of hook tools with `uv.lock` +# - repo: https://github.com/tsvikas/sync-with-uv +# rev: v0.5.0 +# hooks: +# - id: sync-with-uv + +# C++ formatting +- repo: https://github.com/pre-commit/mirrors-clang-format + rev: v22.1.1 + hooks: + - id: clang-format + types_or: [c++, c, cuda] + +# Python linting and formatting +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.6 + hooks: + # Run the linter. + - id: ruff-check + # Run the formatter. + - id: ruff-format + +# CMake formatting +- repo: https://github.com/cheshirekow/cmake-format-precommit + rev: v0.6.13 + hooks: + - id: cmake-format + additional_dependencies: [pyyaml>=5.1] + +# Various config files +- repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.37.0 + hooks: + - id: check-github-workflows + - id: check-readthedocs + - id: check-citation-file-format + - id: check-dependabot + +# Docs linting +- repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v1.0.2 + hooks: + - id: sphinx-lint + files: 'manual/sphinx/.*' diff --git a/CMakeLists.txt b/CMakeLists.txt index 44d100e444..5f054c1c1f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,10 +15,15 @@ if(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.22) cmake_policy(SET CMP0127 NEW) endif() -if ("${CMAKE_CURRENT_BINARY_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") - option(BOUT_ALLOW_INSOURCE_BUILD "Whether BOUT++ should really allow to build in source." OFF) - if (NOT ${BOUT_ALLOW_INSOURCE_BUILD}) - message(FATAL_ERROR "BOUT++ does not recommend in source builds. Try building out of source, e.g. with `cmake -S . -B build` or set -DBOUT_ALLOW_INSOURCE_BUILD=ON - but things may break!") +if("${CMAKE_CURRENT_BINARY_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") + option(BOUT_ALLOW_INSOURCE_BUILD + "Whether BOUT++ should really allow to build in source." OFF + ) + if(NOT ${BOUT_ALLOW_INSOURCE_BUILD}) + message( + FATAL_ERROR + "BOUT++ does not recommend in source builds. Try building out of source, e.g. with `cmake -S . -B build` or set -DBOUT_ALLOW_INSOURCE_BUILD=ON - but things may break!" + ) endif() endif() @@ -29,41 +34,53 @@ set(_bout_previous_version "5.2.0") set(_bout_next_version "5.2.1") execute_process( COMMAND "git" describe --tags --match=v${_bout_previous_version} - COMMAND sed -e s/${_bout_previous_version}-/${_bout_next_version}.dev/ -e s/-/+/ - RESULTS_VARIABLE error_codes + COMMAND sed -e s/${_bout_previous_version}-/${_bout_next_version}.dev/ -e + s/-/+/ RESULTS_VARIABLE error_codes OUTPUT_VARIABLE BOUT_FULL_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE - ) +) foreach(error_code ${error_codes}) - if (NOT ${error_code} STREQUAL 0) + if(NOT ${error_code} STREQUAL 0) set(BOUT_FULL_VERSION ${_bout_next_version}) endif() endforeach() # Remove leading "v" string(REGEX REPLACE "^v(.*)" "\\1" BOUT_FULL_VERSION ${BOUT_FULL_VERSION}) # Remove trailing tag -string(REGEX REPLACE "^([0-9]+\.[0-9]+\.[0-9]+)\..*" "\\1" BOUT_CMAKE_ACCEPTABLE_VERSION ${BOUT_FULL_VERSION}) +string(REGEX REPLACE "^([0-9]+\.[0-9]+\.[0-9]+)\..*" "\\1" + BOUT_CMAKE_ACCEPTABLE_VERSION ${BOUT_FULL_VERSION} +) # Get the trailing tag -string(REGEX REPLACE "^[0-9]+\.[0-9]+\.[0-9]+\.(.*)" "\\1" BOUT_VERSION_TAG ${BOUT_FULL_VERSION}) +string(REGEX REPLACE "^[0-9]+\.[0-9]+\.[0-9]+\.(.*)" "\\1" BOUT_VERSION_TAG + ${BOUT_FULL_VERSION} +) message(STATUS "Configuring BOUT++ version ${BOUT_FULL_VERSION}") -project(BOUT++ +project( + BOUT++ DESCRIPTION "Fluid PDE solver framework" VERSION ${BOUT_CMAKE_ACCEPTABLE_VERSION} - LANGUAGES CXX) + LANGUAGES CXX +) include(CMakeDependentOption) option(BUILD_SHARED_LIBS "Build shared libs" ON) # Override default -option(INSTALL_GTEST "Enable installation of googletest. (Projects embedding googletest may want to turn this OFF.)" OFF) +option( + INSTALL_GTEST + "Enable installation of googletest. (Projects embedding googletest may want to turn this OFF.)" + OFF +) set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) include(BOUT++functions) -option(BOUT_UPDATE_GIT_SUBMODULE "Check submodules are up-to-date during build" ON) +option(BOUT_UPDATE_GIT_SUBMODULE "Check submodules are up-to-date during build" + ON +) # Adapted from https://cliutils.gitlab.io/modern-cmake/chapters/projects/submodule.html # Update submodules as needed function(bout_update_submodules) @@ -73,337 +90,368 @@ function(bout_update_submodules) find_package(Git QUIET) if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git") message(STATUS "Submodule update") - execute_process(COMMAND ${GIT_EXECUTABLE} -c submodule.recurse=false submodule update --init --recursive + execute_process( + COMMAND ${GIT_EXECUTABLE} -c submodule.recurse=false submodule update + --init --recursive WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} - RESULT_VARIABLE GIT_SUBMOD_RESULT) + RESULT_VARIABLE GIT_SUBMOD_RESULT + ) if(NOT GIT_SUBMOD_RESULT EQUAL "0") - message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") + message( + FATAL_ERROR + "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules" + ) endif() endif() endfunction() set(BOUT_SOURCES - ./include/bout/adios_object.hxx - ./include/bout/array.hxx - ./include/bout/assert.hxx - ./include/bout/boundary_factory.hxx - ./include/bout/boundary_op.hxx - ./include/bout/boundary_region.hxx - ./include/bout/boundary_standard.hxx - ./include/bout/bout.hxx - ./include/bout/bout_enum_class.hxx - ./include/bout/bout_types.hxx - ./include/bout/build_config.hxx - ./include/bout/boutcomm.hxx - ./include/bout/boutexception.hxx - ./include/bout/caliper_wrapper.hxx - ./include/bout/constants.hxx - ./include/bout/coordinates.hxx - ./include/bout/coordinates_accessor.hxx - ./include/bout/cyclic_reduction.hxx - ./include/bout/dcomplex.hxx - ./include/bout/deriv_store.hxx - ./include/bout/derivs.hxx - ./include/bout/difops.hxx - ./include/bout/expr.hxx - ./include/bout/fft.hxx - ./include/bout/field.hxx - ./include/bout/field2d.hxx - ./include/bout/field3d.hxx - ./include/bout/field_accessor.hxx - ./include/bout/field_data.hxx - ./include/bout/field_factory.hxx - ./include/bout/fieldgroup.hxx - ./include/bout/fieldperp.hxx - ./include/bout/fv_ops.hxx - ./include/bout/generic_factory.hxx - ./include/bout/globalfield.hxx - ./include/bout/globalindexer.hxx - ./include/bout/globals.hxx - ./include/bout/griddata.hxx - ./include/bout/gyro_average.hxx - ./include/bout/hypre_interface.hxx - ./include/bout/index_derivs.hxx - ./include/bout/index_derivs_interface.hxx - ./include/bout/initialprofiles.hxx - ./include/bout/interpolation.hxx - ./include/bout/interpolation_xz.hxx - ./include/bout/interpolation_z.hxx - ./include/bout/invert/laplacexy.hxx - ./include/bout/invert/laplacexz.hxx - ./include/bout/invert_laplace.hxx - ./include/bout/invert_parderiv.hxx - ./include/bout/invert_pardiv.hxx - ./include/bout/invertable_operator.hxx - ./include/bout/lapack_routines.hxx - ./include/bout/macro_for_each.hxx - ./include/bout/mask.hxx - ./include/bout/mesh.hxx - ./include/bout/monitor.hxx - ./include/bout/mpi_wrapper.hxx - ./include/bout/msg_stack.hxx - ./include/bout/multiostream.hxx - ./include/bout/openmpwrap.hxx - ./include/bout/operatorstencil.hxx - ./include/bout/options.hxx - ./include/bout/options_io.hxx - ./include/bout/optionsreader.hxx - ./include/bout/output.hxx - ./include/bout/output_bout_types.hxx - ./include/bout/parallel_boundary_op.hxx - ./include/bout/parallel_boundary_region.hxx - ./include/bout/paralleltransform.hxx - ./include/bout/petsc_interface.hxx - ./include/bout/petsclib.hxx - ./include/bout/physicsmodel.hxx - ./include/bout/rajalib.hxx - ./include/bout/region.hxx - ./include/bout/rkscheme.hxx - ./include/bout/rvec.hxx - ./include/bout/scorepwrapper.hxx - ./include/bout/single_index_ops.hxx - ./include/bout/slepclib.hxx - ./include/bout/smoothing.hxx - ./include/bout/snb.hxx - ./include/bout/solver.hxx - ./include/bout/solverfactory.hxx - ./include/bout/sourcex.hxx - ./include/bout/stencils.hxx - ./include/bout/sundials_backports.hxx - ./include/bout/surfaceiter.hxx - ./include/bout/sys/expressionparser.hxx - ./include/bout/sys/generator_context.hxx - ./include/bout/sys/gettext.hxx - ./include/bout/sys/range.hxx - ./include/bout/sys/timer.hxx - ./include/bout/sys/type_name.hxx - ./include/bout/sys/uncopyable.hxx - ./include/bout/sys/uuid.h - ./include/bout/sys/variant.hxx - ./include/bout/template_combinations.hxx - ./include/bout/traits.hxx - ./include/bout/unused.hxx - ./include/bout/utils.hxx - ./include/bout/vecops.hxx - ./include/bout/vector2d.hxx - ./include/bout/vector3d.hxx - ./include/bout/where.hxx - ./src/bout++.cxx - ./src/bout++-time.hxx - ./src/field/field.cxx - ./src/field/field2d.cxx - ./src/field/field3d.cxx - ./src/field/field_data.cxx - ./src/field/field_factory.cxx - ./src/field/fieldgenerators.cxx - ./src/field/fieldgenerators.hxx - ./src/field/fieldgroup.cxx - ./src/field/fieldperp.cxx - ./src/field/generated_fieldops.cxx - ./src/field/globalfield.cxx - ./src/field/initialprofiles.cxx - ./src/field/vecops.cxx - ./src/field/vector2d.cxx - ./src/field/vector3d.cxx - ./src/field/where.cxx - ./src/invert/fft_fftw.cxx - ./src/invert/lapack_routines.cxx - ./src/invert/laplace/impls/cyclic/cyclic_laplace.cxx - ./src/invert/laplace/impls/cyclic/cyclic_laplace.hxx - ./src/invert/laplace/impls/iterative_parallel_tri/iterative_parallel_tri.cxx - ./src/invert/laplace/impls/iterative_parallel_tri/iterative_parallel_tri.hxx - ./src/invert/laplace/impls/multigrid/multigrid_alg.cxx - ./src/invert/laplace/impls/multigrid/multigrid_laplace.cxx - ./src/invert/laplace/impls/multigrid/multigrid_laplace.hxx - ./src/invert/laplace/impls/multigrid/multigrid_solver.cxx - ./src/invert/laplace/impls/naulin/naulin_laplace.cxx - ./src/invert/laplace/impls/naulin/naulin_laplace.hxx - ./src/invert/laplace/impls/pcr/pcr.cxx - ./src/invert/laplace/impls/pcr/pcr.hxx - ./src/invert/laplace/impls/pcr_thomas/pcr_thomas.cxx - ./src/invert/laplace/impls/pcr_thomas/pcr_thomas.hxx - ./src/invert/laplace/impls/petsc/petsc_laplace.cxx - ./src/invert/laplace/impls/petsc/petsc_laplace.hxx - ./src/invert/laplace/impls/petsc3damg/petsc3damg.cxx - ./src/invert/laplace/impls/petsc3damg/petsc3damg.hxx - ./src/invert/laplace/impls/serial_band/serial_band.cxx - ./src/invert/laplace/impls/serial_band/serial_band.hxx - ./src/invert/laplace/impls/serial_tri/serial_tri.cxx - ./src/invert/laplace/impls/serial_tri/serial_tri.hxx - ./src/invert/laplace/impls/spt/spt.cxx - ./src/invert/laplace/impls/spt/spt.hxx - ./src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx - ./src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx - ./src/invert/laplace/invert_laplace.cxx - ./src/invert/laplacexy/impls/hypre/laplacexy-hypre.cxx - ./src/invert/laplacexy/impls/hypre/laplacexy-hypre.hxx - ./src/invert/laplacexy/impls/petsc/laplacexy-petsc.cxx - ./src/invert/laplacexy/impls/petsc/laplacexy-petsc.hxx - ./src/invert/laplacexy/impls/petsc2/laplacexy-petsc2.cxx - ./src/invert/laplacexy/impls/petsc2/laplacexy-petsc2.hxx - ./src/invert/laplacexy/laplacexy.cxx - ./src/invert/laplacexz/impls/cyclic/laplacexz-cyclic.cxx - ./src/invert/laplacexz/impls/cyclic/laplacexz-cyclic.hxx - ./src/invert/laplacexz/impls/petsc/laplacexz-petsc.cxx - ./src/invert/laplacexz/impls/petsc/laplacexz-petsc.hxx - ./src/invert/laplacexz/laplacexz.cxx - ./src/invert/parderiv/impls/cyclic/cyclic.cxx - ./src/invert/parderiv/impls/cyclic/cyclic.hxx - ./src/invert/parderiv/invert_parderiv.cxx - ./src/invert/pardiv/impls/cyclic/pardiv_cyclic.cxx - ./src/invert/pardiv/impls/cyclic/pardiv_cyclic.hxx - ./src/invert/pardiv/invert_pardiv.cxx - ./src/mesh/boundary_factory.cxx - ./src/mesh/boundary_region.cxx - ./src/mesh/boundary_standard.cxx - ./src/mesh/coordinates.cxx - ./src/mesh/coordinates_accessor.cxx - ./src/mesh/data/gridfromfile.cxx - ./src/mesh/data/gridfromoptions.cxx - ./src/mesh/difops.cxx - ./src/mesh/fv_ops.cxx - ./src/mesh/impls/bout/boutmesh.cxx - ./src/mesh/impls/bout/boutmesh.hxx - ./src/mesh/index_derivs.cxx - ./src/mesh/interpolation_xz.cxx - ./src/mesh/interpolation/bilinear_xz.cxx - ./src/mesh/interpolation/hermite_spline_xz.cxx - ./src/mesh/interpolation/hermite_spline_z.cxx - ./src/mesh/interpolation/interpolation_z.cxx - ./src/mesh/interpolation/lagrange_4pt_xz.cxx - ./src/mesh/interpolation/monotonic_hermite_spline_xz.cxx - ./src/mesh/invert3x3.hxx - ./src/mesh/mesh.cxx - ./src/mesh/parallel/fci.cxx - ./src/mesh/parallel/fci.hxx - ./src/mesh/parallel/identity.cxx - ./src/mesh/parallel/shiftedmetric.cxx - ./src/mesh/parallel/shiftedmetricinterp.cxx - ./src/mesh/parallel/shiftedmetricinterp.hxx - ./src/mesh/parallel_boundary_op.cxx - ./src/mesh/parallel_boundary_region.cxx - ./src/mesh/surfaceiter.cxx - ./src/physics/gyro_average.cxx - ./src/physics/physicsmodel.cxx - ./src/physics/smoothing.cxx - ./src/physics/snb.cxx - ./src/physics/sourcex.cxx - ./src/solver/impls/adams_bashforth/adams_bashforth.cxx - ./src/solver/impls/adams_bashforth/adams_bashforth.hxx - ./src/solver/impls/arkode/arkode.cxx - ./src/solver/impls/arkode/arkode.hxx - ./src/solver/impls/cvode/cvode.cxx - ./src/solver/impls/cvode/cvode.hxx - ./src/solver/impls/euler/euler.cxx - ./src/solver/impls/euler/euler.hxx - ./src/solver/impls/ida/ida.cxx - ./src/solver/impls/ida/ida.hxx - ./src/solver/impls/imex-bdf2/imex-bdf2.cxx - ./src/solver/impls/imex-bdf2/imex-bdf2.hxx - ./src/solver/impls/petsc/petsc.cxx - ./src/solver/impls/petsc/petsc.hxx - ./src/solver/impls/power/power.cxx - ./src/solver/impls/power/power.hxx - ./src/solver/impls/pvode/pvode.cxx - ./src/solver/impls/pvode/pvode.hxx - ./src/solver/impls/rk3-ssp/rk3-ssp.cxx - ./src/solver/impls/rk3-ssp/rk3-ssp.hxx - ./src/solver/impls/rk4/rk4.cxx - ./src/solver/impls/rk4/rk4.hxx - ./src/solver/impls/rkgeneric/impls/cashkarp/cashkarp.cxx - ./src/solver/impls/rkgeneric/impls/cashkarp/cashkarp.hxx - ./src/solver/impls/rkgeneric/impls/rk4simple/rk4simple.cxx - ./src/solver/impls/rkgeneric/impls/rk4simple/rk4simple.hxx - ./src/solver/impls/rkgeneric/impls/rkf34/rkf34.cxx - ./src/solver/impls/rkgeneric/impls/rkf34/rkf34.hxx - ./src/solver/impls/rkgeneric/impls/rkf45/rkf45.cxx - ./src/solver/impls/rkgeneric/impls/rkf45/rkf45.hxx - ./src/solver/impls/rkgeneric/rkgeneric.cxx - ./src/solver/impls/rkgeneric/rkgeneric.hxx - ./src/solver/impls/rkgeneric/rkscheme.cxx - ./src/solver/impls/slepc/slepc.cxx - ./src/solver/impls/slepc/slepc.hxx - ./src/solver/impls/snes/snes.cxx - ./src/solver/impls/snes/snes.hxx - ./src/solver/impls/split-rk/split-rk.cxx - ./src/solver/impls/split-rk/split-rk.hxx - ./src/solver/solver.cxx - ./src/sys/adios_object.cxx - ./src/sys/bout_types.cxx - ./src/sys/boutcomm.cxx - ./src/sys/boutexception.cxx - ./src/sys/derivs.cxx - ./src/sys/expressionparser.cxx - ./src/sys/generator_context.cxx - ./include/bout/hyprelib.hxx - ./src/sys/hyprelib.cxx - ./src/sys/msg_stack.cxx - ./src/sys/options.cxx - ./src/sys/options/optionparser.hxx - ./src/sys/options/options_ini.cxx - ./src/sys/options/options_ini.hxx - ./src/sys/options/options_io.cxx - ./src/sys/options/options_netcdf.cxx - ./src/sys/options/options_netcdf.hxx - ./src/sys/options/options_adios.cxx - ./src/sys/options/options_adios.hxx - ./src/sys/optionsreader.cxx - ./src/sys/output.cxx - ./src/sys/petsclib.cxx - ./src/sys/range.cxx - ./src/sys/slepclib.cxx - ./src/sys/timer.cxx - ./src/sys/type_name.cxx - ./src/sys/utils.cxx - ${CMAKE_CURRENT_BINARY_DIR}/include/bout/revision.hxx - ${CMAKE_CURRENT_BINARY_DIR}/include/bout/version.hxx - ) - + ./include/bout/adios_object.hxx + ./include/bout/array.hxx + ./include/bout/assert.hxx + ./include/bout/boundary_factory.hxx + ./include/bout/boundary_op.hxx + ./include/bout/boundary_region.hxx + ./include/bout/boundary_standard.hxx + ./include/bout/bout.hxx + ./include/bout/bout_enum_class.hxx + ./include/bout/bout_types.hxx + ./include/bout/build_config.hxx + ./include/bout/boutcomm.hxx + ./include/bout/boutexception.hxx + ./include/bout/caliper_wrapper.hxx + ./include/bout/constants.hxx + ./include/bout/coordinates.hxx + ./include/bout/coordinates_accessor.hxx + ./include/bout/cyclic_reduction.hxx + ./include/bout/dcomplex.hxx + ./include/bout/deriv_store.hxx + ./include/bout/derivs.hxx + ./include/bout/difops.hxx + ./include/bout/expr.hxx + ./include/bout/fft.hxx + ./include/bout/field.hxx + ./include/bout/field2d.hxx + ./include/bout/field3d.hxx + ./include/bout/field_accessor.hxx + ./include/bout/field_data.hxx + ./include/bout/field_factory.hxx + ./include/bout/fieldgroup.hxx + ./include/bout/fieldperp.hxx + ./include/bout/fv_ops.hxx + ./include/bout/generic_factory.hxx + ./include/bout/globalfield.hxx + ./include/bout/globalindexer.hxx + ./include/bout/globals.hxx + ./include/bout/griddata.hxx + ./include/bout/gyro_average.hxx + ./include/bout/hypre_interface.hxx + ./include/bout/index_derivs.hxx + ./include/bout/index_derivs_interface.hxx + ./include/bout/initialprofiles.hxx + ./include/bout/interpolation.hxx + ./include/bout/interpolation_xz.hxx + ./include/bout/interpolation_z.hxx + ./include/bout/invert/laplacexy.hxx + ./include/bout/invert/laplacexz.hxx + ./include/bout/invert_laplace.hxx + ./include/bout/invert_parderiv.hxx + ./include/bout/invert_pardiv.hxx + ./include/bout/invertable_operator.hxx + ./include/bout/lapack_routines.hxx + ./include/bout/macro_for_each.hxx + ./include/bout/mask.hxx + ./include/bout/mesh.hxx + ./include/bout/monitor.hxx + ./include/bout/mpi_wrapper.hxx + ./include/bout/msg_stack.hxx + ./include/bout/multiostream.hxx + ./include/bout/openmpwrap.hxx + ./include/bout/operatorstencil.hxx + ./include/bout/options.hxx + ./include/bout/options_io.hxx + ./include/bout/optionsreader.hxx + ./include/bout/output.hxx + ./include/bout/output_bout_types.hxx + ./include/bout/parallel_boundary_op.hxx + ./include/bout/parallel_boundary_region.hxx + ./include/bout/paralleltransform.hxx + ./include/bout/petsc_interface.hxx + ./include/bout/petsclib.hxx + ./include/bout/physicsmodel.hxx + ./include/bout/rajalib.hxx + ./include/bout/region.hxx + ./include/bout/rkscheme.hxx + ./include/bout/rvec.hxx + ./include/bout/scorepwrapper.hxx + ./include/bout/single_index_ops.hxx + ./include/bout/slepclib.hxx + ./include/bout/smoothing.hxx + ./include/bout/snb.hxx + ./include/bout/solver.hxx + ./include/bout/solverfactory.hxx + ./include/bout/sourcex.hxx + ./include/bout/stencils.hxx + ./include/bout/sundials_backports.hxx + ./include/bout/surfaceiter.hxx + ./include/bout/sys/expressionparser.hxx + ./include/bout/sys/generator_context.hxx + ./include/bout/sys/gettext.hxx + ./include/bout/sys/range.hxx + ./include/bout/sys/timer.hxx + ./include/bout/sys/type_name.hxx + ./include/bout/sys/uncopyable.hxx + ./include/bout/sys/uuid.h + ./include/bout/sys/variant.hxx + ./include/bout/template_combinations.hxx + ./include/bout/traits.hxx + ./include/bout/unused.hxx + ./include/bout/utils.hxx + ./include/bout/vecops.hxx + ./include/bout/vector2d.hxx + ./include/bout/vector3d.hxx + ./include/bout/where.hxx + ./src/bout++.cxx + ./src/bout++-time.hxx + ./src/field/field.cxx + ./src/field/field2d.cxx + ./src/field/field3d.cxx + ./src/field/field_data.cxx + ./src/field/field_factory.cxx + ./src/field/fieldgenerators.cxx + ./src/field/fieldgenerators.hxx + ./src/field/fieldgroup.cxx + ./src/field/fieldperp.cxx + ./src/field/generated_fieldops.cxx + ./src/field/globalfield.cxx + ./src/field/initialprofiles.cxx + ./src/field/vecops.cxx + ./src/field/vector2d.cxx + ./src/field/vector3d.cxx + ./src/field/where.cxx + ./src/invert/fft_fftw.cxx + ./src/invert/lapack_routines.cxx + ./src/invert/laplace/common_transform.cxx + ./src/invert/laplace/common_transform.hxx + ./src/invert/laplace/impls/cyclic/cyclic_laplace.cxx + ./src/invert/laplace/impls/cyclic/cyclic_laplace.hxx + ./src/invert/laplace/impls/iterative_parallel_tri/iterative_parallel_tri.cxx + ./src/invert/laplace/impls/iterative_parallel_tri/iterative_parallel_tri.hxx + ./src/invert/laplace/impls/multigrid/multigrid_alg.cxx + ./src/invert/laplace/impls/multigrid/multigrid_laplace.cxx + ./src/invert/laplace/impls/multigrid/multigrid_laplace.hxx + ./src/invert/laplace/impls/multigrid/multigrid_solver.cxx + ./src/invert/laplace/impls/naulin/naulin_laplace.cxx + ./src/invert/laplace/impls/naulin/naulin_laplace.hxx + ./src/invert/laplace/impls/pcr/pcr.cxx + ./src/invert/laplace/impls/pcr/pcr.hxx + ./src/invert/laplace/impls/pcr_thomas/pcr_thomas.cxx + ./src/invert/laplace/impls/pcr_thomas/pcr_thomas.hxx + ./src/invert/laplace/impls/petsc/petsc_laplace.cxx + ./src/invert/laplace/impls/petsc/petsc_laplace.hxx + ./src/invert/laplace/impls/petsc3damg/petsc3damg.cxx + ./src/invert/laplace/impls/petsc3damg/petsc3damg.hxx + ./src/invert/laplace/impls/serial_band/serial_band.cxx + ./src/invert/laplace/impls/serial_band/serial_band.hxx + ./src/invert/laplace/impls/serial_tri/serial_tri.cxx + ./src/invert/laplace/impls/serial_tri/serial_tri.hxx + ./src/invert/laplace/impls/spt/spt.cxx + ./src/invert/laplace/impls/spt/spt.hxx + ./src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx + ./src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx + ./src/invert/laplace/invert_laplace.cxx + ./src/invert/laplacexy/impls/hypre/laplacexy-hypre.cxx + ./src/invert/laplacexy/impls/hypre/laplacexy-hypre.hxx + ./src/invert/laplacexy/impls/petsc/laplacexy-petsc.cxx + ./src/invert/laplacexy/impls/petsc/laplacexy-petsc.hxx + ./src/invert/laplacexy/impls/petsc2/laplacexy-petsc2.cxx + ./src/invert/laplacexy/impls/petsc2/laplacexy-petsc2.hxx + ./src/invert/laplacexy/laplacexy.cxx + ./src/invert/laplacexz/impls/cyclic/laplacexz-cyclic.cxx + ./src/invert/laplacexz/impls/cyclic/laplacexz-cyclic.hxx + ./src/invert/laplacexz/impls/petsc/laplacexz-petsc.cxx + ./src/invert/laplacexz/impls/petsc/laplacexz-petsc.hxx + ./src/invert/laplacexz/laplacexz.cxx + ./src/invert/parderiv/impls/cyclic/cyclic.cxx + ./src/invert/parderiv/impls/cyclic/cyclic.hxx + ./src/invert/parderiv/invert_parderiv.cxx + ./src/invert/pardiv/impls/cyclic/pardiv_cyclic.cxx + ./src/invert/pardiv/impls/cyclic/pardiv_cyclic.hxx + ./src/invert/pardiv/invert_pardiv.cxx + ./src/mesh/boundary_factory.cxx + ./src/mesh/boundary_region.cxx + ./src/mesh/boundary_standard.cxx + ./src/mesh/coordinates.cxx + ./src/mesh/coordinates_accessor.cxx + ./src/mesh/data/gridfromfile.cxx + ./src/mesh/data/gridfromoptions.cxx + ./src/mesh/difops.cxx + ./src/mesh/fv_ops.cxx + ./src/mesh/impls/bout/boutmesh.cxx + ./src/mesh/impls/bout/boutmesh.hxx + ./src/mesh/index_derivs.cxx + ./src/mesh/interpolation_xz.cxx + ./src/mesh/interpolation/bilinear_xz.cxx + ./src/mesh/interpolation/hermite_spline_xz.cxx + ./src/mesh/interpolation/hermite_spline_z.cxx + ./src/mesh/interpolation/interpolation_z.cxx + ./src/mesh/interpolation/lagrange_4pt_xz.cxx + ./src/mesh/invert3x3.hxx + ./src/mesh/mesh.cxx + ./src/mesh/parallel/fci.cxx + ./src/mesh/parallel/fci.hxx + ./src/mesh/parallel/fci_comm.cxx + ./src/mesh/parallel/fci_comm.hxx + ./src/mesh/parallel/identity.cxx + ./src/mesh/parallel/shiftedmetric.cxx + ./src/mesh/parallel/shiftedmetricinterp.cxx + ./src/mesh/parallel/shiftedmetricinterp.hxx + ./src/mesh/parallel_boundary_op.cxx + ./src/mesh/parallel_boundary_region.cxx + ./src/mesh/surfaceiter.cxx + ./src/physics/gyro_average.cxx + ./src/physics/physicsmodel.cxx + ./src/physics/smoothing.cxx + ./src/physics/snb.cxx + ./src/physics/sourcex.cxx + ./src/solver/impls/adams_bashforth/adams_bashforth.cxx + ./src/solver/impls/adams_bashforth/adams_bashforth.hxx + ./src/solver/impls/arkode/arkode.cxx + ./src/solver/impls/arkode/arkode.hxx + ./src/solver/impls/cvode/cvode.cxx + ./src/solver/impls/cvode/cvode.hxx + ./src/solver/impls/euler/euler.cxx + ./src/solver/impls/euler/euler.hxx + ./src/solver/impls/ida/ida.cxx + ./src/solver/impls/ida/ida.hxx + ./src/solver/impls/imex-bdf2/imex-bdf2.cxx + ./src/solver/impls/imex-bdf2/imex-bdf2.hxx + ./src/solver/impls/petsc/petsc.cxx + ./src/solver/impls/petsc/petsc.hxx + ./src/solver/impls/power/power.cxx + ./src/solver/impls/power/power.hxx + ./src/solver/impls/pvode/pvode.cxx + ./src/solver/impls/pvode/pvode.hxx + ./src/solver/impls/rk3-ssp/rk3-ssp.cxx + ./src/solver/impls/rk3-ssp/rk3-ssp.hxx + ./src/solver/impls/rk4/rk4.cxx + ./src/solver/impls/rk4/rk4.hxx + ./src/solver/impls/rkgeneric/impls/cashkarp/cashkarp.cxx + ./src/solver/impls/rkgeneric/impls/cashkarp/cashkarp.hxx + ./src/solver/impls/rkgeneric/impls/rk4simple/rk4simple.cxx + ./src/solver/impls/rkgeneric/impls/rk4simple/rk4simple.hxx + ./src/solver/impls/rkgeneric/impls/rkf34/rkf34.cxx + ./src/solver/impls/rkgeneric/impls/rkf34/rkf34.hxx + ./src/solver/impls/rkgeneric/impls/rkf45/rkf45.cxx + ./src/solver/impls/rkgeneric/impls/rkf45/rkf45.hxx + ./src/solver/impls/rkgeneric/rkgeneric.cxx + ./src/solver/impls/rkgeneric/rkgeneric.hxx + ./src/solver/impls/rkgeneric/rkscheme.cxx + ./src/solver/impls/slepc/slepc.cxx + ./src/solver/impls/slepc/slepc.hxx + ./src/solver/impls/snes/snes.cxx + ./src/solver/impls/snes/snes.hxx + ./src/solver/impls/split-rk/split-rk.cxx + ./src/solver/impls/split-rk/split-rk.hxx + ./src/solver/solver.cxx + ./src/sys/adios_object.cxx + ./src/sys/bout_types.cxx + ./src/sys/boutcomm.cxx + ./src/sys/boutexception.cxx + ./src/sys/derivs.cxx + ./src/sys/expressionparser.cxx + ./src/sys/generator_context.cxx + ./include/bout/hyprelib.hxx + ./src/sys/hyprelib.cxx + ./src/sys/msg_stack.cxx + ./src/sys/options.cxx + ./src/sys/options/optionparser.hxx + ./src/sys/options/options_ini.cxx + ./src/sys/options/options_ini.hxx + ./src/sys/options/options_io.cxx + ./src/sys/options/options_netcdf.cxx + ./src/sys/options/options_netcdf.hxx + ./src/sys/options/options_adios.cxx + ./src/sys/options/options_adios.hxx + ./src/sys/optionsreader.cxx + ./src/sys/output.cxx + ./src/sys/output_bout_types.cxx + ./src/sys/petsclib.cxx + ./src/sys/range.cxx + ./src/sys/slepclib.cxx + ./src/sys/timer.cxx + ./src/sys/type_name.cxx + ./src/sys/utils.cxx + ${CMAKE_CURRENT_BINARY_DIR}/include/bout/revision.hxx + ${CMAKE_CURRENT_BINARY_DIR}/include/bout/version.hxx +) find_package(Python3) find_package(ClangFormat) -if (Python3_FOUND AND ClangFormat_FOUND) +if(Python3_FOUND AND ClangFormat_FOUND) set(BOUT_GENERATE_FIELDOPS_DEFAULT ON) else() set(BOUT_GENERATE_FIELDOPS_DEFAULT OFF) endif() -execute_process(COMMAND ${Python3_EXECUTABLE} -c "import importlib.util ; import sys; sys.exit(importlib.util.find_spec(\"zoidberg\") is None)" - RESULT_VARIABLE zoidberg_FOUND) -if (zoidberg_FOUND EQUAL 0) +execute_process( + COMMAND + ${Python3_EXECUTABLE} -c + "import importlib.util ; import sys; sys.exit(importlib.util.find_spec(\"zoidberg\") is None)" + RESULT_VARIABLE zoidberg_FOUND +) +if(zoidberg_FOUND EQUAL 0) set(zoidberg_FOUND ON) else() set(zoidberg_FOUND OFF) endif() +message(STATUS "Found Zoidberg for FCI tests: ${zoidberg_FOUND}") -option(BOUT_GENERATE_FIELDOPS "Automatically re-generate the Field arithmetic operators from the Python templates. \ +option( + BOUT_GENERATE_FIELDOPS + "Automatically re-generate the Field arithmetic operators from the Python templates. \ Requires Python3, clang-format, and Jinja2. Turn this OFF to skip generating them if, for example, \ -you are unable to install the Jinja2 Python module. This is only important for BOUT++ developers." ${BOUT_GENERATE_FIELDOPS_DEFAULT}) +you are unable to install the Jinja2 Python module. This is only important for BOUT++ developers." + ${BOUT_GENERATE_FIELDOPS_DEFAULT} +) -if (BOUT_GENERATE_FIELDOPS) - if (NOT Python3_FOUND) - message(FATAL_ERROR "python not found, but you have requested to generate code!") +if(BOUT_GENERATE_FIELDOPS) + if(NOT Python3_FOUND) + message( + FATAL_ERROR "python not found, but you have requested to generate code!" + ) endif() - if (NOT ClangFormat_FOUND) - message(FATAL_ERROR "clang-format not found, but you have requested to generate code!") + if(NOT ClangFormat_FOUND) + message( + FATAL_ERROR + "clang-format not found, but you have requested to generate code!" + ) endif() - add_custom_command( OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/src/field/generated_fieldops.cxx - COMMAND ${Python3_EXECUTABLE} gen_fieldops.py --filename generated_fieldops.cxx.tmp + add_custom_command( + OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/src/field/generated_fieldops.cxx + COMMAND ${Python3_EXECUTABLE} gen_fieldops.py --filename + generated_fieldops.cxx.tmp COMMAND ${ClangFormat_BIN} generated_fieldops.cxx.tmp -i - COMMAND ${CMAKE_COMMAND} -E rename generated_fieldops.cxx.tmp generated_fieldops.cxx - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/src/field/gen_fieldops.jinja ${CMAKE_CURRENT_SOURCE_DIR}/src/field/gen_fieldops.py + COMMAND ${CMAKE_COMMAND} -E rename generated_fieldops.cxx.tmp + generated_fieldops.cxx + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/src/field/gen_fieldops.jinja + ${CMAKE_CURRENT_SOURCE_DIR}/src/field/gen_fieldops.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/field/ - COMMENT "Generating source code" ) + COMMENT "Generating source code" + ) else() - message(AUTHOR_WARNING "'src/field/generated_fieldops.cxx' will not be \ + message( + AUTHOR_WARNING + "'src/field/generated_fieldops.cxx' will not be \ regenerated when you make changes to either \ 'src/field/gen_fieldops.py' or 'src/field/gen_fieldops.jinja'. \ This is because either Python3 or clang-format is missing \ (see above messages for more information) \ or BOUT_GENERATE_FIELDOPS is OFF (current value: ${BOUT_GENERATE_FIELDOPS}). \ This warning is only important for BOUT++ developers and can otherwise be \ -safely ignored.") +safely ignored." + ) endif() include(GNUInstallDirs) @@ -420,21 +468,29 @@ set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") # which point to directories outside the build tree to the install RPATH set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) - -execute_process(COMMAND ${Python3_EXECUTABLE} -c "import site ; print('/'.join(site.getusersitepackages().split('/')[-2:]))" +execute_process( + COMMAND + ${Python3_EXECUTABLE} -c + "import site ; print('/'.join(site.getusersitepackages().split('/')[-2:]))" RESULT_VARIABLE PYTHON_WORKING OUTPUT_VARIABLE PYTHON_SITEPATH_SUFFIX OUTPUT_STRIP_TRAILING_WHITESPACE ) -set(CMAKE_INSTALL_PYTHON_SITEARCH lib/${PYTHON_SITEPATH_SUFFIX} CACHE STRING "Location to install python arch-specific modules") +set(CMAKE_INSTALL_PYTHON_SITEARCH + lib/${PYTHON_SITEPATH_SUFFIX} + CACHE STRING "Location to install python arch-specific modules" +) set(ON_OFF_AUTO ON OFF AUTO) -set(BOUT_ENABLE_PYTHON AUTO CACHE STRING "Build the Python interface") +set(BOUT_ENABLE_PYTHON + AUTO + CACHE STRING "Build the Python interface" +) set_property(CACHE BOUT_ENABLE_PYTHON PROPERTY STRINGS ${ON_OFF_AUTO}) -if (NOT BOUT_ENABLE_PYTHON IN_LIST ON_OFF_AUTO) +if(NOT BOUT_ENABLE_PYTHON IN_LIST ON_OFF_AUTO) message(FATAL_ERROR "BOUT_ENABLE_PYTHON must be one of ${ON_OFF_AUTO}") endif() -if (BOUT_ENABLE_PYTHON OR BOUT_ENABLE_PYTHON STREQUAL "AUTO") +if(BOUT_ENABLE_PYTHON OR BOUT_ENABLE_PYTHON STREQUAL "AUTO") add_subdirectory(tools/pylib/_boutpp_build) else() set(BOUT_ENABLE_PYTHON OFF) @@ -444,35 +500,37 @@ set(BOUT_USE_PYTHON ${BOUT_ENABLE_PYTHON}) # Ensure that the compile date/time is up-to-date when any of the sources change add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bout++-time.cxx - COMMAND ${CMAKE_COMMAND} -P "${CMAKE_CURRENT_LIST_DIR}/cmake/GenerateDateTimeFile.cmake" + COMMAND ${CMAKE_COMMAND} -P + "${CMAKE_CURRENT_LIST_DIR}/cmake/GenerateDateTimeFile.cmake" DEPENDS ${BOUT_SOURCES} MAIN_DEPENDENCY "${CMAKE_CURRENT_LIST_DIR}/cmake/GenerateDateTimeFile.cmake" - ) - +) -add_library(bout++ - ${BOUT_SOURCES} - ${CMAKE_CURRENT_BINARY_DIR}/bout++-time.cxx - ) +add_library(bout++ ${BOUT_SOURCES} ${CMAKE_CURRENT_BINARY_DIR}/bout++-time.cxx) add_library(bout++::bout++ ALIAS bout++) target_link_libraries(bout++ PUBLIC MPI::MPI_CXX) -target_include_directories(bout++ PUBLIC - $ - $ - $ - ) +target_include_directories( + bout++ + PUBLIC $ + $ + $ +) set(BOUT_LIB_PATH "${CMAKE_CURRENT_BINARY_DIR}/lib") -set_target_properties(bout++ PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${BOUT_LIB_PATH}" - ARCHIVE_OUTPUT_DIRECTORY "${BOUT_LIB_PATH}" - SOVERSION 5.2.0) +set_target_properties( + bout++ + PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${BOUT_LIB_PATH}" + ARCHIVE_OUTPUT_DIRECTORY "${BOUT_LIB_PATH}" + SOVERSION 5.2.0 +) # Set some variables for the bout-config script set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} -L\$BOUT_LIB_PATH -lbout++") set(BOUT_INCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/include") -set(CONFIG_CFLAGS "${CONFIG_CFLAGS} -I\${BOUT_INCLUDE_PATH} -I${CMAKE_CURRENT_BINARY_DIR}/include ${CMAKE_CXX_FLAGS} -std=c++17") +set(CONFIG_CFLAGS + "${CONFIG_CFLAGS} -I\${BOUT_INCLUDE_PATH} -I${CMAKE_CURRENT_BINARY_DIR}/include ${CMAKE_CXX_FLAGS} -std=c++20" +) -target_compile_features(bout++ PUBLIC cxx_std_17) +target_compile_features(bout++ PUBLIC cxx_std_20) set_target_properties(bout++ PROPERTIES CXX_EXTENSIONS OFF) # Optional compiler features @@ -504,58 +562,77 @@ message(STATUS "Git revision: ${BOUT_REVISION}") # Build the file containing the version information configure_file( "${PROJECT_SOURCE_DIR}/include/bout/version.hxx.in" - "${PROJECT_BINARY_DIR}/include/bout/version.hxx") + "${PROJECT_BINARY_DIR}/include/bout/version.hxx" +) # Build the file containing just the commit hash # This will be rebuilt on every commit! configure_file( "${PROJECT_SOURCE_DIR}/include/bout/revision.hxx.in" - "${PROJECT_BINARY_DIR}/include/bout/revision.hxx") + "${PROJECT_BINARY_DIR}/include/bout/revision.hxx" +) ################################################## option(BOUT_ENABLE_WARNINGS "Enable compiler warnings" ON) -if (BOUT_ENABLE_WARNINGS) - target_compile_options(bout++ PRIVATE - $<$>: - $<$,$,$>: - -Wall -Wextra > > - $<$: - /W4 > - $<$:-Xcompiler=-Wall -Xcompiler=-Wextra > - ) - - include(EnableCXXWarningIfSupport) - # Note we explicitly turn off -Wcast-function-type as PETSc *requires* - # we cast a function to the wrong type in MatFDColoringSetFunction - target_enable_cxx_warning_if_supported(bout++ - FLAGS -Wnull-dereference -Wno-cast-function-type - ) +if(BOUT_ENABLE_WARNINGS) + target_compile_options( + bout++ + PRIVATE + $<$>: + $<$,$,$>: + -Wall + -Wextra + > + > + $<$: + /W4 + > + $<$:-Xcompiler=-Wall + -Xcompiler=-Wextra + > + ) + + include(EnableCXXWarningIfSupport) + # Note we explicitly turn off -Wcast-function-type as PETSc *requires* + # we cast a function to the wrong type in MatFDColoringSetFunction + target_enable_cxx_warning_if_supported( + bout++ FLAGS -Wnull-dereference -Wno-cast-function-type + ) endif() # Compile time features set(CHECK_LEVELS 0 1 2 3 4) -set(CHECK 2 CACHE STRING "Set run-time checking level") +set(CHECK + 2 + CACHE STRING "Set run-time checking level" +) set_property(CACHE CHECK PROPERTY STRINGS ${CHECK_LEVELS}) -if (NOT CHECK IN_LIST CHECK_LEVELS) +if(NOT CHECK IN_LIST CHECK_LEVELS) message(FATAL_ERROR "CHECK must be one of ${CHECK_LEVELS}") endif() message(STATUS "Runtime checking level: CHECK=${CHECK}") target_compile_definitions(bout++ PUBLIC "CHECK=${CHECK}") set(BOUT_CHECK_LEVEL ${CHECK}) -if (CHECK GREATER 1) +if(CHECK GREATER 1) set(bout_use_msgstack_default ON) else() set(bout_use_msgstack_default OFF) endif() -set(BOUT_ENABLE_MSGSTACK ${bout_use_msgstack_default} CACHE BOOL "Enable debug message stack") +set(BOUT_ENABLE_MSGSTACK + ${bout_use_msgstack_default} + CACHE BOOL "Enable debug message stack" +) message(STATUS "Message stack: BOUT_USE_MSGSTACK=${BOUT_ENABLE_MSGSTACK}") set(BOUT_USE_MSGSTACK ${BOUT_ENABLE_MSGSTACK}) -cmake_dependent_option(BOUT_ENABLE_OUTPUT_DEBUG "Enable extra debug output" OFF - "CHECK LESS 3" ON) -message(STATUS "Extra debug output: BOUT_USE_OUTPUT_DEBUG=${BOUT_ENABLE_OUTPUT_DEBUG}") +cmake_dependent_option( + BOUT_ENABLE_OUTPUT_DEBUG "Enable extra debug output" OFF "CHECK LESS 3" ON +) +message( + STATUS "Extra debug output: BOUT_USE_OUTPUT_DEBUG=${BOUT_ENABLE_OUTPUT_DEBUG}" +) set(BOUT_USE_OUTPUT_DEBUG ${BOUT_ENABLE_OUTPUT_DEBUG}) option(BOUT_ENABLE_SIGNAL "SegFault handling" ON) @@ -571,21 +648,12 @@ message(STATUS "Field name tracking: BOUT_USE_TRACK=${BOUT_ENABLE_TRACK}") set(BOUT_USE_TRACK ${BOUT_ENABLE_TRACK}) option(BOUT_ENABLE_SIGFPE "Signalling floating point exceptions" OFF) -message(STATUS "Signalling floating point exceptions: BOUT_USE_SIGFPE=${BOUT_ENABLE_SIGFPE}") +message( + STATUS + "Signalling floating point exceptions: BOUT_USE_SIGFPE=${BOUT_ENABLE_SIGFPE}" +) set(BOUT_USE_SIGFPE ${BOUT_ENABLE_SIGFPE}) -option(BOUT_ENABLE_BACKTRACE "Enable backtrace" ON) -if (BOUT_ENABLE_BACKTRACE) - find_program(ADDR2LINE_FOUND addr2line) - if (NOT ADDR2LINE_FOUND) - message(FATAL_ERROR "addr2line not found. Disable backtrace by setting BOUT_ENABLE_BACKTRACE=Off") - endif() - target_link_libraries(bout++ PUBLIC ${CMAKE_DL_LIBS}) - set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} -l${CMAKE_DL_LIBS}") -endif() -message(STATUS "Enable backtrace: BOUT_USE_BACKTRACE=${BOUT_ENABLE_BACKTRACE}") -set(BOUT_USE_BACKTRACE ${BOUT_ENABLE_BACKTRACE}) - option(BOUT_ENABLE_METRIC_3D "Enable 3D metric support" OFF) if(BOUT_ENABLE_METRIC_3D) set(BOUT_METRIC_TYPE "3D") @@ -595,12 +663,15 @@ endif() set(BOUT_USE_METRIC_3D ${BOUT_ENABLE_METRIC_3D}) include(CheckCXXSourceCompiles) -check_cxx_source_compiles("int main() { const char* name = __PRETTY_FUNCTION__; }" - HAS_PRETTY_FUNCTION) +check_cxx_source_compiles( + "int main() { const char* name = __PRETTY_FUNCTION__; }" HAS_PRETTY_FUNCTION +) set(BOUT_HAS_PRETTY_FUNCTION ${HAS_PRETTY_FUNCTION}) # Locations of the various Python modules, including the generated boutconfig module -set(BOUT_PYTHONPATH "${CMAKE_CURRENT_BINARY_DIR}/tools/pylib:${CMAKE_CURRENT_SOURCE_DIR}/tools/pylib") +set(BOUT_PYTHONPATH + "${CMAKE_CURRENT_BINARY_DIR}/tools/pylib:${CMAKE_CURRENT_SOURCE_DIR}/tools/pylib" +) # Variables for boutconfig module -- note that these will contain # generator expressions and CMake targets, and not generally be very # useful @@ -613,18 +684,22 @@ get_target_property(BOUT_CFLAGS bout++ INTERFACE_INCLUDE_DIRECTORIES) # 1. Get the macro definitions. They come as a ;-separated list and # without the -D. We also need to also stick a -D on the front of # the first item -get_property(BOUT_COMPILE_DEFINITIONS +get_property( + BOUT_COMPILE_DEFINITIONS TARGET bout++ - PROPERTY COMPILE_DEFINITIONS) + PROPERTY COMPILE_DEFINITIONS +) string(REPLACE ";" " -D" BOUT_COMPILE_DEFINITIONS "${BOUT_COMPILE_DEFINITIONS}") string(CONCAT BOUT_COMPILE_DEFINITIONS " -D" "${BOUT_COMPILE_DEFINITIONS}") # 2. Get the compiler options. Again, they come as a ;-separated # list. Note that they don't include optimisation or debug flags: # they're in the CMAKE_CXX_FLAGS* variables -get_property(BOUT_COMPILE_OPTIONS +get_property( + BOUT_COMPILE_OPTIONS TARGET bout++ - PROPERTY COMPILE_OPTIONS) + PROPERTY COMPILE_OPTIONS +) string(REPLACE ";" " " BOUT_COMPILE_OPTIONS "${BOUT_COMPILE_OPTIONS}") # 3. The optimisation and/or debug flags are in the CMAKE_CXX_FLAGS* @@ -636,31 +711,33 @@ string(REPLACE ";" " " BOUT_COMPILE_OPTIONS "${BOUT_COMPILE_OPTIONS}") include(BuildType) # Here CMAKE_BUILD_TYPE is always set string(TOUPPER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_UPPER) -string(CONCAT BOUT_COMPILE_BUILD_FLAGS - " " - "${CMAKE_CXX_FLAGS}" - "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}") +string(CONCAT BOUT_COMPILE_BUILD_FLAGS " " "${CMAKE_CXX_FLAGS}" + "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}" +) # 4. Now we join all the flags from the first three steps together -string(CONCAT BOUT_FLAGS_STRING - "${BOUT_COMPILE_OPTIONS}" - "${BOUT_COMPILE_DEFINITIONS}" - "${BOUT_COMPILE_BUILD_FLAGS}") +string(CONCAT BOUT_FLAGS_STRING "${BOUT_COMPILE_OPTIONS}" + "${BOUT_COMPILE_DEFINITIONS}" "${BOUT_COMPILE_BUILD_FLAGS}" +) # 5. Finally actually add the flags as a define -target_compile_definitions(bout++ - PRIVATE BOUT_FLAGS_STRING=${BOUT_FLAGS_STRING}) +target_compile_definitions( + bout++ PRIVATE BOUT_FLAGS_STRING=${BOUT_FLAGS_STRING} +) ################################################## # Tests # Are we building BOUT++ directly, or as part of another project -string(COMPARE EQUAL - "${PROJECT_NAME}" "${CMAKE_PROJECT_NAME}" - PROJECT_IS_TOP_LEVEL +string(COMPARE EQUAL "${PROJECT_NAME}" "${CMAKE_PROJECT_NAME}" + PROJECT_IS_TOP_LEVEL ) option(BOUT_TESTS "Build the tests" ${PROJECT_IS_TOP_LEVEL}) -option(BOUT_ENABLE_ALL_TESTS "Enable running all of the tests, rather then the standard selection of fast tests" OFF) +option( + BOUT_ENABLE_ALL_TESTS + "Enable running all of the tests, rather then the standard selection of fast tests" + OFF +) if(BOUT_TESTS) enable_testing() # Targets for just building the tests @@ -671,31 +748,35 @@ if(BOUT_TESTS) # Build all the tests add_custom_target(build-check) - add_dependencies(build-check build-check-unit-tests build-check-integrated-tests build-check-mms-tests) + add_dependencies( + build-check build-check-unit-tests build-check-integrated-tests + build-check-mms-tests + ) add_subdirectory(tests/unit EXCLUDE_FROM_ALL) add_subdirectory(tests/integrated EXCLUDE_FROM_ALL) add_subdirectory(tests/MMS EXCLUDE_FROM_ALL) # Targets for running the tests - if (BOUT_ENABLE_UNIT_TESTS) - add_custom_target(check-unit-tests - COMMAND ctest -R serial_tests --output-on-failure) + if(BOUT_ENABLE_UNIT_TESTS) + add_custom_target( + check-unit-tests COMMAND ctest -R serial_tests --output-on-failure + ) add_dependencies(check-unit-tests build-check-unit-tests) endif() - add_custom_target(check-integrated-tests - COMMAND ctest -R "test-" --output-on-failure) + add_custom_target( + check-integrated-tests COMMAND ctest -R "test-" --output-on-failure + ) add_dependencies(check-integrated-tests build-check-integrated-tests) - add_custom_target(check-mms-tests - COMMAND ctest -R "MMS-" --output-on-failure) + add_custom_target(check-mms-tests COMMAND ctest -R "MMS-" --output-on-failure) add_dependencies(check-mms-tests build-check-mms-tests) # Run all the tests add_custom_target(check) add_dependencies(check check-integrated-tests check-mms-tests) - if (BOUT_ENABLE_UNIT_TESTS) + if(BOUT_ENABLE_UNIT_TESTS) add_dependencies(check check-unit-tests) endif() @@ -707,21 +788,22 @@ if(BOUT_BUILD_EXAMPLES) add_subdirectory(examples EXCLUDE_FROM_ALL) endif() - ################################################## # L10N: localisation - include translations find_package(Gettext) -if (GETTEXT_FOUND) +if(GETTEXT_FOUND) #add_custom_target(mofiles ALL) set(bout_langs es de fr zh_CN zh_TW) foreach(_lang IN LISTS bout_langs) set(_gmoFile ${CMAKE_CURRENT_BINARY_DIR}/locale/${_lang}/libbout.gmo) set(_poFile ${CMAKE_CURRENT_SOURCE_DIR}/locale/${_lang}/libbout.po) - add_custom_command(OUTPUT ${_gmoFile} _mo_file_${_lang} - COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/locale/${_lang}/ + add_custom_command( + OUTPUT ${_gmoFile} _mo_file_${_lang} + COMMAND ${CMAKE_COMMAND} -E make_directory + ${CMAKE_CURRENT_BINARY_DIR}/locale/${_lang}/ COMMAND ${GETTEXT_MSGFMT_EXECUTABLE} -o ${_gmoFile} ${_poFile} WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" DEPENDS ${_poFile} @@ -729,41 +811,46 @@ if (GETTEXT_FOUND) list(APPEND _gmoFiles ${_gmoFile}) - install(FILES ${_gmoFile} DESTINATION ${CMAKE_INSTALL_LOCALEDIR}/${_lang}/LC_MESSAGES/ RENAME libbout.mo) + install( + FILES ${_gmoFile} + DESTINATION ${CMAKE_INSTALL_LOCALEDIR}/${_lang}/LC_MESSAGES/ + RENAME libbout.mo + ) endforeach() - add_custom_target(mofiles ALL - DEPENDS ${_gmoFiles}) + add_custom_target(mofiles ALL DEPENDS ${_gmoFiles}) endif() - ################################################## # Documentation option(BOUT_BUILD_DOCS "Build the documentation" OFF) -if (BOUT_BUILD_DOCS) - add_subdirectory(manual EXCLUDE_FROM_ALL) +if(BOUT_BUILD_DOCS) + add_subdirectory(manual) endif() - -add_custom_target(dist - COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/pylib/_boutpp_build/backend.py dist +add_custom_target( + dist + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_SOURCE_DIR}/tools/pylib/_boutpp_build/backend.py dist # there is no cmake equivalent to `mv` - so only works on systems that are not inentionally non-POSIX complient COMMAND mv BOUT++-v${BOUT_FULL_VERSION}.tar.gz ${CMAKE_CURRENT_BINARY_DIR}/ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - ) +) ################################################## # Generate the build config header -if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/include/bout/build_defines.hxx") +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/include/bout/build_defines.hxx") # If we do in source builds, this is fine - if (NOT ${CMAKE_CURRENT_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR}) - message(FATAL_ERROR "Generated build_defines.hxx header already exists; please remove '${CMAKE_CURRENT_SOURCE_DIR}/include/bout/build_defines.hxx' before continuing") + if(NOT ${CMAKE_CURRENT_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR}) + message( + FATAL_ERROR + "Generated build_defines.hxx header already exists; please remove '${CMAKE_CURRENT_SOURCE_DIR}/include/bout/build_defines.hxx' before continuing" + ) endif() endif() configure_file(cmake_build_defines.hxx.in include/bout/build_defines.hxx) - ################################################## # Generate the bout-config script @@ -775,9 +862,11 @@ set(BOUT_HAS_PNETCDF OFF) # For shared libraries we only need to know how to link against BOUT++, # while for static builds we need the dependencies too -if (BUILD_SHARED_LIBS) +if(BUILD_SHARED_LIBS) # Include rpath linker flag so user doesn't need to set LD_LIBRARY_PATH - set(CONFIG_LDFLAGS "${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG}\$BOUT_LIB_PATH -L\$BOUT_LIB_PATH -lbout++ -lfmt ${CONFIG_LDFLAGS_SHARED}") + set(CONFIG_LDFLAGS + "${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG}\$BOUT_LIB_PATH -L\$BOUT_LIB_PATH -lbout++ -lfmt ${CONFIG_LDFLAGS_SHARED}" + ) else() set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS}") endif() @@ -786,9 +875,9 @@ set(ISINSTALLED "FALSE") set(_CONFIG_LDFLAGS) string(REPLACE " " ";" CONFIG_LDFLAGS_LIST ${CONFIG_LDFLAGS}) -foreach (flag ${CONFIG_LDFLAGS_LIST}) +foreach(flag ${CONFIG_LDFLAGS_LIST}) string(REGEX MATCH "^-.*$" isopt "${flag}") - if (isopt) + if(isopt) # message("${flag} is an option") set(_CONFIG_LDFLAGS "${_CONFIG_LDFLAGS} ${flag}") # All good @@ -798,70 +887,93 @@ foreach (flag ${CONFIG_LDFLAGS_LIST}) set(_CONFIG_LDFLAGS "${_CONFIG_LDFLAGS} ${flag}") else() string(FIND "${flag}" "::" hascolcol) - if (${hascolcol} EQUAL -1) - message("Fixing ${flag} to -l${flag}") - set(_CONFIG_LDFLAGS "${_CONFIG_LDFLAGS} -l${flag}") + if(${hascolcol} EQUAL -1) + message("Fixing ${flag} to -l${flag}") + set(_CONFIG_LDFLAGS "${_CONFIG_LDFLAGS} -l${flag}") else() - string(REGEX MATCH "[^:]*$" flag2 "${flag}") - message("Fixing ${flag} to -l${flag2}") - set(_CONFIG_LDFLAGS "${_CONFIG_LDFLAGS} -l${flag2}") + string(REGEX MATCH "[^:]*$" flag2 "${flag}") + message("Fixing ${flag} to -l${flag2}") + set(_CONFIG_LDFLAGS "${_CONFIG_LDFLAGS} -l${flag2}") endif() endif() endif() endforeach() -set( CONFIG_LDFLAGS ${_CONFIG_LDFLAGS}) +set(CONFIG_LDFLAGS ${_CONFIG_LDFLAGS}) # This version of the file allows the build directory to be used directly configure_file(bin/bout-config.in bin/bout-config @ONLY) -configure_file(tools/pylib/boutconfig/__init__.py.cin tools/pylib/boutconfig/__init__.py @ONLY) +configure_file( + tools/pylib/boutconfig/__init__.py.cin tools/pylib/boutconfig/__init__.py + @ONLY +) configure_file(bout++Config.cmake.in bout++Config.cmake @ONLY) # We need to generate a separate version for installation, with the # correct install paths. So first we need to replace the build # directory library path with the installation path -string(REPLACE - "${CMAKE_BINARY_DIR}/lib" "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}" - CONFIG_LDFLAGS "${CONFIG_LDFLAGS}") +string(REPLACE "${CMAKE_BINARY_DIR}/lib" + "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}" CONFIG_LDFLAGS + "${CONFIG_LDFLAGS}" +) set(BOUT_LIB_PATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") # Update mpark.variant and fmt include paths if we're building them -if (NOT BOUT_USE_SYSTEM_MPARK_VARIANT) - set(MPARK_VARIANT_INCLUDE_PATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}") +if(NOT BOUT_USE_SYSTEM_MPARK_VARIANT) + set(MPARK_VARIANT_INCLUDE_PATH + "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}" + ) endif() -if (NOT BOUT_USE_SYSTEM_FMT) +if(NOT BOUT_USE_SYSTEM_FMT) set(FMT_INCLUDE_PATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}") endif() +if(NOT BOUT_USE_SYSTEM_CPPTRACE) + set(CPPTRACE_INCLUDE_PATH + "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}" + ) +endif() set(BOUT_INCLUDE_PATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}") # We don't need the build include path any more -string(REPLACE "-I${CMAKE_CURRENT_BINARY_DIR}/include" "" CONFIG_CFLAGS "${CONFIG_CFLAGS}") +string(REPLACE "-I${CMAKE_CURRENT_BINARY_DIR}/include" "" CONFIG_CFLAGS + "${CONFIG_CFLAGS}" +) set(PYTHONCONFIGPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_PYTHON_SITEARCH}") set(ISINSTALLED "TRUE") # This version now has the correct paths to use the final installation configure_file(bin/bout-config.in bin/bout-config-install @ONLY) -configure_file(tools/pylib/boutconfig/__init__.py.cin tools/pylib/boutconfig/__init__.py-install @ONLY) +configure_file( + tools/pylib/boutconfig/__init__.py.cin + tools/pylib/boutconfig/__init__.py-install @ONLY +) configure_file(bout++Config.cmake.in bout++Config.cmake-install @ONLY) ################################################## # Installation -install(TARGETS bout++ +install( + TARGETS bout++ EXPORT bout++Targets LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" - INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" - ) + INCLUDES + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" +) # Repo files -install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - FILES_MATCHING PATTERN "*.hxx") +install( + DIRECTORY include/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + FILES_MATCHING + PATTERN "*.hxx" +) # Generated headers install(DIRECTORY "${PROJECT_BINARY_DIR}/include/" - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) # The various helper scripts -install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/bin/" - USE_SOURCE_PERMISSIONS +install( + DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/bin/" USE_SOURCE_PERMISSIONS DESTINATION "${CMAKE_INSTALL_BINDIR}" REGEX "bout-squashoutput" EXCLUDE REGEX "bout-config\.in" EXCLUDE @@ -872,69 +984,71 @@ install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/bin/" # it. Note this MUST be done after the installation of bin/, to make # sure we clobber any versions of bout-config hanging around from an # autotools build -install(PROGRAMS "${CMAKE_CURRENT_BINARY_DIR}/bin/bout-config-install" +install( + PROGRAMS "${CMAKE_CURRENT_BINARY_DIR}/bin/bout-config-install" DESTINATION "${CMAKE_INSTALL_BINDIR}" RENAME "bout-config" - ) +) install( FILES "${CMAKE_CURRENT_BINARY_DIR}/tools/pylib/boutconfig/__init__.py-install" DESTINATION "${CMAKE_INSTALL_PYTHON_SITEARCH}/boutconfig" RENAME "__init__.py" - ) +) include(CMakePackageConfigHelpers) write_basic_package_version_file( bout++ConfigVersion.cmake VERSION ${PACKAGE_VERSION} COMPATIBILITY SameMajorVersion - ) +) -install(EXPORT bout++Targets +install( + EXPORT bout++Targets FILE bout++Targets.cmake NAMESPACE bout++:: DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/bout++" - ) +) # CMake configuration files install( - FILES - "${CMAKE_CURRENT_BINARY_DIR}/bout++ConfigVersion.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/BOUT++functions.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/CorrectWindowsPaths.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindClangFormat.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindFFTW.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindHYPRE.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindnetCDF.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindnetCDFCxx.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindPackageMultipass.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindLibuuid.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindPETSc.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindScoreP.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindSLEPc.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindSUNDIALS.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindSphinx.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/ResolveCompilerPaths.cmake" + FILES "${CMAKE_CURRENT_BINARY_DIR}/bout++ConfigVersion.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/BOUT++functions.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/CorrectWindowsPaths.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindClangFormat.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindFFTW.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindHYPRE.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindnetCDF.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindnetCDFCxx.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindPackageMultipass.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindLibuuid.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindPETSc.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindScoreP.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindSLEPc.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindSUNDIALS.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindSphinx.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/ResolveCompilerPaths.cmake" DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/bout++" - ) +) install( - FILES - "${CMAKE_CURRENT_BINARY_DIR}/bout++Config.cmake-install" + FILES "${CMAKE_CURRENT_BINARY_DIR}/bout++Config.cmake-install" DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/bout++" RENAME "bout++Config.cmake" - ) +) -export(EXPORT bout++Targets +export( + EXPORT bout++Targets FILE "${CMAKE_CURRENT_BINARY_DIR}/bout++Targets.cmake" NAMESPACE bout++:: - ) +) export(PACKAGE bout) ################################################## # Configure summary -message(" +message( + " -------------------------------- BOUT++ Configuration Summary -------------------------------- @@ -958,7 +1072,6 @@ message(" Output coloring : ${BOUT_USE_COLOR} Field name tracking : ${BOUT_USE_TRACK} Floating point exceptions: ${BOUT_USE_SIGFPE} - Backtrace enabled : ${BOUT_USE_BACKTRACE} RAJA enabled : ${BOUT_HAS_RAJA} Umpire enabled : ${BOUT_HAS_UMPIRE} Caliper enabled : ${BOUT_HAS_CALIPER} @@ -975,4 +1088,5 @@ message(" export PYTHONPATH=${BOUT_PYTHONPATH}:\$PYTHONPATH *** Now run `cmake --build ${CMAKE_BINARY_DIR}` to compile BOUT++ *** -") +" +) diff --git a/bin/bout-add-mod-path b/bin/bout-add-mod-path index 9faf1be3de..ee7ea66d25 100755 --- a/bin/bout-add-mod-path +++ b/bin/bout-add-mod-path @@ -104,8 +104,7 @@ def create_mod(modulepath, name, top, build): else: prereq = "" with open(filename, "w") as f: - f.write( - f"""#%Module 1.0 + f.write(f"""#%Module 1.0 # # BOUT++ module for use with 'environment-modules' package # Created by bout-add-mod-path v0.9 @@ -119,17 +118,14 @@ setenv BOUT_TOP {top} prepend-path PATH {top}/bin prepend-path PYTHONPATH {top}/tools/pylib prepend-path IDL_PATH +{top}/tools/idllib:'' -""" - ) +""") if build != top: - f.write( - f"""#%Module 1.0 + f.write(f"""#%Module 1.0 setenv BOUT_BUILD {build} prepend-path PATH {build}/bin prepend-path LD_LIBRARY_PATH {build}/lib prepend-path PYTHONPATH {build}/tools/pylib -""" - ) +""") print(f"created `{filename}`") diff --git a/bin/bout-pylib-cmd-to-bin b/bin/bout-pylib-cmd-to-bin index 8f88a5dbf4..8d6dbed7ae 100755 --- a/bin/bout-pylib-cmd-to-bin +++ b/bin/bout-pylib-cmd-to-bin @@ -126,8 +126,7 @@ def createwrapper(mod, func_name, func, name): out += end f.write(out) - fprint( - """#!/usr/bin/env python3 + fprint("""#!/usr/bin/env python3 # PYTHON_ARGCOMPLETE_OK import argparse @@ -136,8 +135,7 @@ try: except ImportError: argcomplete=None -""" - ) +""") doc = True para = False ret = False @@ -183,19 +181,16 @@ except ImportError: arg_help[curarg].append(esc(blas)) # Print functions that are needed if "str_to_slice" in arg_type.values(): - fprint( - """ + fprint(""" def str_to_slice(sstr): args=[] for s in sstr.split(','): args.append(int(s)) print(args) return slice(*args) -""" - ) +""") if "str_to_bool" in arg_type.values(): - fprint( - """ + fprint(""" def str_to_bool(sstr): low=sstr.lower() # no or false @@ -206,8 +201,7 @@ def str_to_bool(sstr): return True else: raise ArgumentTypeError("Cannot parse %s to bool type"%sstr) -""" - ) +""") # Create the parser fprint("parser = argparse.ArgumentParser(%s)" % (esc(docs))) spec = inspect.signature(func) @@ -247,24 +241,19 @@ def str_to_bool(sstr): pre = "\n " fprint(")") - fprint( - """ + fprint(""" if argcomplete: argcomplete.autocomplete(parser) -# late import for faster auto-complete""" - ) +# late import for faster auto-complete""") fprint("from %s import %s" % (mod, func_name)) - fprint( - """ + fprint(""" args = parser.parse_args() # Call the function %s, using command line arguments %s(**args.__dict__) -""" - % (func_name, func_name) - ) +""" % (func_name, func_name)) # alternative, but I think 0o755 is easier to read # import stat # os.chmod(filename,stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH) diff --git a/bin/bout-v5-factory-upgrader.py b/bin/bout-v5-factory-upgrader.py index 29fc07db30..ee24572a94 100755 --- a/bin/bout-v5-factory-upgrader.py +++ b/bin/bout-v5-factory-upgrader.py @@ -5,7 +5,6 @@ import difflib import re - # Dictionary of factory methods that may need updating factories = { "Interpolation": { @@ -62,9 +61,7 @@ def find_factory_calls(factory, source): \s*=\s* {factory_name}:: .*{create_method}.* - """.format( - **factory - ), + """.format(**factory), source, re.VERBOSE, ) @@ -75,9 +72,7 @@ def find_type_pointers(factory, source): r""" \b{type_name}\s*\*\s* # Type name and pointer ([\w_]+)\s*; # Variable name - """.format( - **factory - ), + """.format(**factory), source, re.VERBOSE, ) @@ -107,9 +102,7 @@ def fix_declarations(factory, variables, source): (.*?)(class\s*)? # optional "class" keyword \b({type_name})\s*\*\s* # Type-pointer ({variable_name})\s*; # Variable - """.format( - type_name=factory["type_name"], variable_name=variable - ), + """.format(type_name=factory["type_name"], variable_name=variable), r"\1std::unique_ptr<\3> \4{nullptr};", source, flags=re.VERBOSE, @@ -123,9 +116,7 @@ def fix_declarations(factory, variables, source): ({variable_name})\s* # Variable =\s* # Assignment from factory ({factory_name}::.*{create_method}.*); - """.format( - variable_name=variable, **factory - ), + """.format(variable_name=variable, **factory), r"\1auto \4 = \5;", source, flags=re.VERBOSE, @@ -139,9 +130,7 @@ def fix_declarations(factory, variables, source): ({variable_name})\s* # Variable =\s* # Assignment (0|nullptr|NULL); - """.format( - variable_name=variable, **factory - ), + """.format(variable_name=variable, **factory), r"\1std::unique_ptr<\2> \3{nullptr};", source, flags=re.VERBOSE, diff --git a/bin/bout-v5-format-upgrader.py b/bin/bout-v5-format-upgrader.py index 7c7d13ac7f..a534ca240a 100755 --- a/bin/bout-v5-format-upgrader.py +++ b/bin/bout-v5-format-upgrader.py @@ -5,7 +5,6 @@ import difflib import re - format_replacements = { "c": "c", "d": "d", diff --git a/bin/bout-v5-header-upgrader.py b/bin/bout-v5-header-upgrader.py index 49a8fbcbe4..77794ab920 100755 --- a/bin/bout-v5-header-upgrader.py +++ b/bin/bout-v5-header-upgrader.py @@ -9,7 +9,6 @@ from typing import List from subprocess import run - header_shim_sentinel = "// BOUT++ header shim" header_warning = f"""\ @@ -122,8 +121,7 @@ def create_patch(filename, original, modified): if __name__ == "__main__": parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, - description=textwrap.dedent( - """\ + description=textwrap.dedent("""\ Fix deprecated header locations for BOUT++ v4 -> v5 All BOUT++ headers are now under ``include/bout`` and @@ -142,8 +140,7 @@ def create_patch(filename, original, modified): If you have staged changes, this tool will not work, so to avoid committing undesired or unrelated changes. - """ - ), + """), ) parser.add_argument( diff --git a/bin/bout-v5-input-file-upgrader.py b/bin/bout-v5-input-file-upgrader.py index e2940ff58a..ea979005d5 100755 --- a/bin/bout-v5-input-file-upgrader.py +++ b/bin/bout-v5-input-file-upgrader.py @@ -271,8 +271,7 @@ def possibly_apply_patch(patch, options_file, quiet=False, force=False): if __name__ == "__main__": parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, - description=textwrap.dedent( - """\ + description=textwrap.dedent("""\ Fix input files for BOUT++ v5+ Please note that this will only fix input options in sections with @@ -300,8 +299,7 @@ def possibly_apply_patch(patch, options_file, quiet=False, force=False): Files that change in this way will have the "canonicalisation" patch presented first. If you choose not to apply this patch, the "upgrade - fixer" patch will still include it.""" - ), + fixer" patch will still include it."""), ) parser.add_argument("files", action="store", nargs="+", help="Input files") diff --git a/bin/bout-v5-macro-upgrader.py b/bin/bout-v5-macro-upgrader.py index 11b4926255..d644fed9e8 100755 --- a/bin/bout-v5-macro-upgrader.py +++ b/bin/bout-v5-macro-upgrader.py @@ -342,8 +342,7 @@ def create_patch(filename, original, modified): if __name__ == "__main__": parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, - description=textwrap.dedent( - """\ + description=textwrap.dedent("""\ Fix macro defines for BOUT++ v4 -> v5 Please note that this is only slightly better than dumb text replacement. It @@ -359,8 +358,7 @@ def create_patch(filename, original, modified): still replace them in strings or comments. Please check the diff output carefully! - """ - ), + """), ) parser.add_argument("files", action="store", nargs="+", help="Input files") diff --git a/bin/bout-v5-physics-model-upgrader.py b/bin/bout-v5-physics-model-upgrader.py index 26fb8ef6e0..260d1d59ee 100755 --- a/bin/bout-v5-physics-model-upgrader.py +++ b/bin/bout-v5-physics-model-upgrader.py @@ -8,7 +8,6 @@ import textwrap import warnings - PHYSICS_MODEL_INCLUDE = '#include "bout/physicsmodel.hxx"' PHYSICS_MODEL_SKELETON = """ @@ -213,13 +212,11 @@ def fix_bout_constrain(source, error_on_warning): "\n ".join(["{}:{}".format(i, source_lines[i]) for i in line_range]) ) - message = textwrap.dedent( - """\ + message = textwrap.dedent("""\ Some uses of `bout_constrain` remain, but we could not automatically convert them to use `Solver::constraint`. Please fix them before continuing: - """ - ) + """) message += " " + "\n ".join(lines_context) if error_on_warning: @@ -389,8 +386,7 @@ def create_patch(filename, original, modified): if __name__ == "__main__": parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, - description=textwrap.dedent( - """\ + description=textwrap.dedent("""\ Upgrade legacy physics models to use the PhysicsModel class This will do the bare minimum required to compile, and @@ -403,8 +399,7 @@ def create_patch(filename, original, modified): By default, this will use the file name stripped of file extensions as the name of the new class. Use '--name=' to give a different name. - """ - ), + """), ) parser.add_argument("files", action="store", nargs="+", help="Files to fix") diff --git a/bin/bout-v5-xzinterpolation-upgrader.py b/bin/bout-v5-xzinterpolation-upgrader.py index 37c79e0de8..e70c3c54ae 100755 --- a/bin/bout-v5-xzinterpolation-upgrader.py +++ b/bin/bout-v5-xzinterpolation-upgrader.py @@ -54,9 +54,7 @@ def fix_header_includes(old_header, new_header, source): (<|") ({header}) # Header name (>|") - """.format( - header=old_header - ), + """.format(header=old_header), r"\1\2{header}\4".format(header=new_header), source, flags=re.VERBOSE, @@ -67,9 +65,7 @@ def fix_interpolations(old_interpolation, new_interpolation, source): return re.sub( r""" \b{}\b - """.format( - old_interpolation - ), + """.format(old_interpolation), r"{}".format(new_interpolation), source, flags=re.VERBOSE, @@ -120,9 +116,7 @@ def fix_factories(old_factory, new_factory, source): return re.sub( r""" \b{}\b - """.format( - old_factory - ), + """.format(old_factory), r"{}".format(new_factory), source, flags=re.VERBOSE, diff --git a/bin/update_version_number_in_files.py b/bin/update_version_number_in_files.py index ec9a31bc32..3e9758a8c1 100755 --- a/bin/update_version_number_in_files.py +++ b/bin/update_version_number_in_files.py @@ -158,8 +158,7 @@ def create_patch(filename, original, modified): if __name__ == "__main__": parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, - description=textwrap.dedent( - """\ + description=textwrap.dedent("""\ Update the software version number to the specified version, to be given in the form major.minor.patch, e.g. 5.10.3 @@ -172,8 +171,7 @@ def create_patch(filename, original, modified): the 'minor' version number of the provided version will be incremented by 1, e.g. 5.10.3 -> 5.11.3 - """ - ), + """), ) parser.add_argument( diff --git a/bout++Config.cmake.in b/bout++Config.cmake.in index d461e58e4b..5cc68aefde 100644 --- a/bout++Config.cmake.in +++ b/bout++Config.cmake.in @@ -6,10 +6,9 @@ set(BOUT_USE_SIGNAL @BOUT_USE_SIGNAL@) set(BOUT_USE_COLOR @BOUT_USE_COLOR@) set(BOUT_USE_TRACK @BOUT_USE_TRACK@) set(BOUT_USE_SIGFPE @BOUT_USE_SIGFPE@) -set(BOUT_USE_BACKTRACE @BOUT_USE_BACKTRACE@) set(BOUT_USE_OPENMP @BOUT_USE_OPENMP@) set(BOUT_HAS_CUDA @BOUT_HAS_CUDA@) -set(BOUT_HAS_OUTPUT_DEBUG @BOUT_HAS_OUTPUT_DEBUG@) +set(BOUT_USE_OUTPUT_DEBUG @BOUT_USE_OUTPUT_DEBUG@) set(BOUT_CHECK_LEVEL @BOUT_CHECK_LEVEL@) set(BOUT_USE_METRIC_3D @BOUT_USE_METRIC_3D@) @@ -109,6 +108,14 @@ elseif(EXISTS "@ADIOS2_BINARY_DIR@") # If we downloaded ADIOS2, then we need to add its build directory to our search paths list(APPEND CMAKE_PREFIX_PATH "@ADIOS2_BINARY_DIR@") endif() +if(EXISTS "@cpptrace_ROOT@") + set(cpptrace_ROOT "@cpptrace_ROOT@") +elseif(EXISTS "@cpptrace_BINARY_DIR@") + list(APPEND CMAKE_PREFIX_PATH "@cpptrace_BINARY_DIR@") + list(APPEND CMAKE_PREFIX_PATH "@cpptrace_BINARY_DIR@/cmake") + list(APPEND CMAKE_PREFIX_PATH "@libdwarf_BINARY_DIR@/src/lib/libdwarf") + list(APPEND CMAKE_PREFIX_PATH "@zstd_BINARY_DIR@") +endif() if(@BOUT_USE_SYSTEM_MPARK_VARIANT@) set(mpark_variant_ROOT "@mpark_variant_ROOT@") @@ -160,6 +167,7 @@ if (BOUT_HAS_GETTEXT) endif() find_dependency(mpark_variant @mpark_variant_VERSION@) find_dependency(fmt @fmt_VERSION@) +find_dependency(cpptrace @cpptrace_VERSION@) if (BOUT_HAS_SLEPC) find_dependency(SLEPc @SLEPC_VERSION@) endif() @@ -174,3 +182,30 @@ if (BOUT_HAS_ADIOS2) endif() include("${CMAKE_CURRENT_LIST_DIR}/bout++Targets.cmake") + +add_library(bout++ INTERFACE IMPORTED) +set_target_properties(bout++ + PROPERTIES BOUT_USE_SIGNAL @BOUT_USE_SIGNAL@ + BOUT_USE_COLOR @BOUT_USE_COLOR@ + BOUT_USE_TRACK @BOUT_USE_TRACK@ + BOUT_USE_SIGFPE @BOUT_USE_SIGFPE@ + BOUT_USE_OPENMP @BOUT_USE_OPENMP@ + BOUT_HAS_CUDA @BOUT_HAS_CUDA@ + BOUT_USE_OUTPUT_DEBUG @BOUT_USE_OUTPUT_DEBUG@ + BOUT_CHECK_LEVEL @BOUT_CHECK_LEVEL@ + BOUT_USE_METRIC_3D @BOUT_USE_METRIC_3D@ + BOUT_HAS_PVODE @BOUT_HAS_PVODE@ + BOUT_HAS_NETCDF @BOUT_HAS_NETCDF@ + BOUT_HAS_ADIOS2 @BOUT_HAS_ADIOS2@ + BOUT_HAS_FFTW @BOUT_HAS_FFTW@ + BOUT_HAS_LAPACK @BOUT_HAS_LAPACK@ + BOUT_HAS_PETSC @BOUT_HAS_PETSC@ + BOUT_HAS_SLEPC @BOUT_HAS_SLEPC@ + BOUT_HAS_SCOREP @BOUT_HAS_SCOREP@ + BOUT_USE_UUID_SYSTEM_GENERATOR @BOUT_USE_UUID_SYSTEM_GENERATOR@ + BOUT_HAS_SUNDIALS @BOUT_HAS_SUNDIALS@ + BOUT_HAS_HYPRE @BOUT_HAS_HYPRE@ + BOUT_HAS_GETTEXT @BOUT_HAS_GETTEXT@ + BOUT_HAS_UMPIRE @BOUT_HAS_UMPIRE@ + BOUT_HAS_RAJA @BOUT_HAS_RAJA@ +) diff --git a/cmake-format.yaml b/cmake-format.yaml new file mode 100644 index 0000000000..082e7b5ef1 --- /dev/null +++ b/cmake-format.yaml @@ -0,0 +1,48 @@ +parse: + additional_commands: + bout_add_example: + pargs: 1 + kwargs: + CONFLICTS: '*' + DATA_DIRS: '*' + EXTRA_FILES: '*' + REQUIRES: '*' + SOURCES: '*' + bout_add_integrated_test: + pargs: 1 + flags: [USE_DATA_BOUT_INP, USE_RUNTEST] + kwargs: + CONFLICTS: '*' + DOWNLOAD: 1 + DOWNLOAD_NAME: 1 + EXECUTABLE_NAME: 1 + EXTRA_DEPENDS: '*' + EXTRA_FILES: '*' + PROCESSORS: 1 + REQUIRES: '*' + SOURCES: '*' + TESTARGS: '*' + bout_add_mms_test: + pargs: 1 + flags: [USE_DATA_BOUT_INP, USE_RUNTEST] + kwargs: + CONFLICTS: '*' + DOWNLOAD: 1 + DOWNLOAD_NAME: 1 + EXECUTABLE_NAME: 1 + EXTRA_DEPENDS: '*' + EXTRA_FILES: '*' + PROCESSORS: 1 + REQUIRES: '*' + SOURCES: '*' + TESTARGS: '*' + bout_handle_requires_conflicts: + pargs: 2 + kwargs: + CONFLICTS: '*' + REQUIRES: '*' +format: + dangle_parens: true +markup: + # markup messes up comments - disable it + enable_markup: false \ No newline at end of file diff --git a/cmake/BOUT++functions.cmake b/cmake/BOUT++functions.cmake index 77279dfd4b..cff7e4b808 100644 --- a/cmake/BOUT++functions.cmake +++ b/cmake/BOUT++functions.cmake @@ -3,9 +3,9 @@ # Copy FILENAME from source directory to build directory macro(bout_copy_file FILENAME) configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/${FILENAME} - ${CMAKE_CURRENT_BINARY_DIR}/${FILENAME} - COPYONLY) + ${CMAKE_CURRENT_SOURCE_DIR}/${FILENAME} + ${CMAKE_CURRENT_BINARY_DIR}/${FILENAME} COPYONLY + ) endmacro() # Handle the REQUIRES and CONFLICTS arguments for models, examples, @@ -15,16 +15,22 @@ macro(bout_handle_requires_conflicts TYPENAME TYPEVAR) set(multiValueArgs REQUIRES CONFLICTS) cmake_parse_arguments(BOUT_HANDLE_OPTIONS "" "" "${multiValueArgs}" ${ARGN}) - foreach (REQUIREMENT IN LISTS BOUT_HANDLE_OPTIONS_REQUIRES) - if (NOT ${REQUIREMENT}) - message(STATUS "Not building ${TYPENAME} ${TYPEVAR}, requirement not met: ${REQUIREMENT}") + foreach(REQUIREMENT IN LISTS BOUT_HANDLE_OPTIONS_REQUIRES) + if(NOT ${REQUIREMENT}) + message( + STATUS + "Not building ${TYPENAME} ${TYPEVAR}, requirement not met: ${REQUIREMENT}" + ) return() endif() endforeach() - foreach (CONFLICT IN LISTS BOUT_HANDLE_OPTIONS_CONFLICTS) - if (${CONFLICT}) - message(STATUS "Not building ${TYPENAME} ${TYPEVAR}, conflicts with: ${CONFLICT}") + foreach(CONFLICT IN LISTS BOUT_HANDLE_OPTIONS_CONFLICTS) + if(${CONFLICT}) + message( + STATUS + "Not building ${TYPENAME} ${TYPEVAR}, conflicts with: ${CONFLICT}" + ) return() endif() endforeach() @@ -46,25 +52,29 @@ function(bout_add_model MODEL) set(multiValueArgs SOURCES REQUIRES CONFLICTS) cmake_parse_arguments(BOUT_MODEL_OPTIONS "" "" "${multiValueArgs}" ${ARGN}) - bout_handle_requires_conflicts("model" MODEL + bout_handle_requires_conflicts( + "model" MODEL REQUIRES ${BOUT_MODEL_OPTIONS_REQUIRES} CONFLICTS ${BOUT_MODEL_OPTIONS_CONFLICTS} - ) + ) - if (NOT BOUT_MODEL_OPTIONS_SOURCES) - message(FATAL_ERROR "Required argument SOURCES missing from 'bout_add_model'") + if(NOT BOUT_MODEL_OPTIONS_SOURCES) + message( + FATAL_ERROR "Required argument SOURCES missing from 'bout_add_model'" + ) endif() - if ("SOURCES" IN_LIST BOUT_MODEL_OPTIONS_KEYWORDS_MISSING_VALUES) + if("SOURCES" IN_LIST BOUT_MODEL_OPTIONS_KEYWORDS_MISSING_VALUES) message(FATAL_ERROR "SOURCES missing values from 'bout_add_model'") endif() add_executable(${MODEL} ${BOUT_MODEL_OPTIONS_SOURCES}) target_link_libraries(${MODEL} bout++::bout++) - target_include_directories(${MODEL} PRIVATE $) + target_include_directories( + ${MODEL} PRIVATE $ + ) endfunction() - # Build a BOUT++ example # # If called from a standalone project, just builds the example as a @@ -85,37 +95,38 @@ function(bout_add_example EXAMPLENAME) set(multiValueArgs SOURCES REQUIRES CONFLICTS DATA_DIRS EXTRA_FILES) cmake_parse_arguments(BOUT_EXAMPLE_OPTIONS "" "" "${multiValueArgs}" ${ARGN}) - bout_handle_requires_conflicts("example" ${EXAMPLENAME} + bout_handle_requires_conflicts( + "example" ${EXAMPLENAME} REQUIRES ${BOUT_EXAMPLE_OPTIONS_REQUIRES} CONFLICTS ${BOUT_EXAMPLE_OPTIONS_CONFLICTS} - ) + ) bout_add_model(${EXAMPLENAME} SOURCES ${BOUT_EXAMPLE_OPTIONS_SOURCES}) # If this is a standalone project, we can stop here. Otherwise, we # need to copy the various input files to the build directory get_directory_property(HAS_PARENT PARENT_DIRECTORY) - if (NOT HAS_PARENT) + if(NOT HAS_PARENT) return() endif() # Copy the documentation if it exists - if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/README.md) + if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/README.md) bout_copy_file(README.md) endif() # Copy the input file - if (NOT BOUT_EXAMPLE_OPTIONS_DATA_DIRS) + if(NOT BOUT_EXAMPLE_OPTIONS_DATA_DIRS) bout_copy_file(data/BOUT.inp) else() - foreach (DATA_DIR IN LISTS BOUT_EXAMPLE_OPTIONS_DATA_DIRS) + foreach(DATA_DIR IN LISTS BOUT_EXAMPLE_OPTIONS_DATA_DIRS) bout_copy_file(${DATA_DIR}/BOUT.inp) endforeach() endif() # Copy any other needed files - if (BOUT_EXAMPLE_OPTIONS_EXTRA_FILES) - foreach (FILE ${BOUT_EXAMPLE_OPTIONS_EXTRA_FILES}) + if(BOUT_EXAMPLE_OPTIONS_EXTRA_FILES) + foreach(FILE ${BOUT_EXAMPLE_OPTIONS_EXTRA_FILES}) bout_copy_file("${FILE}") endforeach() endif() @@ -125,7 +136,6 @@ function(bout_add_example EXAMPLENAME) add_dependencies(build-all-examples ${EXAMPLENAME}) endfunction() - # Add a new integrated or MMS test. By default, the executable is # named like the first source, stripped of its file extension. If no # sources are given, then you probably at least want to set @@ -163,37 +173,51 @@ endfunction() function(bout_add_integrated_or_mms_test BUILD_CHECK_TARGET TESTNAME) set(options USE_RUNTEST USE_DATA_BOUT_INP) set(oneValueArgs EXECUTABLE_NAME PROCESSORS DOWNLOAD DOWNLOAD_NAME) - set(multiValueArgs SOURCES EXTRA_FILES REQUIRES CONFLICTS TESTARGS EXTRA_DEPENDS) - cmake_parse_arguments(BOUT_TEST_OPTIONS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + set(multiValueArgs SOURCES EXTRA_FILES REQUIRES CONFLICTS TESTARGS + EXTRA_DEPENDS + ) + cmake_parse_arguments( + BOUT_TEST_OPTIONS "${options}" "${oneValueArgs}" "${multiValueArgs}" + ${ARGN} + ) - bout_handle_requires_conflicts("test" ${TESTNAME} + bout_handle_requires_conflicts( + "test" ${TESTNAME} REQUIRES ${BOUT_TEST_OPTIONS_REQUIRES} CONFLICTS ${BOUT_TEST_OPTIONS_CONFLICTS} - ) + ) - if (BOUT_TEST_OPTIONS_SOURCES) + if(BOUT_TEST_OPTIONS_SOURCES) # We've got some sources, so compile them into an executable and # link against BOUT++ add_executable(${TESTNAME} ${BOUT_TEST_OPTIONS_SOURCES}) target_link_libraries(${TESTNAME} bout++) - target_include_directories(${TESTNAME} PRIVATE $) + target_include_directories( + ${TESTNAME} PRIVATE $ + ) set_target_properties(${TESTNAME} PROPERTIES FOLDER tests/integrated) # Set the name of the executable. We either take it as an option, # or use the first source file, stripping the file suffix - if (BOUT_TEST_OPTIONS_EXECUTABLE_NAME) - set_target_properties(${TESTNAME} PROPERTIES OUTPUT_NAME ${BOUT_TEST_OPTIONS_EXECUTABLE_NAME}) + if(BOUT_TEST_OPTIONS_EXECUTABLE_NAME) + set_target_properties( + ${TESTNAME} PROPERTIES OUTPUT_NAME ${BOUT_TEST_OPTIONS_EXECUTABLE_NAME} + ) else() # If more than one source file, just get the first one list(LENGTH ${BOUT_TEST_OPTIONS_SOURCES} BOUT_SOURCES_LENGTH) - if (BOUT_SOURCES_LENGTH GREATER 0) + if(BOUT_SOURCES_LENGTH GREATER 0) list(GET ${BOUT_TEST_OPTIONS_SOURCES} 0 BOUT_TEST_FIRST_SOURCE) else() set(BOUT_TEST_FIRST_SOURCE ${BOUT_TEST_OPTIONS_SOURCES}) endif() # Strip the directory and file extension from the source file - get_filename_component(BOUT_TEST_EXECUTABLE_NAME ${BOUT_TEST_FIRST_SOURCE} NAME_WE) - set_target_properties(${TESTNAME} PROPERTIES OUTPUT_NAME ${BOUT_TEST_EXECUTABLE_NAME}) + get_filename_component( + BOUT_TEST_EXECUTABLE_NAME ${BOUT_TEST_FIRST_SOURCE} NAME_WE + ) + set_target_properties( + ${TESTNAME} PROPERTIES OUTPUT_NAME ${BOUT_TEST_EXECUTABLE_NAME} + ) endif() # Add the test to the build-check-integrated-tests target @@ -202,54 +226,57 @@ function(bout_add_integrated_or_mms_test BUILD_CHECK_TARGET TESTNAME) add_custom_target(${TESTNAME}) endif() - if (BOUT_TEST_OPTIONS_DOWNLOAD) - if (NOT BOUT_TEST_OPTIONS_DOWNLOAD_NAME) + if(BOUT_TEST_OPTIONS_DOWNLOAD) + if(NOT BOUT_TEST_OPTIONS_DOWNLOAD_NAME) message(FATAL_ERROR "We need DOWNLOAD_NAME if we should DOWNLOAD!") endif() - set(output ) - add_custom_command(OUTPUT ${BOUT_TEST_OPTIONS_DOWNLOAD_NAME} - COMMAND wget ${BOUT_TEST_OPTIONS_DOWNLOAD} -O ${BOUT_TEST_OPTIONS_DOWNLOAD_NAME} + set(output) + add_custom_command( + OUTPUT ${BOUT_TEST_OPTIONS_DOWNLOAD_NAME} + COMMAND wget ${BOUT_TEST_OPTIONS_DOWNLOAD} -O + ${BOUT_TEST_OPTIONS_DOWNLOAD_NAME} $ENV{BOUT_TEST_DOWNLOAD_FLAGS} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Downloading ${BOUT_TEST_OPTIONS_DOWNLOAD_NAME}" - ) - add_custom_target(download_test_data DEPENDS ${BOUT_TEST_OPTIONS_DOWNLOAD_NAME}) + ) + add_custom_target( + download_test_data DEPENDS ${BOUT_TEST_OPTIONS_DOWNLOAD_NAME} + ) add_dependencies(${TESTNAME} download_test_data) endif() - if (BOUT_TEST_OPTIONS_EXTRA_DEPENDS) + if(BOUT_TEST_OPTIONS_EXTRA_DEPENDS) add_dependencies(${TESTNAME} ${BOUT_TEST_OPTIONS_EXTRA_DEPENDS}) endif() - if (NOT BOUT_TEST_OPTIONS_PROCESSORS) + if(NOT BOUT_TEST_OPTIONS_PROCESSORS) set(BOUT_TEST_OPTIONS_PROCESSORS 1) endif() # Set the actual test command - if (BOUT_TEST_OPTIONS_USE_RUNTEST) - add_test(NAME ${TESTNAME} - COMMAND ./runtest ${BOUT_TEST_OPTIONS_TESTARGS} - ) - set_tests_properties(${TESTNAME} PROPERTIES - ENVIRONMENT PYTHONPATH=${BOUT_PYTHONPATH}:$ENV{PYTHONPATH} - ) + if(BOUT_TEST_OPTIONS_USE_RUNTEST) + add_test(NAME ${TESTNAME} COMMAND ./runtest ${BOUT_TEST_OPTIONS_TESTARGS}) + set_tests_properties( + ${TESTNAME} PROPERTIES ENVIRONMENT + PYTHONPATH=${BOUT_PYTHONPATH}:$ENV{PYTHONPATH} + ) bout_copy_file(runtest) else() add_test(NAME ${TESTNAME} COMMAND ${TESTNAME} ${BOUT_TEST_OPTIONS_TESTARGS}) endif() - set_tests_properties(${TESTNAME} PROPERTIES - PROCESSORS ${BOUT_TEST_OPTIONS_PROCESSORS} - PROCESSOR_AFFINITY ON - ) + set_tests_properties( + ${TESTNAME} PROPERTIES PROCESSORS ${BOUT_TEST_OPTIONS_PROCESSORS} + PROCESSOR_AFFINITY ON + ) # Copy the input file if needed - if (BOUT_TEST_OPTIONS_USE_DATA_BOUT_INP) + if(BOUT_TEST_OPTIONS_USE_DATA_BOUT_INP) bout_copy_file(data/BOUT.inp) endif() # Copy any other needed files - if (BOUT_TEST_OPTIONS_EXTRA_FILES) - foreach (FILE ${BOUT_TEST_OPTIONS_EXTRA_FILES}) + if(BOUT_TEST_OPTIONS_EXTRA_FILES) + foreach(FILE ${BOUT_TEST_OPTIONS_EXTRA_FILES}) bout_copy_file("${FILE}") endforeach() endif() @@ -257,7 +284,9 @@ endfunction() # Add a new integrated test. See `bout_add_integrated_or_mms_test` for arguments function(bout_add_integrated_test TESTNAME) - bout_add_integrated_or_mms_test(build-check-integrated-tests ${TESTNAME} ${ARGV}) + bout_add_integrated_or_mms_test( + build-check-integrated-tests ${TESTNAME} ${ARGV} + ) endfunction() # Add a new MMS test. See `bout_add_integrated_or_mms_test` for arguments @@ -270,13 +299,18 @@ endfunction() # Taken from https://github.com/conan-io/conan/issues/2125#issuecomment-351176653 function(bout_add_library_alias dst src) add_library(${dst} INTERFACE IMPORTED) - foreach(name INTERFACE_LINK_LIBRARIES INTERFACE_INCLUDE_DIRECTORIES INTERFACE_COMPILE_DEFINITIONS INTERFACE_COMPILE_OPTIONS) - get_property(value TARGET ${src} PROPERTY ${name} ) + foreach(name INTERFACE_LINK_LIBRARIES INTERFACE_INCLUDE_DIRECTORIES + INTERFACE_COMPILE_DEFINITIONS INTERFACE_COMPILE_OPTIONS + ) + get_property( + value + TARGET ${src} + PROPERTY ${name} + ) set_property(TARGET ${dst} PROPERTY ${name} ${value}) endforeach() endfunction() - # Call nx-config with an argument, and append the resulting path to a list # Taken from https://github.com/LiamBindle/geos-chem/blob/feature/CMake/CMakeScripts/FindNetCDF.cmake function(bout_inspect_netcdf_config VAR NX_CONFIG ARG) @@ -285,7 +319,10 @@ function(bout_inspect_netcdf_config VAR NX_CONFIG ARG) OUTPUT_VARIABLE NX_CONFIG_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE ) - if (NX_CONFIG_OUTPUT) - set(${VAR} ${NX_CONFIG_OUTPUT} PARENT_SCOPE) + if(NX_CONFIG_OUTPUT) + set(${VAR} + ${NX_CONFIG_OUTPUT} + PARENT_SCOPE + ) endif() endfunction() diff --git a/cmake/BuildType.cmake b/cmake/BuildType.cmake index c4dd7ff73d..7aced6d8ad 100644 --- a/cmake/BuildType.cmake +++ b/cmake/BuildType.cmake @@ -12,10 +12,17 @@ set(default_build_type "RelWithDebInfo") if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - message(STATUS "Setting build type to '${default_build_type}' as none was specified.") - set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE - STRING "Choose the type of build." FORCE) + message( + STATUS + "Setting build type to '${default_build_type}' as none was specified." + ) + set(CMAKE_BUILD_TYPE + "${default_build_type}" + CACHE STRING "Choose the type of build." FORCE + ) # Set the possible values of build type for cmake-gui - set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS - "Debug" "Release" "MinSizeRel" "RelWithDebInfo") + set_property( + CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" + "RelWithDebInfo" + ) endif() diff --git a/cmake/CorrectWindowsPaths.cmake b/cmake/CorrectWindowsPaths.cmake index 09bcdd67dc..cd2ac529ca 100644 --- a/cmake/CorrectWindowsPaths.cmake +++ b/cmake/CorrectWindowsPaths.cmake @@ -4,11 +4,9 @@ # This uses the command cygpath (provided by cygwin) to convert # unix-style paths into paths useable by cmake on windows -macro (CONVERT_CYGWIN_PATH _path) - if (WIN32) - EXECUTE_PROCESS(COMMAND cygpath.exe -m ${${_path}} - OUTPUT_VARIABLE ${_path}) - string (STRIP ${${_path}} ${_path}) - endif (WIN32) -endmacro (CONVERT_CYGWIN_PATH) - +macro(CONVERT_CYGWIN_PATH _path) + if(WIN32) + execute_process(COMMAND cygpath.exe -m ${${_path}} OUTPUT_VARIABLE ${_path}) + string(STRIP ${${_path}} ${_path}) + endif(WIN32) +endmacro(CONVERT_CYGWIN_PATH) diff --git a/cmake/EnableCXXWarningIfSupport.cmake b/cmake/EnableCXXWarningIfSupport.cmake index 6d7a64265f..9b9ae52af5 100644 --- a/cmake/EnableCXXWarningIfSupport.cmake +++ b/cmake/EnableCXXWarningIfSupport.cmake @@ -5,7 +5,7 @@ function(target_enable_cxx_warning_if_supported TARGET) set(multiValueArgs FLAGS) cmake_parse_arguments(TARGET_ENABLE_WARNING "" "" "${multiValueArgs}" ${ARGN}) - foreach (WARNING_FLAG IN LISTS TARGET_ENABLE_WARNING_FLAGS) + foreach(WARNING_FLAG IN LISTS TARGET_ENABLE_WARNING_FLAGS) string(REPLACE "-" "_" WARNING_FLAG_STRIPPED ${WARNING_FLAG}) # Note that gcc ignores unknown flags of the form "-Wno-warning" @@ -13,31 +13,33 @@ function(target_enable_cxx_warning_if_supported TARGET) # positive form as an additional flag which it will choke on (if # it doesn't exist). See: https://gcc.gnu.org/wiki/FAQ#wnowarning string(FIND ${WARNING_FLAG} "Wno-" NEGATIVE_FLAG_${WARNING_FLAG_STRIPPED}) - if (NEGATIVE_FLAG_${WARNING_FLAG_STRIPPED} EQUAL -1) + if(NEGATIVE_FLAG_${WARNING_FLAG_STRIPPED} EQUAL -1) set(IS_NEGATIVE_FLAG_${WARNING_FLAG_STRIPPED} FALSE) else() set(IS_NEGATIVE_FLAG_${WARNING_FLAG_STRIPPED} TRUE) endif() - if (IS_NEGATIVE_FLAG_${WARNING_FLAG_STRIPPED}) + if(IS_NEGATIVE_FLAG_${WARNING_FLAG_STRIPPED}) set(ORIGINAL_FLAG ${WARNING_FLAG}) string(REPLACE "no-" "" WARNING_FLAG ${WARNING_FLAG}) message(STATUS "Found negative flag: ${ORIGINAL_FLAG}\n" - " replaced with ${WARNING_FLAG}") + " replaced with ${WARNING_FLAG}" + ) endif() check_cxx_compiler_flag(${WARNING_FLAG} HAS_FLAG_${WARNING_FLAG_STRIPPED}) - if (IS_NEGATIVE_FLAG_${WARNING_FLAG_STRIPPED}) + if(IS_NEGATIVE_FLAG_${WARNING_FLAG_STRIPPED}) set(WARNING_FLAG ${ORIGINAL_FLAG}) endif() - if (HAS_FLAG_${WARNING_FLAG_STRIPPED}) - message(STATUS "Warning flag is supported by compiler: ${WARNING_FLAG}") + if(HAS_FLAG_${WARNING_FLAG_STRIPPED}) + message(STATUS "Warning flag is supported by compiler: ${WARNING_FLAG}") - target_compile_options(${TARGET} PRIVATE - $<$>:${WARNING_FLAG} > - $<$:-Xcompiler=${WARNING_FLAG} > + target_compile_options( + ${TARGET} + PRIVATE $<$>:${WARNING_FLAG} > + $<$:-Xcompiler=${WARNING_FLAG} > ) else() message(STATUS "Warning flag not supported by compiler: ${WARNING_FLAG}") diff --git a/cmake/FindBash.cmake b/cmake/FindBash.cmake index feb195f9f1..faa5303646 100644 --- a/cmake/FindBash.cmake +++ b/cmake/FindBash.cmake @@ -11,27 +11,30 @@ # Bash_VERSION - Bash version # Bash_EXECUTABLE - Path to bash executable -find_program(Bash_EXECUTABLE - bash - ) +find_program(Bash_EXECUTABLE bash) mark_as_advanced(Bash_EXECUTABLE) -if (Bash_EXECUTABLE) - execute_process(COMMAND "${Bash_EXECUTABLE}" --version +if(Bash_EXECUTABLE) + execute_process( + COMMAND "${Bash_EXECUTABLE}" --version RESULT_VARIABLE _bash_runs OUTPUT_VARIABLE _bash_stdout OUTPUT_STRIP_TRAILING_WHITESPACE - ) + ) if(_bash_stdout MATCHES "version ([0-9]+\\.[0-9]+\\.[0-9]+)") set(Bash_VERSION "${CMAKE_MATCH_1}") else() - message (WARNING "Failed to determine version of Bash interpreter (${Bash_EXECUTABLE})! Error:\n${_Bash_STDERR}") + message( + WARNING + "Failed to determine version of Bash interpreter (${Bash_EXECUTABLE})! Error:\n${_Bash_STDERR}" + ) endif() endif() include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Bash +find_package_handle_standard_args( + Bash VERSION_VAR Bash_VERSION REQUIRED_VARS Bash_EXECUTABLE - ) +) diff --git a/cmake/FindClangFormat.cmake b/cmake/FindClangFormat.cmake index c940002c3b..cd5ca5865b 100644 --- a/cmake/FindClangFormat.cmake +++ b/cmake/FindClangFormat.cmake @@ -3,27 +3,23 @@ # Taken from https://github.com/ttroy50/cmake-examples commit 64bd54a # This file is under MIT Licence -if (NOT ClangFormat_BIN_NAME) +if(NOT ClangFormat_BIN_NAME) set(ClangFormat_BIN_NAME clang-format) endif() # if custom path check there first -if (ClangFormat_ROOT_DIR) - find_program(ClangFormat_BIN - NAMES - ${ClangFormat_BIN_NAME} - PATHS - "${ClangFormat_ROOT_DIR}" - NO_DEFAULT_PATH) +if(ClangFormat_ROOT_DIR) + find_program( + ClangFormat_BIN + NAMES ${ClangFormat_BIN_NAME} + PATHS "${ClangFormat_ROOT_DIR}" + NO_DEFAULT_PATH + ) endif() find_program(ClangFormat_BIN NAMES ${ClangFormat_BIN_NAME}) include(FindPackageHandleStandardArgs) -find_package_handle_standard_args( - ClangFormat - DEFAULT_MSG - ClangFormat_BIN) +find_package_handle_standard_args(ClangFormat DEFAULT_MSG ClangFormat_BIN) -mark_as_advanced( - ClangFormat_BIN) +mark_as_advanced(ClangFormat_BIN) diff --git a/cmake/FindCython.cmake b/cmake/FindCython.cmake index 3b98cde89e..5a58cf4335 100644 --- a/cmake/FindCython.cmake +++ b/cmake/FindCython.cmake @@ -10,20 +10,22 @@ # CYTHON_FOUND - true if Cython was found # CYTHON_VERSION - Cython version -execute_process(COMMAND ${Python3_EXECUTABLE} -c "import cython ; print(cython.__version__)" +execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import cython ; print(cython.__version__)" RESULT_VARIABLE _cython_runs OUTPUT_VARIABLE CYTHON_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE - ) +) -if (${_cython_runs} EQUAL 0) +if(${_cython_runs} EQUAL 0) set(CYTHON_RUNS TRUE) else() set(CYTHON_RUNS FALSE) endif() include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Cython +find_package_handle_standard_args( + Cython VERSION_VAR CYTHON_VERSION REQUIRED_VARS CYTHON_RUNS - ) +) diff --git a/cmake/FindFFTW.cmake b/cmake/FindFFTW.cmake index e1940c687d..7ca5518a6f 100644 --- a/cmake/FindFFTW.cmake +++ b/cmake/FindFFTW.cmake @@ -24,72 +24,78 @@ # ``FFTW_DEBUG`` # Set to TRUE to get extra debugging output -if (FFTW_INCLUDE_DIRS) +if(FFTW_INCLUDE_DIRS) # Already in cache, be silent - set (FFTW_FIND_QUIETLY TRUE) -endif (FFTW_INCLUDE_DIRS) + set(FFTW_FIND_QUIETLY TRUE) +endif(FFTW_INCLUDE_DIRS) -if (EXISTS ${FFTW_ROOT}) +if(EXISTS ${FFTW_ROOT}) # Make sure FFTW_ROOT is an absolute path by setting it as a 'FILEPATH' - set (FFTW_ROOT "" CACHE FILEPATH "Location of the FFTW library") + set(FFTW_ROOT + "" + CACHE FILEPATH "Location of the FFTW library" + ) endif() -find_program(FFTW_WISDOM "fftw-wisdom" +find_program( + FFTW_WISDOM "fftw-wisdom" PATHS "${FFTW_ROOT}" PATH_SUFFIXES bin NO_DEFAULT_PATH DOC "Path to fftw-wisdom executable" - ) +) -find_program(FFTW_WISDOM "fftw-wisdom" - DOC "Path to fftw-wisdom executable" - ) -if (FFTW_DEBUG) +find_program(FFTW_WISDOM "fftw-wisdom" DOC "Path to fftw-wisdom executable") +if(FFTW_DEBUG) message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " FFTW_WISDOM = ${FFTW_WISDOM}" - ) + " FFTW_WISDOM = ${FFTW_WISDOM}" + ) endif() get_filename_component(FFTW_WISDOM_TMP "${FFTW_WISDOM}" DIRECTORY) get_filename_component(FFTW_HINT_DIR "${FFTW_WISDOM_TMP}" DIRECTORY) -find_path(FFTW_INCLUDE_DIRS +find_path( + FFTW_INCLUDE_DIRS NAMES fftw3.h DOC "FFTW include directory" HINTS "${FFTW_HINT_DIR}" PATH_SUFFIXES "include" - ) -if (FFTW_DEBUG) +) +if(FFTW_DEBUG) message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " FFTW_INCLUDE_DIRS = ${FFTW_INCLUDE_DIRS}" - " FFTW_HINT_DIR = ${FFTW_HINT_DIR}" - ) + " FFTW_INCLUDE_DIRS = ${FFTW_INCLUDE_DIRS}" + " FFTW_HINT_DIR = ${FFTW_HINT_DIR}" + ) endif() -find_library (FFTW_LIBRARIES +find_library( + FFTW_LIBRARIES NAMES fftw3 DOC "FFTW library location" HINTS "${FFTW_HINT_DIR}" PATH_SUFFIXES "lib" "lib64" - ) -if (FFTW_DEBUG) +) +if(FFTW_DEBUG) message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " FFTW_LIBRARIES = ${FFTW_LIBRARIES}" - " FFTW_HINT_DIR = ${FFTW_HINT_DIR}" - ) + " FFTW_LIBRARIES = ${FFTW_LIBRARIES}" + " FFTW_HINT_DIR = ${FFTW_HINT_DIR}" + ) endif() # handle the QUIETLY and REQUIRED arguments and set FFTW_FOUND to TRUE if # all listed variables are TRUE -include (FindPackageHandleStandardArgs) -find_package_handle_standard_args (FFTW DEFAULT_MSG FFTW_LIBRARIES FFTW_INCLUDE_DIRS) +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + FFTW DEFAULT_MSG FFTW_LIBRARIES FFTW_INCLUDE_DIRS +) -mark_as_advanced (FFTW_LIBRARIES FFTW_INCLUDE_DIRS) +mark_as_advanced(FFTW_LIBRARIES FFTW_INCLUDE_DIRS) -if (FFTW_FOUND AND NOT TARGET FFTW::FFTW) +if(FFTW_FOUND AND NOT TARGET FFTW::FFTW) add_library(FFTW::FFTW UNKNOWN IMPORTED) - set_target_properties(FFTW::FFTW PROPERTIES - IMPORTED_LOCATION "${FFTW_LIBRARIES}" - INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIRS}" - ) + set_target_properties( + FFTW::FFTW PROPERTIES IMPORTED_LOCATION "${FFTW_LIBRARIES}" + INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIRS}" + ) endif() diff --git a/cmake/FindHYPRE.cmake b/cmake/FindHYPRE.cmake index 1b9a5ca6f9..d2a1b4746c 100644 --- a/cmake/FindHYPRE.cmake +++ b/cmake/FindHYPRE.cmake @@ -6,54 +6,62 @@ include(FindPackageHandleStandardArgs) find_package(HYPRE CONFIG QUIET) -if (HYPRE_FOUND) +if(HYPRE_FOUND) message(STATUS "Found HYPRE: ${HYPRE_VERSION}") return() endif() -find_path(HYPRE_INCLUDE_DIR +find_path( + HYPRE_INCLUDE_DIR NAMES HYPRE.h - DOC "HYPRE include directories" - REQUIRED + DOC "HYPRE include directories" REQUIRED PATH_SUFFIXES include include/hypre ) -find_library(HYPRE_LIBRARY +find_library( + HYPRE_LIBRARY NAMES HYPRE - DOC "HYPRE library" - REQUIRED + DOC "HYPRE library" REQUIRED PATH_SUFFIXES lib64 lib - ) +) -if (HYPRE_INCLUDE_DIR) +if(HYPRE_INCLUDE_DIR) file(READ "${HYPRE_INCLUDE_DIR}/HYPRE_config.h" HYPRE_CONFIG_FILE) - string(REGEX MATCH ".*#define HYPRE_RELEASE_VERSION \"([0-9]+)\\.([0-9]+)\\.([0-9]+)\".*" - _ "${HYPRE_CONFIG_FILE}") + string( + REGEX MATCH + ".*#define HYPRE_RELEASE_VERSION \"([0-9]+)\\.([0-9]+)\\.([0-9]+)\".*" + _ "${HYPRE_CONFIG_FILE}" + ) set(HYPRE_VERSION_MAJOR ${CMAKE_MATCH_1}) set(HYPRE_VERSION_MINOR ${CMAKE_MATCH_2}) set(HYPRE_VERSION_PATCH ${CMAKE_MATCH_3}) - set(HYPRE_VERSION "${HYPRE_VERSION_MAJOR}.${HYPRE_VERSION_MINOR}.${HYPRE_VERSION_PATCH}") + set(HYPRE_VERSION + "${HYPRE_VERSION_MAJOR}.${HYPRE_VERSION_MINOR}.${HYPRE_VERSION_PATCH}" + ) endif() -if (HYPRE_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ]" - " HYPRE_ROOT = ${HYPRE_ROOT}" - " HYPRE_INCLUDE_DIR = ${HYPRE_INCLUDE_DIR}" - " HYPRE_LIBRARY = ${HYPRE_LIBRARY}" - ) +if(HYPRE_DEBUG) + message( + STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ]" + " HYPRE_ROOT = ${HYPRE_ROOT}" + " HYPRE_INCLUDE_DIR = ${HYPRE_INCLUDE_DIR}" + " HYPRE_LIBRARY = ${HYPRE_LIBRARY}" + ) endif() mark_as_advanced(HYPRE_INCLUDE_DIR HYPRE_LIBRARY) -find_package_handle_standard_args(HYPRE +find_package_handle_standard_args( + HYPRE REQUIRED_VARS HYPRE_LIBRARY HYPRE_INCLUDE_DIR VERSION_VAR HYPRE_VERSION - ) +) -if (HYPRE_FOUND AND NOT TARGET HYPRE::HYPRE) +if(HYPRE_FOUND AND NOT TARGET HYPRE::HYPRE) add_library(HYPRE::HYPRE UNKNOWN IMPORTED) - set_target_properties(HYPRE::HYPRE PROPERTIES - IMPORTED_LOCATION "${HYPRE_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${HYPRE_INCLUDE_DIR}" - ) + set_target_properties( + HYPRE::HYPRE + PROPERTIES IMPORTED_LOCATION "${HYPRE_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${HYPRE_INCLUDE_DIR}" + ) endif() diff --git a/cmake/FindLibuuid.cmake b/cmake/FindLibuuid.cmake index 91880487a2..2492388b32 100644 --- a/cmake/FindLibuuid.cmake +++ b/cmake/FindLibuuid.cmake @@ -21,38 +21,41 @@ # ``Libuuid_DEBUG`` # Set to TRUE to get extra debugging output -include (FindPackageHandleStandardArgs) +include(FindPackageHandleStandardArgs) -if (WIN32) +if(WIN32) find_package_handle_standard_args(Libuuid DEFAULT_MSG) return() endif() -if (APPLE) +if(APPLE) find_library(CFLIB CoreFoundation) find_package_handle_standard_args(Libuuid DEFAULT_MSG CFLIB) mark_as_advanced(${CFLIB}) - if (Libuuid_FOUND AND NOT TARGET Libuuid::libuuid) + if(Libuuid_FOUND AND NOT TARGET Libuuid::libuuid) add_library(Libuuid::libuuid UNKNOWN IMPORTED) - set_target_properties(Libuuid::libuuid PROPERTIES - IMPORTED_LOCATION ${CFLIB} - ) + set_target_properties( + Libuuid::libuuid PROPERTIES IMPORTED_LOCATION ${CFLIB} + ) endif() return() -endif () +endif() find_path(Libuuid_INCLUDE_DIRS uuid/uuid.h) find_library(Libuuid_LIBRARIES uuid) -find_package_handle_standard_args(Libuuid DEFAULT_MSG Libuuid_LIBRARIES Libuuid_INCLUDE_DIRS) +find_package_handle_standard_args( + Libuuid DEFAULT_MSG Libuuid_LIBRARIES Libuuid_INCLUDE_DIRS +) mark_as_advanced(Libuuid_LIBRARIES Libuuid_INCLUDE_DIRS) -if (Libuuid_FOUND AND NOT TARGET Libuuid::libuuid) +if(Libuuid_FOUND AND NOT TARGET Libuuid::libuuid) add_library(Libuuid::libuuid UNKNOWN IMPORTED) - set_target_properties(Libuuid::libuuid PROPERTIES - IMPORTED_LOCATION "${Libuuid_LIBRARIES}" - INTERFACE_INCLUDE_DIRECTORIES "${Libuuid_INCLUDE_DIRS}" - ) + set_target_properties( + Libuuid::libuuid + PROPERTIES IMPORTED_LOCATION "${Libuuid_LIBRARIES}" + INTERFACE_INCLUDE_DIRECTORIES "${Libuuid_INCLUDE_DIRS}" + ) endif() diff --git a/cmake/FindNumpy.cmake b/cmake/FindNumpy.cmake index b6de6e3e35..cfdbdcc96c 100644 --- a/cmake/FindNumpy.cmake +++ b/cmake/FindNumpy.cmake @@ -11,46 +11,57 @@ # Numpy_VERSION # Numpy_INCLUDE_DIR - find_package(Python3 3.6 COMPONENTS Interpreter Development) -if (NOT Python3_FOUND) - message(STATUS "Could not find numpy as python was not found. Maybe the developement package is missing?") +if(NOT Python3_FOUND) + message( + STATUS + "Could not find numpy as python was not found. Maybe the developement package is missing?" + ) set(Numpy_FOUND ${Python3_FOUND}) return() endif() -if (NOT Numpy_FOUND) - execute_process(COMMAND ${Python3_EXECUTABLE} -c "import numpy ; print(numpy.__version__)" +if(NOT Numpy_FOUND) + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import numpy ; print(numpy.__version__)" OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE Numpy_VERSION - ) - execute_process(COMMAND ${Python3_EXECUTABLE} -c "import numpy ; print(numpy.get_include())" + ) + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import numpy ; print(numpy.get_include())" OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE _numpy_include_dirs - ) + ) endif() -if (Numpy_DEBUG) - message(STATUS "Looking for numpy headers in: ${_numpy_include_dirs} ${Python3_INCLUDE_DIRS}") +if(Numpy_DEBUG) + message( + STATUS + "Looking for numpy headers in: ${_numpy_include_dirs} ${Python3_INCLUDE_DIRS}" + ) endif() -find_path(Numpy_INCLUDE_DIR - numpy/arrayobject.h +find_path( + Numpy_INCLUDE_DIR numpy/arrayobject.h PATHS "${_numpy_include_dirs}" "${Python3_INCLUDE_DIRS}" PATH_SUFFIXES numpy/core/include - ) +) -if (NOT Numpy_INCLUDE_DIR) - message(STATUS "Numpy headers not found -- do you need to install the development package?") +if(NOT Numpy_INCLUDE_DIR) + message( + STATUS + "Numpy headers not found -- do you need to install the development package?" + ) endif() set(Numpy_INCLUDE_DIRS ${Numpy_INCLUDE_DIR}) include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Numpy +find_package_handle_standard_args( + Numpy VERSION_VAR Numpy_VERSION REQUIRED_VARS Numpy_INCLUDE_DIR - ) +) mark_as_advanced(Numpy_INCLUDE_DIR) diff --git a/cmake/FindPETSc.cmake b/cmake/FindPETSc.cmake index c93d464673..f0595a7a68 100644 --- a/cmake/FindPETSc.cmake +++ b/cmake/FindPETSc.cmake @@ -25,23 +25,23 @@ find_package(MPI REQUIRED) -set(PETSC_VALID_COMPONENTS - C - CXX) +set(PETSC_VALID_COMPONENTS C CXX) if(NOT PETSc_FIND_COMPONENTS) - get_property (_enabled_langs GLOBAL PROPERTY ENABLED_LANGUAGES) - if ("C" IN_LIST _enabled_langs) + get_property(_enabled_langs GLOBAL PROPERTY ENABLED_LANGUAGES) + if("C" IN_LIST _enabled_langs) set(PETSC_LANGUAGE_BINDINGS "C") - else () + else() set(PETSC_LANGUAGE_BINDINGS "CXX") - endif () + endif() else() # Right now, this is designed for compatability with the --with-clanguage option, so # only allow one item in the components list. list(LENGTH ${PETSc_FIND_COMPONENTS} components_length) if(${components_length} GREATER 1) - message(FATAL_ERROR "Only one component for PETSc is allowed to be specified") + message( + FATAL_ERROR "Only one component for PETSc is allowed to be specified" + ) endif() # This is a stub for allowing multiple components should that time ever come. Perhaps # to also test Fortran bindings? @@ -61,218 +61,328 @@ if(NOT PETSC_DIR) endif() endif() -function (petsc_get_version) - if (EXISTS "${PETSC_DIR}/include/petscversion.h") - file (STRINGS "${PETSC_DIR}/include/petscversion.h" vstrings REGEX "#define PETSC_VERSION_(RELEASE|MAJOR|MINOR|SUBMINOR|PATCH) ") - foreach (line ${vstrings}) - string (REGEX REPLACE " +" ";" fields ${line}) # break line into three fields (the first is always "#define") - list (GET fields 1 var) - list (GET fields 2 val) - set (${var} ${val} PARENT_SCOPE) - set (${var} ${val}) # Also in local scope so we have access below - endforeach () - if (PETSC_VERSION_RELEASE) - if ($(PETSC_VERSION_PATCH) GREATER 0) - set (PETSC_VERSION "${PETSC_VERSION_MAJOR}.${PETSC_VERSION_MINOR}.${PETSC_VERSION_SUBMINOR}p${PETSC_VERSION_PATCH}" CACHE INTERNAL "PETSc version") - else () - set (PETSC_VERSION "${PETSC_VERSION_MAJOR}.${PETSC_VERSION_MINOR}.${PETSC_VERSION_SUBMINOR}" CACHE INTERNAL "PETSc version") - endif () - else () +function(petsc_get_version) + if(EXISTS "${PETSC_DIR}/include/petscversion.h") + file(STRINGS "${PETSC_DIR}/include/petscversion.h" vstrings + REGEX "#define PETSC_VERSION_(RELEASE|MAJOR|MINOR|SUBMINOR|PATCH) " + ) + foreach(line ${vstrings}) + string(REGEX REPLACE " +" ";" fields ${line} + )# break line into three fields (the first is always "#define") + list(GET fields 1 var) + list(GET fields 2 val) + set(${var} + ${val} + PARENT_SCOPE + ) + set(${var} ${val}) # Also in local scope so we have access below + endforeach() + if(PETSC_VERSION_RELEASE) + if($(PETSC_VERSION_PATCH) GREATER 0) + set(PETSC_VERSION + "${PETSC_VERSION_MAJOR}.${PETSC_VERSION_MINOR}.${PETSC_VERSION_SUBMINOR}p${PETSC_VERSION_PATCH}" + CACHE INTERNAL "PETSc version" + ) + else() + set(PETSC_VERSION + "${PETSC_VERSION_MAJOR}.${PETSC_VERSION_MINOR}.${PETSC_VERSION_SUBMINOR}" + CACHE INTERNAL "PETSc version" + ) + endif() + else() # make dev version compare higher than any patch level of a released version - set (PETSC_VERSION "${PETSC_VERSION_MAJOR}.${PETSC_VERSION_MINOR}.${PETSC_VERSION_SUBMINOR}.99" CACHE INTERNAL "PETSc version") - endif () - else () - message (SEND_ERROR "PETSC_DIR can not be used, ${PETSC_DIR}/include/petscversion.h does not exist") - endif () -endfunction () + set(PETSC_VERSION + "${PETSC_VERSION_MAJOR}.${PETSC_VERSION_MINOR}.${PETSC_VERSION_SUBMINOR}.99" + CACHE INTERNAL "PETSc version" + ) + endif() + else() + message( + SEND_ERROR + "PETSC_DIR can not be used, ${PETSC_DIR}/include/petscversion.h does not exist" + ) + endif() +endfunction() # Debian uses versioned paths e.g /usr/lib/petscdir/3.5/ -file (GLOB DEB_PATHS "/usr/lib/petscdir/*") +file(GLOB DEB_PATHS "/usr/lib/petscdir/*") -find_path (PETSC_DIR include/petsc.h +find_path( + PETSC_DIR include/petsc.h HINTS ENV PETSC_DIR - PATHS - /usr/lib/petsc - # Debian paths - ${DEB_PATHS} - # Arch Linux path - /opt/petsc/linux-c-opt - # MacPorts path - /opt/local/lib/petsc - $ENV{HOME}/petsc - DOC "PETSc Directory") - -find_program (MAKE_EXECUTABLE NAMES make gmake) - -if (PETSC_DIR AND NOT PETSC_ARCH) - set (_petsc_arches - $ENV{PETSC_ARCH} # If set, use environment variable first - linux-gnu-c-debug linux-gnu-c-opt # Debian defaults - x86_64-unknown-linux-gnu i386-unknown-linux-gnu) - set (petscconf "NOTFOUND" CACHE FILEPATH "Cleared" FORCE) - foreach (arch ${_petsc_arches}) - if (NOT PETSC_ARCH) - find_path (petscconf petscconf.h + PATHS /usr/lib/petsc + # Debian paths + ${DEB_PATHS} + # Arch Linux path + /opt/petsc/linux-c-opt + # MacPorts path + /opt/local/lib/petsc + $ENV{HOME}/petsc + DOC "PETSc Directory" +) + +find_program(MAKE_EXECUTABLE NAMES make gmake) + +if(PETSC_DIR AND NOT PETSC_ARCH) + set(_petsc_arches + $ENV{PETSC_ARCH} # If set, use environment variable first + linux-gnu-c-debug linux-gnu-c-opt # Debian defaults + x86_64-unknown-linux-gnu i386-unknown-linux-gnu + ) + set(petscconf + "NOTFOUND" + CACHE FILEPATH "Cleared" FORCE + ) + foreach(arch ${_petsc_arches}) + if(NOT PETSC_ARCH) + find_path( + petscconf petscconf.h HINTS ${PETSC_DIR} PATH_SUFFIXES ${arch}/include bmake/${arch} - NO_DEFAULT_PATH) - if (petscconf) - set (PETSC_ARCH "${arch}" CACHE STRING "PETSc build architecture") - endif (petscconf) - endif (NOT PETSC_ARCH) - endforeach (arch) - set (petscconf "NOTFOUND" CACHE INTERNAL "Scratch variable" FORCE) -endif (PETSC_DIR AND NOT PETSC_ARCH) - -set (petsc_slaves LIBRARIES_SYS LIBRARIES_VEC LIBRARIES_MAT LIBRARIES_DM LIBRARIES_KSP LIBRARIES_SNES LIBRARIES_TS - INCLUDE_DIR INCLUDE_CONF) -include (FindPackageMultipass) -find_package_multipass (PETSc petsc_config_current - STATES DIR ARCH - DEPENDENTS INCLUDES LIBRARIES COMPILER MPIEXEC ${petsc_slaves}) + NO_DEFAULT_PATH + ) + if(petscconf) + set(PETSC_ARCH + "${arch}" + CACHE STRING "PETSc build architecture" + ) + endif(petscconf) + endif(NOT PETSC_ARCH) + endforeach(arch) + set(petscconf + "NOTFOUND" + CACHE INTERNAL "Scratch variable" FORCE + ) +endif(PETSC_DIR AND NOT PETSC_ARCH) + +set(petsc_slaves + LIBRARIES_SYS + LIBRARIES_VEC + LIBRARIES_MAT + LIBRARIES_DM + LIBRARIES_KSP + LIBRARIES_SNES + LIBRARIES_TS + INCLUDE_DIR + INCLUDE_CONF +) +include(FindPackageMultipass) +find_package_multipass( + PETSc + petsc_config_current + STATES + DIR + ARCH + DEPENDENTS + INCLUDES + LIBRARIES + COMPILER + MPIEXEC + ${petsc_slaves} +) # Determine whether the PETSc layout is old-style (through 2.3.3) or # new-style (>= 3.0.0) -if (EXISTS "${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/petscvariables") # > 3.5 - set (petsc_conf_rules "${PETSC_DIR}/lib/petsc/conf/rules") - set (petsc_conf_variables "${PETSC_DIR}/lib/petsc/conf/variables") -elseif (EXISTS "${PETSC_DIR}/${PETSC_ARCH}/include/petscconf.h") # > 2.3.3 - set (petsc_conf_rules "${PETSC_DIR}/conf/rules") - set (petsc_conf_variables "${PETSC_DIR}/conf/variables") -elseif (EXISTS "${PETSC_DIR}/bmake/${PETSC_ARCH}/petscconf.h") # <= 2.3.3 - set (petsc_conf_rules "${PETSC_DIR}/bmake/common/rules") - set (petsc_conf_variables "${PETSC_DIR}/bmake/common/variables") -elseif (PETSC_DIR) - message (SEND_ERROR "The pair PETSC_DIR=${PETSC_DIR} PETSC_ARCH=${PETSC_ARCH} do not specify a valid PETSc installation") -endif () - -if (petsc_conf_rules AND petsc_conf_variables AND NOT petsc_config_current) +if(EXISTS "${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/petscvariables") # > 3.5 + set(petsc_conf_rules "${PETSC_DIR}/lib/petsc/conf/rules") + set(petsc_conf_variables "${PETSC_DIR}/lib/petsc/conf/variables") +elseif(EXISTS "${PETSC_DIR}/${PETSC_ARCH}/include/petscconf.h") # > 2.3.3 + set(petsc_conf_rules "${PETSC_DIR}/conf/rules") + set(petsc_conf_variables "${PETSC_DIR}/conf/variables") +elseif(EXISTS "${PETSC_DIR}/bmake/${PETSC_ARCH}/petscconf.h") # <= 2.3.3 + set(petsc_conf_rules "${PETSC_DIR}/bmake/common/rules") + set(petsc_conf_variables "${PETSC_DIR}/bmake/common/variables") +elseif(PETSC_DIR) + message( + SEND_ERROR + "The pair PETSC_DIR=${PETSC_DIR} PETSC_ARCH=${PETSC_ARCH} do not specify a valid PETSc installation" + ) +endif() + +if(petsc_conf_rules + AND petsc_conf_variables + AND NOT petsc_config_current +) petsc_get_version() # Put variables into environment since they are needed to get # configuration (petscvariables) in the PETSc makefile - set (ENV{PETSC_DIR} "${PETSC_DIR}") - set (ENV{PETSC_ARCH} "${PETSC_ARCH}") + set(ENV{PETSC_DIR} "${PETSC_DIR}") + set(ENV{PETSC_ARCH} "${PETSC_ARCH}") # A temporary makefile to probe the PETSc configuration - set (petsc_config_makefile "${PROJECT_BINARY_DIR}/Makefile.petsc") - file (WRITE "${petsc_config_makefile}" -"## This file was autogenerated by FindPETSc.cmake + set(petsc_config_makefile "${PROJECT_BINARY_DIR}/Makefile.petsc") + file( + WRITE "${petsc_config_makefile}" + "## This file was autogenerated by FindPETSc.cmake # PETSC_DIR = ${PETSC_DIR} # PETSC_ARCH = ${PETSC_ARCH} include ${petsc_conf_rules} include ${petsc_conf_variables} show : \t-@echo -n \${\${VARIABLE}} -") - - macro (PETSC_GET_VARIABLE name var) - set (${var} "NOTFOUND" CACHE INTERNAL "Cleared" FORCE) - execute_process (COMMAND ${MAKE_EXECUTABLE} --no-print-directory -f ${petsc_config_makefile} show VARIABLE=${name} +" + ) + + macro(PETSC_GET_VARIABLE name var) + set(${var} + "NOTFOUND" + CACHE INTERNAL "Cleared" FORCE + ) + execute_process( + COMMAND ${MAKE_EXECUTABLE} --no-print-directory -f + ${petsc_config_makefile} show VARIABLE=${name} OUTPUT_VARIABLE ${var} - RESULT_VARIABLE petsc_return) - endmacro (PETSC_GET_VARIABLE) - petsc_get_variable (PETSC_LIB_DIR petsc_lib_dir) - petsc_get_variable (PETSC_EXTERNAL_LIB_BASIC petsc_libs_external) - petsc_get_variable (PETSC_CCPPFLAGS petsc_cpp_line) - petsc_get_variable (PETSC_INCLUDE petsc_include) - petsc_get_variable (PCC petsc_cc) - petsc_get_variable (PCC_FLAGS petsc_cc_flags) - petsc_get_variable (MPIEXEC petsc_mpiexec) + RESULT_VARIABLE petsc_return + ) + endmacro(PETSC_GET_VARIABLE) + petsc_get_variable(PETSC_LIB_DIR petsc_lib_dir) + petsc_get_variable(PETSC_EXTERNAL_LIB_BASIC petsc_libs_external) + petsc_get_variable(PETSC_CCPPFLAGS petsc_cpp_line) + petsc_get_variable(PETSC_INCLUDE petsc_include) + petsc_get_variable(PCC petsc_cc) + petsc_get_variable(PCC_FLAGS petsc_cc_flags) + petsc_get_variable(MPIEXEC petsc_mpiexec) # We are done with the temporary Makefile, calling PETSC_GET_VARIABLE after this point is invalid! - file (REMOVE ${petsc_config_makefile}) + file(REMOVE ${petsc_config_makefile}) - include (ResolveCompilerPaths) + include(ResolveCompilerPaths) # Extract include paths and libraries from compile command line - resolve_includes (petsc_includes_all "${petsc_cpp_line}") + resolve_includes(petsc_includes_all "${petsc_cpp_line}") #on windows we need to make sure we're linking against the right #runtime library - if (WIN32) - if (petsc_cc_flags MATCHES "-MT") + if(WIN32) + if(petsc_cc_flags MATCHES "-MT") set(using_md False) - foreach(flag_var - CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE - CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO - CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE - CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO) + foreach( + flag_var + CMAKE_C_FLAGS + CMAKE_C_FLAGS_DEBUG + CMAKE_C_FLAGS_RELEASE + CMAKE_C_FLAGS_MINSIZEREL + CMAKE_C_FLAGS_RELWITHDEBINFO + CMAKE_CXX_FLAGS + CMAKE_CXX_FLAGS_DEBUG + CMAKE_CXX_FLAGS_RELEASE + CMAKE_CXX_FLAGS_MINSIZEREL + CMAKE_CXX_FLAGS_RELWITHDEBINFO + ) if(${flag_var} MATCHES "/MD") set(using_md True) endif(${flag_var} MATCHES "/MD") endforeach(flag_var) if(${using_md} MATCHES "True") - message(WARNING "PETSc was built with /MT, but /MD is currently set. - See http://www.cmake.org/Wiki/CMake_FAQ#How_can_I_build_my_MSVC_application_with_a_static_runtime.3F") + message( + WARNING + "PETSc was built with /MT, but /MD is currently set. + See http://www.cmake.org/Wiki/CMake_FAQ#How_can_I_build_my_MSVC_application_with_a_static_runtime.3F" + ) endif(${using_md} MATCHES "True") - endif (petsc_cc_flags MATCHES "-MT") - endif (WIN32) + endif(petsc_cc_flags MATCHES "-MT") + endif(WIN32) - include (CorrectWindowsPaths) + include(CorrectWindowsPaths) convert_cygwin_path(petsc_lib_dir) - message (STATUS "petsc_lib_dir ${petsc_lib_dir}") - - macro (PETSC_FIND_LIBRARY suffix name) - set (PETSC_LIBRARY_${suffix} "NOTFOUND" CACHE INTERNAL "Cleared" FORCE) # Clear any stale value, if we got here, we need to find it again - if (WIN32) - set (libname lib${name}) #windows expects "libfoo", linux expects "foo" - else (WIN32) - set (libname ${name}) - endif (WIN32) - find_library (PETSC_LIBRARY_${suffix} NAMES ${libname} HINTS ${petsc_lib_dir} NO_DEFAULT_PATH) - set (PETSC_LIBRARIES_${suffix} "${PETSC_LIBRARY_${suffix}}") - mark_as_advanced (PETSC_LIBRARY_${suffix}) - endmacro (PETSC_FIND_LIBRARY suffix name) + message(STATUS "petsc_lib_dir ${petsc_lib_dir}") + + macro(PETSC_FIND_LIBRARY suffix name) + set(PETSC_LIBRARY_${suffix} + "NOTFOUND" + CACHE INTERNAL "Cleared" FORCE + ) # Clear any stale value, if we got here, we need to find it again + if(WIN32) + set(libname lib${name}) #windows expects "libfoo", linux expects "foo" + else(WIN32) + set(libname ${name}) + endif(WIN32) + find_library( + PETSC_LIBRARY_${suffix} + NAMES ${libname} + HINTS ${petsc_lib_dir} + NO_DEFAULT_PATH + ) + set(PETSC_LIBRARIES_${suffix} "${PETSC_LIBRARY_${suffix}}") + mark_as_advanced(PETSC_LIBRARY_${suffix}) + endmacro( + PETSC_FIND_LIBRARY + suffix + name + ) # Look for petscvec first, if it doesn't exist, we must be using single-library - petsc_find_library (VEC petscvec) - if (PETSC_LIBRARY_VEC) - petsc_find_library (SYS "petscsys;petsc") # libpetscsys is called libpetsc prior to 3.1 (when single-library was introduced) - petsc_find_library (MAT petscmat) - petsc_find_library (DM petscdm) - petsc_find_library (KSP petscksp) - petsc_find_library (SNES petscsnes) - petsc_find_library (TS petscts) - macro (PETSC_JOIN libs deps) - list (APPEND PETSC_LIBRARIES_${libs} ${PETSC_LIBRARIES_${deps}}) - endmacro (PETSC_JOIN libs deps) - petsc_join (VEC SYS) - petsc_join (MAT VEC) - petsc_join (DM MAT) - petsc_join (KSP DM) - petsc_join (SNES KSP) - petsc_join (TS SNES) - petsc_join (ALL TS) - else () - set (PETSC_LIBRARY_VEC "NOTFOUND" CACHE INTERNAL "Cleared" FORCE) # There is no libpetscvec - petsc_find_library (SINGLE petsc) + petsc_find_library(VEC petscvec) + if(PETSC_LIBRARY_VEC) + petsc_find_library(SYS "petscsys;petsc") + # libpetscsys is called libpetsc prior to 3.1 (when single-library was introduced) + petsc_find_library(MAT petscmat) + petsc_find_library(DM petscdm) + petsc_find_library(KSP petscksp) + petsc_find_library(SNES petscsnes) + petsc_find_library(TS petscts) + macro(PETSC_JOIN libs deps) + list(APPEND PETSC_LIBRARIES_${libs} ${PETSC_LIBRARIES_${deps}}) + endmacro( + PETSC_JOIN + libs + deps + ) + petsc_join(VEC SYS) + petsc_join(MAT VEC) + petsc_join(DM MAT) + petsc_join(KSP DM) + petsc_join(SNES KSP) + petsc_join(TS SNES) + petsc_join(ALL TS) + else() + set(PETSC_LIBRARY_VEC + "NOTFOUND" + CACHE INTERNAL "Cleared" FORCE + ) # There is no libpetscvec + petsc_find_library(SINGLE petsc) # Debian 9/Ubuntu 16.04 uses _real and _complex extensions when using libraries in /usr/lib/petsc. - if (NOT PETSC_LIBRARY_SINGLE) - petsc_find_library (SINGLE petsc_real) + if(NOT PETSC_LIBRARY_SINGLE) + petsc_find_library(SINGLE petsc_real) endif() - if (NOT PETSC_LIBRARY_SINGLE) - petsc_find_library (SINGLE petsc_complex) + if(NOT PETSC_LIBRARY_SINGLE) + petsc_find_library(SINGLE petsc_complex) endif() - foreach (pkg SYS VEC MAT DM KSP SNES TS ALL) - set (PETSC_LIBRARIES_${pkg} "${PETSC_LIBRARY_SINGLE}") - endforeach () - endif () - if (PETSC_LIBRARY_TS) - message (STATUS "Recognized PETSc install with separate libraries for each package") - else () - message (STATUS "Recognized PETSc install with single library for all packages") - endif () + foreach( + pkg + SYS + VEC + MAT + DM + KSP + SNES + TS + ALL + ) + set(PETSC_LIBRARIES_${pkg} "${PETSC_LIBRARY_SINGLE}") + endforeach() + endif() + if(PETSC_LIBRARY_TS) + message( + STATUS "Recognized PETSc install with separate libraries for each package" + ) + else() + message( + STATUS "Recognized PETSc install with single library for all packages" + ) + endif() include(Check${PETSC_LANGUAGE_BINDINGS}SourceRuns) - macro (petsc_test_compiles includes libraries runs) - message(STATUS "PETSc test with : ${includes} ${libraries}" ) - if (PETSC_VERSION VERSION_GREATER 3.1) - set (_PETSC_TSDestroy "TSDestroy(&ts)") - else () - set (_PETSC_TSDestroy "TSDestroy(ts)") - endif () + macro(petsc_test_compiles includes libraries runs) + message(STATUS "PETSc test with : ${includes} ${libraries}") + if(PETSC_VERSION VERSION_GREATER 3.1) + set(_PETSC_TSDestroy "TSDestroy(&ts)") + else() + set(_PETSC_TSDestroy "TSDestroy(ts)") + endif() - set(_PETSC_TEST_SOURCE " + set(_PETSC_TEST_SOURCE + " static const char help[] = \"PETSc test program.\"; #include int main(int argc,char *argv[]) { @@ -286,116 +396,197 @@ int main(int argc,char *argv[]) { ierr = PetscFinalize();CHKERRQ(ierr); return 0; } -") - multipass_source_compiles ("${includes}" "${libraries}" "${_PETSC_TEST_SOURCE}" ${runs} "${PETSC_LANGUAGE_BINDINGS}") - if (${${runs}}) - set (PETSC_EXECUTABLE_COMPILES "YES" CACHE BOOL - "Can the system successfully run a PETSc executable? This variable can be manually set to \"YES\" to force CMake to accept a given PETSc configuration, but this will almost always result in a broken build. If you change PETSC_DIR, PETSC_ARCH, or PETSC_CURRENT you would have to reset this variable." FORCE) - endif (${${runs}}) - endmacro () - +" + ) + multipass_source_compiles( + "${includes}" "${libraries}" "${_PETSC_TEST_SOURCE}" ${runs} + "${PETSC_LANGUAGE_BINDINGS}" + ) + if(${${runs}}) + set(PETSC_EXECUTABLE_COMPILES + "YES" + CACHE + BOOL + "Can the system successfully run a PETSc executable? This variable can be manually set to \"YES\" to force CMake to accept a given PETSc configuration, but this will almost always result in a broken build. If you change PETSC_DIR, PETSC_ARCH, or PETSC_CURRENT you would have to reset this variable." + FORCE + ) + endif(${${runs}}) + endmacro() - find_path (PETSC_INCLUDE_DIR petscts.h + find_path( + PETSC_INCLUDE_DIR petscts.h HINTS "${PETSC_DIR}" PATH_SUFFIXES include - NO_DEFAULT_PATH) - find_path (PETSC_INCLUDE_CONF petscconf.h + NO_DEFAULT_PATH + ) + find_path( + PETSC_INCLUDE_CONF petscconf.h HINTS "${PETSC_DIR}" PATH_SUFFIXES "${PETSC_ARCH}/include" "bmake/${PETSC_ARCH}" - NO_DEFAULT_PATH) - mark_as_advanced (PETSC_INCLUDE_DIR PETSC_INCLUDE_CONF) - set (petsc_includes_minimal ${PETSC_INCLUDE_CONF} ${PETSC_INCLUDE_DIR}) - - file (STRINGS "${PETSC_INCLUDE_CONF}/petscconf.h" PETSC_HAS_OPENMP REGEX "#define PETSC_HAVE_OPENMP 1") - if (PETSC_HAS_OPENMP) + NO_DEFAULT_PATH + ) + mark_as_advanced(PETSC_INCLUDE_DIR PETSC_INCLUDE_CONF) + set(petsc_includes_minimal ${PETSC_INCLUDE_CONF} ${PETSC_INCLUDE_DIR}) + + file(STRINGS "${PETSC_INCLUDE_CONF}/petscconf.h" PETSC_HAS_OPENMP + REGEX "#define PETSC_HAVE_OPENMP 1" + ) + if(PETSC_HAS_OPENMP) find_package(OpenMP REQUIRED) - set (petsc_openmp_library ";OpenMP::OpenMP_${PETSC_LANGUAGE_BINDINGS}") + set(petsc_openmp_library ";OpenMP::OpenMP_${PETSC_LANGUAGE_BINDINGS}") endif() - set (petsc_mpi_include_dirs "${MPI_${PETSC_LANGUAGE_BINDINGS}_INCLUDE_DIRS}") + set(petsc_mpi_include_dirs "${MPI_${PETSC_LANGUAGE_BINDINGS}_INCLUDE_DIRS}") #set (petsc_additional_libraries "MPI::MPI_${PETSC_LANGUAGE_BINDINGS}${petsc_openmp_library}") - petsc_test_compiles ("${petsc_includes_minimal};${petsc_mpi_include_dirs}" - "${PETSC_LIBRARIES_TS};${petsc_additional_libraries}" - petsc_works_minimal) - if (petsc_works_minimal) - message (STATUS "Minimal PETSc includes and libraries work. This probably means we are building with shared libs.") - set (petsc_includes_needed "${petsc_includes_minimal}") - else (petsc_works_minimal) # Minimal includes fail, see if just adding full includes fixes it - petsc_test_compiles ("${petsc_includes_all};${petsc_mpi_include_dirs}" + petsc_test_compiles( + "${petsc_includes_minimal};${petsc_mpi_include_dirs}" + "${PETSC_LIBRARIES_TS};${petsc_additional_libraries}" petsc_works_minimal + ) + if(petsc_works_minimal) + message( + STATUS + "Minimal PETSc includes and libraries work. This probably means we are building with shared libs." + ) + set(petsc_includes_needed "${petsc_includes_minimal}") + else(petsc_works_minimal + )# Minimal includes fail, see if just adding full includes fixes it + petsc_test_compiles( + "${petsc_includes_all};${petsc_mpi_include_dirs}" "${PETSC_LIBRARIES_TS};${petsc_additional_libraries}" - petsc_works_allincludes) - if (petsc_works_allincludes) # It does, we just need all the includes ( - message (STATUS "PETSc requires extra include paths, but links correctly with only interface libraries. This is an unexpected configuration (but it seems to work fine).") - set (petsc_includes_needed ${petsc_includes_all}) - else (petsc_works_allincludes) # We are going to need to link the external libs explicitly - resolve_libraries (petsc_libraries_external "${petsc_libs_external}") - foreach (pkg SYS VEC MAT DM KSP SNES TS ALL) - list (APPEND PETSC_LIBRARIES_${pkg} ${petsc_libraries_external}) - endforeach (pkg) - petsc_test_compiles ("${petsc_includes_minimal};${petsc_mpi_include_dirs}" + petsc_works_allincludes + ) + if(petsc_works_allincludes) # It does, we just need all the includes ( + message( + STATUS + "PETSc requires extra include paths, but links correctly with only interface libraries. This is an unexpected configuration (but it seems to work fine)." + ) + set(petsc_includes_needed ${petsc_includes_all}) + else(petsc_works_allincludes + )# We are going to need to link the external libs explicitly + resolve_libraries(petsc_libraries_external "${petsc_libs_external}") + foreach( + pkg + SYS + VEC + MAT + DM + KSP + SNES + TS + ALL + ) + list(APPEND PETSC_LIBRARIES_${pkg} ${petsc_libraries_external}) + endforeach(pkg) + petsc_test_compiles( + "${petsc_includes_minimal};${petsc_mpi_include_dirs}" "${PETSC_LIBRARIES_TS};${petsc_additional_libraries}" - petsc_works_alllibraries) - if (petsc_works_alllibraries) - message (STATUS "PETSc only need minimal includes, but requires explicit linking to all dependencies. This is expected when PETSc is built with static libraries.") - set (petsc_includes_needed ${petsc_includes_minimal}) - else (petsc_works_alllibraries) + petsc_works_alllibraries + ) + if(petsc_works_alllibraries) + message( + STATUS + "PETSc only need minimal includes, but requires explicit linking to all dependencies. This is expected when PETSc is built with static libraries." + ) + set(petsc_includes_needed ${petsc_includes_minimal}) + else(petsc_works_alllibraries) # It looks like we really need everything, should have listened to Matt - set (petsc_includes_needed ${petsc_includes_all}) - petsc_test_compiles ("${petsc_includes_all};${petsc_mpi_include_dirs}" - "${PETSC_LIBRARIES_TS};${petsc_additional_libraries}" - petsc_works_all) - if (petsc_works_all) # We fail anyways - message (STATUS "PETSc requires extra include paths and explicit linking to all dependencies. This probably means you have static libraries and something unexpected in PETSc headers.") - else (petsc_works_all) # We fail anyways - message (STATUS "PETSc could not be used, maybe the install is broken.") - endif (petsc_works_all) - endif (petsc_works_alllibraries) - endif (petsc_works_allincludes) - endif (petsc_works_minimal) + set(petsc_includes_needed ${petsc_includes_all}) + petsc_test_compiles( + "${petsc_includes_all};${petsc_mpi_include_dirs}" + "${PETSC_LIBRARIES_TS};${petsc_additional_libraries}" petsc_works_all + ) + if(petsc_works_all) # We fail anyways + message( + STATUS + "PETSc requires extra include paths and explicit linking to all dependencies. This probably means you have static libraries and something unexpected in PETSc headers." + ) + else(petsc_works_all) # We fail anyways + message( + STATUS "PETSc could not be used, maybe the install is broken." + ) + endif(petsc_works_all) + endif(petsc_works_alllibraries) + endif(petsc_works_allincludes) + endif(petsc_works_minimal) # We do an out-of-source build so __FILE__ will be an absolute path, hence __INSDIR__ is superfluous - if (${PETSC_VERSION} VERSION_LESS 3.1) - set (PETSC_DEFINITIONS "-D__SDIR__=\"\"" CACHE STRING "PETSc definitions" FORCE) - else () - set (PETSC_DEFINITIONS "-D__INSDIR__=" CACHE STRING "PETSc definitions" FORCE) - endif () + if(${PETSC_VERSION} VERSION_LESS 3.1) + set(PETSC_DEFINITIONS + "-D__SDIR__=\"\"" + CACHE STRING "PETSc definitions" FORCE + ) + else() + set(PETSC_DEFINITIONS + "-D__INSDIR__=" + CACHE STRING "PETSc definitions" FORCE + ) + endif() # Sometimes this can be used to assist FindMPI.cmake - set (PETSC_MPIEXEC ${petsc_mpiexec} CACHE FILEPATH "Executable for running PETSc MPI programs" FORCE) - set (PETSC_INCLUDES ${petsc_includes_needed} CACHE STRING "PETSc include path" FORCE) - set (PETSC_LIBRARIES ${PETSC_LIBRARIES_ALL} CACHE STRING "PETSc libraries" FORCE) - set (PETSC_COMPILER ${petsc_cc} CACHE FILEPATH "PETSc compiler" FORCE) -endif () + set(PETSC_MPIEXEC + ${petsc_mpiexec} + CACHE FILEPATH "Executable for running PETSc MPI programs" FORCE + ) + set(PETSC_INCLUDES + ${petsc_includes_needed} + CACHE STRING "PETSc include path" FORCE + ) + set(PETSC_LIBRARIES + ${PETSC_LIBRARIES_ALL} + CACHE STRING "PETSc libraries" FORCE + ) + set(PETSC_COMPILER + ${petsc_cc} + CACHE FILEPATH "PETSc compiler" FORCE + ) +endif() -if (NOT PETSC_INCLUDES AND NOT TARGET PETSc::PETSc) +if(NOT PETSC_INCLUDES AND NOT TARGET PETSc::PETSc) find_package(PkgConfig) - if (PkgConfig_FOUND) + if(PkgConfig_FOUND) pkg_search_module(PkgPETSC PETSc>3.4.0 petsc>3.4.0) - set (PETSC_LIBRARIES ${PkgPETSC_LINK_LIBRARIES} CACHE STRING "PETSc libraries" FORCE) - set (PETSC_INCLUDES ${PkgPETSC_INCLUDE_DIRS} CACHE STRING "PETSc include path" FORCE) - set (PETSC_EXECUTABLE_COMPILES "YES" CACHE BOOL - "Can the system successfully run a PETSc executable? This variable can be manually set to \"YES\" to force CMake to accept a given PETSc configuration, but this will almost always result in a broken build. If you change PETSC_DIR, PETSC_ARCH, or PETSC_CURRENT you would have to reset this variable." FORCE) + set(PETSC_LIBRARIES + ${PkgPETSC_LINK_LIBRARIES} + CACHE STRING "PETSc libraries" FORCE + ) + set(PETSC_INCLUDES + ${PkgPETSC_INCLUDE_DIRS} + CACHE STRING "PETSc include path" FORCE + ) + set(PETSC_EXECUTABLE_COMPILES + "YES" + CACHE + BOOL + "Can the system successfully run a PETSc executable? This variable can be manually set to \"YES\" to force CMake to accept a given PETSc configuration, but this will almost always result in a broken build. If you change PETSC_DIR, PETSC_ARCH, or PETSC_CURRENT you would have to reset this variable." + FORCE + ) endif() endif() # Note that we have forced values for all these choices. If you # change these, you are telling the system to trust you that they # work. It is likely that you will end up with a broken build. -mark_as_advanced (PETSC_INCLUDES PETSC_LIBRARIES PETSC_COMPILER PETSC_DEFINITIONS PETSC_MPIEXEC PETSC_EXECUTABLE_COMPILES) - -include (FindPackageHandleStandardArgs) -find_package_handle_standard_args (PETSc +mark_as_advanced( + PETSC_INCLUDES PETSC_LIBRARIES PETSC_COMPILER PETSC_DEFINITIONS PETSC_MPIEXEC + PETSC_EXECUTABLE_COMPILES +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + PETSc REQUIRED_VARS PETSC_INCLUDES PETSC_LIBRARIES VERSION_VAR PETSC_VERSION - FAIL_MESSAGE "PETSc could not be found. Be sure to set PETSC_DIR and PETSC_ARCH.") + FAIL_MESSAGE + "PETSc could not be found. Be sure to set PETSC_DIR and PETSC_ARCH." +) -if (PETSC_FOUND) - if (NOT TARGET PETSc::PETSc) +if(PETSC_FOUND) + if(NOT TARGET PETSc::PETSc) add_library(PETSc::PETSc UNKNOWN IMPORTED) list(GET PETSC_LIBRARIES 0 PETSC_LIBRARY) target_link_libraries(PETSc::PETSc INTERFACE "${PETSC_LIBRARIES}") - set_target_properties(PETSc::PETSc PROPERTIES - IMPORTED_LOCATION "${PETSC_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${PETSC_INCLUDES}" - ) + set_target_properties( + PETSc::PETSc PROPERTIES IMPORTED_LOCATION "${PETSC_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${PETSC_INCLUDES}" + ) endif() endif() diff --git a/cmake/FindPackageMultipass.cmake b/cmake/FindPackageMultipass.cmake index 99bbace448..b2762f5d28 100644 --- a/cmake/FindPackageMultipass.cmake +++ b/cmake/FindPackageMultipass.cmake @@ -34,99 +34,124 @@ include(CheckCXXSourceCompiles) -macro (FIND_PACKAGE_MULTIPASS _name _current) - string (TOUPPER ${_name} _NAME) - set (_args ${ARGV}) - list (REMOVE_AT _args 0 1) +macro(FIND_PACKAGE_MULTIPASS _name _current) + string(TOUPPER ${_name} _NAME) + set(_args ${ARGV}) + list(REMOVE_AT _args 0 1) - set (_states_current "YES") - list (GET _args 0 _cmd) - if (_cmd STREQUAL "STATES") - list (REMOVE_AT _args 0) - list (GET _args 0 _state) - while (_state AND NOT _state STREQUAL "DEPENDENTS") + set(_states_current "YES") + list(GET _args 0 _cmd) + if(_cmd STREQUAL "STATES") + list(REMOVE_AT _args 0) + list(GET _args 0 _state) + while(_state AND NOT _state STREQUAL "DEPENDENTS") # The name of the stored value for the given state - set (_stored_var PACKAGE_MULTIPASS_${_NAME}_${_state}) - if (NOT "${${_stored_var}}" STREQUAL "${${_NAME}_${_state}}") - set (_states_current "NO") - endif (NOT "${${_stored_var}}" STREQUAL "${${_NAME}_${_state}}") - set (${_stored_var} "${${_NAME}_${_state}}" CACHE INTERNAL "Stored state for ${_name}." FORCE) - list (REMOVE_AT _args 0) - list (GET _args 0 _state) - endwhile (_state AND NOT _state STREQUAL "DEPENDENTS") - endif (_cmd STREQUAL "STATES") + set(_stored_var PACKAGE_MULTIPASS_${_NAME}_${_state}) + if(NOT "${${_stored_var}}" STREQUAL "${${_NAME}_${_state}}") + set(_states_current "NO") + endif(NOT "${${_stored_var}}" STREQUAL "${${_NAME}_${_state}}") + set(${_stored_var} + "${${_NAME}_${_state}}" + CACHE INTERNAL "Stored state for ${_name}." FORCE + ) + list(REMOVE_AT _args 0) + list(GET _args 0 _state) + endwhile(_state AND NOT _state STREQUAL "DEPENDENTS") + endif(_cmd STREQUAL "STATES") - set (_stored ${_NAME}_CURRENT) - if (NOT ${_stored}) - set (${_stored} "YES" CACHE BOOL "Is the configuration for ${_name} current? Set to \"NO\" to reconfigure." FORCE) - set (_states_current "NO") - endif (NOT ${_stored}) + set(_stored ${_NAME}_CURRENT) + if(NOT ${_stored}) + set(${_stored} + "YES" + CACHE + BOOL + "Is the configuration for ${_name} current? Set to \"NO\" to reconfigure." + FORCE + ) + set(_states_current "NO") + endif(NOT ${_stored}) - set (${_current} ${_states_current}) - if (NOT ${_current} AND PACKAGE_MULTIPASS_${_name}_CALLED) - message (STATUS "Clearing ${_name} dependent variables") + set(${_current} ${_states_current}) + if(NOT ${_current} AND PACKAGE_MULTIPASS_${_name}_CALLED) + message(STATUS "Clearing ${_name} dependent variables") # Clear all the dependent variables so that the module can reset them - list (GET _args 0 _cmd) - if (_cmd STREQUAL "DEPENDENTS") - list (REMOVE_AT _args 0) - foreach (dep ${_args}) - set (${_NAME}_${dep} "NOTFOUND" CACHE INTERNAL "Cleared" FORCE) - endforeach (dep) - endif (_cmd STREQUAL "DEPENDENTS") - set (${_NAME}_FOUND "NOTFOUND" CACHE INTERNAL "Cleared" FORCE) - endif () - set (PACKAGE_MULTIPASS_${name}_CALLED YES CACHE INTERNAL "Private" FORCE) -endmacro (FIND_PACKAGE_MULTIPASS) - + list(GET _args 0 _cmd) + if(_cmd STREQUAL "DEPENDENTS") + list(REMOVE_AT _args 0) + foreach(dep ${_args}) + set(${_NAME}_${dep} + "NOTFOUND" + CACHE INTERNAL "Cleared" FORCE + ) + endforeach(dep) + endif(_cmd STREQUAL "DEPENDENTS") + set(${_NAME}_FOUND + "NOTFOUND" + CACHE INTERNAL "Cleared" FORCE + ) + endif() + set(PACKAGE_MULTIPASS_${name}_CALLED + YES + CACHE INTERNAL "Private" FORCE + ) +endmacro(FIND_PACKAGE_MULTIPASS) -macro (MULTIPASS_SOURCE_RUNS includes libraries source runs language) - include (Check${language}SourceRuns) +macro(MULTIPASS_SOURCE_RUNS includes libraries source runs language) + include(Check${language}SourceRuns) # This is a ridiculous hack. CHECK_${language}_SOURCE_* thinks that if the # *name* of the return variable doesn't change, then the test does # not need to be re-run. We keep an internal count which we # increment to guarantee that every test name is unique. If we've # gotten here, then the configuration has changed enough that the # test *needs* to be rerun. - if (NOT MULTIPASS_TEST_COUNT) - set (MULTIPASS_TEST_COUNT 00) - endif (NOT MULTIPASS_TEST_COUNT) - math (EXPR _tmp "${MULTIPASS_TEST_COUNT} + 1") # Why can't I add to a cache variable? - set (MULTIPASS_TEST_COUNT ${_tmp} CACHE INTERNAL "Unique test ID") - set (testname MULTIPASS_TEST_${MULTIPASS_TEST_COUNT}_${runs}) - set (CMAKE_REQUIRED_INCLUDES ${includes}) - set (CMAKE_REQUIRED_LIBRARIES ${libraries}) + if(NOT MULTIPASS_TEST_COUNT) + set(MULTIPASS_TEST_COUNT 00) + endif(NOT MULTIPASS_TEST_COUNT) + math(EXPR _tmp "${MULTIPASS_TEST_COUNT} + 1" + )# Why can't I add to a cache variable? + set(MULTIPASS_TEST_COUNT + ${_tmp} + CACHE INTERNAL "Unique test ID" + ) + set(testname MULTIPASS_TEST_${MULTIPASS_TEST_COUNT}_${runs}) + set(CMAKE_REQUIRED_INCLUDES ${includes}) + set(CMAKE_REQUIRED_LIBRARIES ${libraries}) if(${language} STREQUAL "C") - check_c_source_runs ("${source}" ${testname}) + check_c_source_runs("${source}" ${testname}) elseif(${language} STREQUAL "CXX") - check_cxx_source_runs ("${source}" ${testname}) + check_cxx_source_runs("${source}" ${testname}) endif() - set (${runs} "${${testname}}") -endmacro (MULTIPASS_SOURCE_RUNS) + set(${runs} "${${testname}}") +endmacro(MULTIPASS_SOURCE_RUNS) -macro (MULTIPASS_C_SOURCE_RUNS includes libraries source runs) +macro(MULTIPASS_C_SOURCE_RUNS includes libraries source runs) multipass_source_runs("${includes}" "${libraries}" "${source}" ${runs} "C") -endmacro (MULTIPASS_C_SOURCE_RUNS) +endmacro(MULTIPASS_C_SOURCE_RUNS) -macro (MULTIPASS_SOURCE_COMPILES includes libraries source runs language) - include (Check${language}SourceCompiles) +macro(MULTIPASS_SOURCE_COMPILES includes libraries source runs language) + include(Check${language}SourceCompiles) # This is a ridiculous hack. CHECK_${language}_SOURCE_* thinks that if the # *name* of the return variable doesn't change, then the test does # not need to be re-run. We keep an internal count which we # increment to guarantee that every test name is unique. If we've # gotten here, then the configuration has changed enough that the # test *needs* to be rerun. - if (NOT MULTIPASS_TEST_COUNT) - set (MULTIPASS_TEST_COUNT 00) - endif (NOT MULTIPASS_TEST_COUNT) - math (EXPR _tmp "${MULTIPASS_TEST_COUNT} + 1") # Why can't I add to a cache variable? - set (MULTIPASS_TEST_COUNT ${_tmp} CACHE INTERNAL "Unique test ID") - set (testname MULTIPASS_TEST_${MULTIPASS_TEST_COUNT}_${runs}) - set (CMAKE_REQUIRED_INCLUDES ${includes}) - set (CMAKE_REQUIRED_LIBRARIES ${libraries}) + if(NOT MULTIPASS_TEST_COUNT) + set(MULTIPASS_TEST_COUNT 00) + endif(NOT MULTIPASS_TEST_COUNT) + math(EXPR _tmp "${MULTIPASS_TEST_COUNT} + 1" + )# Why can't I add to a cache variable? + set(MULTIPASS_TEST_COUNT + ${_tmp} + CACHE INTERNAL "Unique test ID" + ) + set(testname MULTIPASS_TEST_${MULTIPASS_TEST_COUNT}_${runs}) + set(CMAKE_REQUIRED_INCLUDES ${includes}) + set(CMAKE_REQUIRED_LIBRARIES ${libraries}) if(${language} STREQUAL "C") - check_c_source_compiles ("${source}" ${testname}) + check_c_source_compiles("${source}" ${testname}) elseif(${language} STREQUAL "CXX") - check_cxx_source_compiles ("${source}" ${testname}) + check_cxx_source_compiles("${source}" ${testname}) endif() - set (${runs} "${${testname}}") -endmacro () + set(${runs} "${${testname}}") +endmacro() diff --git a/cmake/FindSLEPc.cmake b/cmake/FindSLEPc.cmake index 3add8eba8b..4df855569b 100644 --- a/cmake/FindSLEPc.cmake +++ b/cmake/FindSLEPc.cmake @@ -48,74 +48,92 @@ find_package(PETSc REQUIRED) find_package(MPI REQUIRED) # Set debian_arches (PETSC_ARCH for Debian-style installations) -foreach (debian_arches linux kfreebsd) - if ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") - set(DEBIAN_FLAVORS ${debian_arches}-gnu-c-debug ${debian_arches}-gnu-c-opt ${DEBIAN_FLAVORS}) +foreach(debian_arches linux kfreebsd) + if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") + set(DEBIAN_FLAVORS ${debian_arches}-gnu-c-debug ${debian_arches}-gnu-c-opt + ${DEBIAN_FLAVORS} + ) else() - set(DEBIAN_FLAVORS ${debian_arches}-gnu-c-opt ${debian_arches}-gnu-c-debug ${DEBIAN_FLAVORS}) + set(DEBIAN_FLAVORS ${debian_arches}-gnu-c-opt ${debian_arches}-gnu-c-debug + ${DEBIAN_FLAVORS} + ) endif() endforeach() # List of possible locations for SLEPC_DIR set(slepc_dir_locations "") list(APPEND slepc_dir_locations "/usr/lib/slepc") -list(APPEND slepc_dir_locations "/opt/local/lib/petsc") # Macports +list(APPEND slepc_dir_locations "/opt/local/lib/petsc") # Macports list(APPEND slepc_dir_locations "/usr/local/lib/slepc") list(APPEND slepc_dir_locations "$ENV{HOME}/slepc") # Try to figure out SLEPC_DIR by finding slepc.h -find_path(SLEPC_DIR include/slepc.h +find_path( + SLEPC_DIR include/slepc.h HINTS ${SLEPC_DIR} $ENV{SLEPC_DIR} PATHS ${slepc_dir_locations} - DOC "SLEPc directory") + DOC "SLEPc directory" +) # Report result of search for SLEPC_DIR -if (DEFINED SLEPC_DIR) +if(DEFINED SLEPC_DIR) message(STATUS "SLEPC_DIR is ${SLEPC_DIR}") else() message(STATUS "SLEPC_DIR is empty") endif() # Get variables from SLEPc configuration -if (SLEPC_DIR) +if(SLEPC_DIR) - find_library(SLEPC_LIBRARY + find_library( + SLEPC_LIBRARY NAMES slepc - HINTS - ${SLEPC_DIR}/lib - $ENV{SLEPC_DIR}/lib - ${SLEPC_DIR}/${PETSC_ARCH}/lib - $ENV{SLEPC_DIR}/$ENV{PETSC_ARCH}/lib + HINTS ${SLEPC_DIR}/lib $ENV{SLEPC_DIR}/lib ${SLEPC_DIR}/${PETSC_ARCH}/lib + $ENV{SLEPC_DIR}/$ENV{PETSC_ARCH}/lib NO_DEFAULT_PATH - DOC "The SLEPc library") - find_library(SLEPC_LIBRARY + DOC "The SLEPc library" + ) + find_library( + SLEPC_LIBRARY NAMES slepc - DOC "The SLEPc library") + DOC "The SLEPc library" + ) mark_as_advanced(SLEPC_LIBRARY) # Find SLEPc config file - find_file(SLEPC_CONFIG_FILE NAMES slepc_common PATHS - ${SLEPC_DIR}/lib/slepc/conf - ${SLEPC_DIR}/lib/slepc-conf ${SLEPC_DIR}/conf) + find_file( + SLEPC_CONFIG_FILE + NAMES slepc_common + PATHS ${SLEPC_DIR}/lib/slepc/conf ${SLEPC_DIR}/lib/slepc-conf + ${SLEPC_DIR}/conf + ) # Create a temporary Makefile to probe the SLEPc configuration set(slepc_config_makefile ${PROJECT_BINARY_DIR}/Makefile.slepc) - file(WRITE ${slepc_config_makefile} -"# This file was autogenerated by FindSLEPc.cmake + file( + WRITE ${slepc_config_makefile} + "# This file was autogenerated by FindSLEPc.cmake SLEPC_DIR = ${SLEPC_DIR} PETSC_ARCH = ${PETSC_ARCH} PETSC_DIR = ${PETSC_DIR} include ${SLEPC_CONFIG_FILE} show : -@echo -n \${\${VARIABLE}} -") +" + ) # Define macro for getting SLEPc variables from Makefile macro(SLEPC_GET_VARIABLE var name) - set(${var} "NOTFOUND" CACHE INTERNAL "Cleared" FORCE) - execute_process(COMMAND ${MAKE_EXECUTABLE} --no-print-directory -f ${slepc_config_makefile} show VARIABLE=${name} + set(${var} + "NOTFOUND" + CACHE INTERNAL "Cleared" FORCE + ) + execute_process( + COMMAND ${MAKE_EXECUTABLE} --no-print-directory -f + ${slepc_config_makefile} show VARIABLE=${name} OUTPUT_VARIABLE ${var} - RESULT_VARIABLE slepc_return) + RESULT_VARIABLE slepc_return + ) endmacro() # Call macro to get the SLEPc variables @@ -131,15 +149,21 @@ show : resolve_libraries(SLEPC_EXTERNAL_LIBRARIES "${SLEPC_EXTERNAL_LIB}") # Add variables to CMake cache and mark as advanced - set(SLEPC_INCLUDE_DIRS ${SLEPC_INCLUDE_DIRS} CACHE STRING "SLEPc include paths." FORCE) - set(SLEPC_LIBRARIES ${SLEPC_LIBRARY} CACHE STRING "SLEPc libraries." FORCE) + set(SLEPC_INCLUDE_DIRS + ${SLEPC_INCLUDE_DIRS} + CACHE STRING "SLEPc include paths." FORCE + ) + set(SLEPC_LIBRARIES + ${SLEPC_LIBRARY} + CACHE STRING "SLEPc libraries." FORCE + ) mark_as_advanced(SLEPC_INCLUDE_DIRS SLEPC_LIBRARIES) endif() -if (SLEPC_SKIP_BUILD_TESTS) +if(SLEPC_SKIP_BUILD_TESTS) set(SLEPC_VERSION "UNKNOWN") set(SLEPC_VERSION_OK TRUE) -elseif (SLEPC_LIBRARIES AND SLEPC_INCLUDE_DIRS) +elseif(SLEPC_LIBRARIES AND SLEPC_INCLUDE_DIRS) # Set flags for building test program set(CMAKE_REQUIRED_INCLUDES ${SLEPC_INCLUDE_DIRS}) @@ -147,8 +171,11 @@ elseif (SLEPC_LIBRARIES AND SLEPC_INCLUDE_DIRS) # Check SLEPc version set(SLEPC_CONFIG_TEST_VERSION_CPP - "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/slepc_config_test_version.cpp") - file(WRITE ${SLEPC_CONFIG_TEST_VERSION_CPP} " + "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/slepc_config_test_version.cpp" + ) + file( + WRITE ${SLEPC_CONFIG_TEST_VERSION_CPP} + " #include #include \"slepcversion.h\" @@ -158,56 +185,69 @@ int main() { << SLEPC_VERSION_SUBMINOR; return 0; } -") +" + ) try_run( - SLEPC_CONFIG_TEST_VERSION_EXITCODE - SLEPC_CONFIG_TEST_VERSION_COMPILED - ${CMAKE_CURRENT_BINARY_DIR} - ${SLEPC_CONFIG_TEST_VERSION_CPP} - CMAKE_FLAGS - "-DINCLUDE_DIRECTORIES:STRING=${CMAKE_REQUIRED_INCLUDES}" + SLEPC_CONFIG_TEST_VERSION_EXITCODE SLEPC_CONFIG_TEST_VERSION_COMPILED + ${CMAKE_CURRENT_BINARY_DIR} ${SLEPC_CONFIG_TEST_VERSION_CPP} + CMAKE_FLAGS "-DINCLUDE_DIRECTORIES:STRING=${CMAKE_REQUIRED_INCLUDES}" COMPILE_OUTPUT_VARIABLE COMPILE_OUTPUT RUN_OUTPUT_VARIABLE OUTPUT - ) + ) - if (SLEPC_CONFIG_TEST_VERSION_EXITCODE EQUAL 0) - set(SLEPC_VERSION "${OUTPUT}" CACHE STRING "SLEPC version number") + if(SLEPC_CONFIG_TEST_VERSION_EXITCODE EQUAL 0) + set(SLEPC_VERSION + "${OUTPUT}" + CACHE STRING "SLEPC version number" + ) string(REPLACE "." ";" SLEPC_VERSION_LIST ${SLEPC_VERSION}) list(GET SLEPC_VERSION_LIST 0 SLEPC_VERSION_MAJOR) list(GET SLEPC_VERSION_LIST 1 SLEPC_VERSION_MINOR) list(GET SLEPC_VERSION_LIST 2 SLEPC_VERSION_SUBMINOR) mark_as_advanced(SLEPC_VERSION) - mark_as_advanced(SLEPC_VERSION_MAJOR SLEPC_VERSION_MINOR SLEPC_VERSION_SUBMINOR) + mark_as_advanced( + SLEPC_VERSION_MAJOR SLEPC_VERSION_MINOR SLEPC_VERSION_SUBMINOR + ) endif() - if (SLEPc_FIND_VERSION) + if(SLEPc_FIND_VERSION) # Check if version found is >= required version - if (NOT "${SLEPC_VERSION}" VERSION_LESS "${SLEPc_FIND_VERSION}") - set(SLEPC_VERSION_OK TRUE CACHE BOOL "") + if(NOT "${SLEPC_VERSION}" VERSION_LESS "${SLEPc_FIND_VERSION}") + set(SLEPC_VERSION_OK + TRUE + CACHE BOOL "" + ) endif() else() # No specific version requested - set(SLEPC_VERSION_OK TRUE CACHE BOOL "") + set(SLEPC_VERSION_OK + TRUE + CACHE BOOL "" + ) endif() mark_as_advanced(SLEPC_VERSION_OK) endif() # Standard package handling include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(SLEPc +find_package_handle_standard_args( + SLEPc FOUND_VAR SLEPC_FOUND - FAIL_MESSAGE "SLEPc could not be found. Be sure to set SLEPC_DIR, PETSC_DIR, and PETSC_ARCH." + FAIL_MESSAGE + "SLEPc could not be found. Be sure to set SLEPC_DIR, PETSC_DIR, and PETSC_ARCH." VERSION_VAR SLEPC_VERSION - REQUIRED_VARS SLEPC_LIBRARIES SLEPC_DIR SLEPC_INCLUDE_DIRS SLEPC_VERSION_OK) + REQUIRED_VARS SLEPC_LIBRARIES SLEPC_DIR SLEPC_INCLUDE_DIRS SLEPC_VERSION_OK +) -if (SLEPC_FOUND) - if (NOT TARGET SLEPc::SLEPc) +if(SLEPC_FOUND) + if(NOT TARGET SLEPc::SLEPc) add_library(SLEPc::SLEPc UNKNOWN IMPORTED) - set_target_properties(SLEPc::SLEPc PROPERTIES - IMPORTED_LOCATION "${SLEPC_LIBRARIES}" - INTERFACE_INCLUDE_DIRECTORIES "${SLEPC_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES PETSc::PETSc - ) + set_target_properties( + SLEPc::SLEPc + PROPERTIES IMPORTED_LOCATION "${SLEPC_LIBRARIES}" + INTERFACE_INCLUDE_DIRECTORIES "${SLEPC_INCLUDE_DIRS}" + INTERFACE_LINK_LIBRARIES PETSc::PETSc + ) endif() endif() diff --git a/cmake/FindSUNDIALS.cmake b/cmake/FindSUNDIALS.cmake index 15b266d06a..e5cec34510 100644 --- a/cmake/FindSUNDIALS.cmake +++ b/cmake/FindSUNDIALS.cmake @@ -33,47 +33,48 @@ include(FindPackageHandleStandardArgs) find_package(SUNDIALS CONFIG QUIET) -if (SUNDIALS_FOUND) - if (TARGET SUNDIALS::nvecparallel) +if(SUNDIALS_FOUND) + if(TARGET SUNDIALS::nvecparallel) return() else() message(STATUS "SUNDIALS found but not SUNDIALS::nvecparallel") endif() endif() -find_path(SUNDIALS_INCLUDE_DIR - sundials_config.h - HINTS - "${SUNDIALS_ROOT}" - ENV SUNDIALS_DIR +find_path( + SUNDIALS_INCLUDE_DIR sundials_config.h + HINTS "${SUNDIALS_ROOT}" ENV SUNDIALS_DIR PATH_SUFFIXES include include/sundials - DOC "SUNDIALS Directory") + DOC "SUNDIALS Directory" +) -if (SUNDIALS_DEBUG) +if(SUNDIALS_DEBUG) message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " SUNDIALS_INCLUDE_DIR = ${SUNDIALS_INCLUDE_DIR}" - " SUNDIALS_ROOT = ${SUNDIALS_ROOT}") + " SUNDIALS_INCLUDE_DIR = ${SUNDIALS_INCLUDE_DIR}" + " SUNDIALS_ROOT = ${SUNDIALS_ROOT}" + ) endif() set(SUNDIALS_INCLUDE_DIRS - "${SUNDIALS_INCLUDE_DIR}" - "${SUNDIALS_INCLUDE_DIR}/.." - CACHE STRING "SUNDIALS include directories") + "${SUNDIALS_INCLUDE_DIR}" "${SUNDIALS_INCLUDE_DIR}/.." + CACHE STRING "SUNDIALS include directories" +) -find_library(SUNDIALS_nvecparallel_LIBRARY +find_library( + SUNDIALS_nvecparallel_LIBRARY NAMES sundials_nvecparallel - HINTS - "${SUNDIALS_INCLUDE_DIR}/.." - "${SUNDIALS_INCLUDE_DIR}/../.." + HINTS "${SUNDIALS_INCLUDE_DIR}/.." "${SUNDIALS_INCLUDE_DIR}/../.." PATH_SUFFIXES lib lib64 - ) +) -if (SUNDIALS_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " SUNDIALS_nvecparallel_LIBRARY = ${SUNDIALS_nvecparallel_LIBRARY}") +if(SUNDIALS_DEBUG) + message( + STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " SUNDIALS_nvecparallel_LIBRARY = ${SUNDIALS_nvecparallel_LIBRARY}" + ) endif() -if (NOT SUNDIALS_nvecparallel_LIBRARY) +if(NOT SUNDIALS_nvecparallel_LIBRARY) message(FATAL_ERROR "Sundials requested but SUNDIALS nvecparallel not found.") endif() list(APPEND SUNDIALS_LIBRARIES "${SUNDIALS_nvecparallel_LIBRARY}") @@ -81,62 +82,86 @@ mark_as_advanced(SUNDIALS_nvecparallel_LIBRARY) set(SUNDIALS_COMPONENTS arkode cvode ida) -foreach (LIB ${SUNDIALS_COMPONENTS}) - find_library(SUNDIALS_${LIB}_LIBRARY +foreach(LIB ${SUNDIALS_COMPONENTS}) + find_library( + SUNDIALS_${LIB}_LIBRARY NAMES sundials_${LIB} - HINTS - "${SUNDIALS_INCLUDE_DIR}/.." - "${SUNDIALS_INCLUDE_DIR}/../.." + HINTS "${SUNDIALS_INCLUDE_DIR}/.." "${SUNDIALS_INCLUDE_DIR}/../.." PATH_SUFFIXES lib lib64 - ) + ) - if (SUNDIALS_DEBUG) + if(SUNDIALS_DEBUG) message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " SUNDIALS_${LIB}_LIBRARY = ${SUNDIALS_${LIB}_LIBRARY}") + " SUNDIALS_${LIB}_LIBRARY = ${SUNDIALS_${LIB}_LIBRARY}" + ) endif() - if (NOT SUNDIALS_${LIB}_LIBRARY) + if(NOT SUNDIALS_${LIB}_LIBRARY) message(FATAL_ERROR "Sundials requested but SUNDIALS ${LIB} not found.") endif() list(APPEND SUNDIALS_LIBRARIES "${SUNDIALS_${LIB}_LIBRARY}") mark_as_advanced(SUNDIALS_${LIB}_LIBRARY) endforeach() -if (SUNDIALS_INCLUDE_DIR) +if(SUNDIALS_INCLUDE_DIR) file(READ "${SUNDIALS_INCLUDE_DIR}/sundials_config.h" SUNDIALS_CONFIG_FILE) set(SUNDIALS_VERSION_REGEX_PATTERN - ".*#define SUNDIALS_VERSION \"([0-9]+)\\.([0-9]+)\\.([0-9]+)\".*") - string(REGEX MATCH ${SUNDIALS_VERSION_REGEX_PATTERN} _ "${SUNDIALS_CONFIG_FILE}") - set(SUNDIALS_VERSION_MAJOR ${CMAKE_MATCH_1} CACHE STRING "") - set(SUNDIALS_VERSION_MINOR ${CMAKE_MATCH_2} CACHE STRING "") - set(SUNDIALS_VERSION_PATCH ${CMAKE_MATCH_3} CACHE STRING "") - set(SUNDIALS_VERSION "${SUNDIALS_VERSION_MAJOR}.${SUNDIALS_VERSION_MINOR}.${SUNDIALS_VERSION_PATCH}" CACHE STRING "SUNDIALS version") + ".*#define SUNDIALS_VERSION \"([0-9]+)\\.([0-9]+)\\.([0-9]+)\".*" + ) + string(REGEX MATCH ${SUNDIALS_VERSION_REGEX_PATTERN} _ + "${SUNDIALS_CONFIG_FILE}" + ) + set(SUNDIALS_VERSION_MAJOR + ${CMAKE_MATCH_1} + CACHE STRING "" + ) + set(SUNDIALS_VERSION_MINOR + ${CMAKE_MATCH_2} + CACHE STRING "" + ) + set(SUNDIALS_VERSION_PATCH + ${CMAKE_MATCH_3} + CACHE STRING "" + ) + set(SUNDIALS_VERSION + "${SUNDIALS_VERSION_MAJOR}.${SUNDIALS_VERSION_MINOR}.${SUNDIALS_VERSION_PATCH}" + CACHE STRING "SUNDIALS version" + ) endif() -if (SUNDIALS_DEBUG) +if(SUNDIALS_DEBUG) message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " SUNDIALS_VERSION = ${SUNDIALS_VERSION}") + " SUNDIALS_VERSION = ${SUNDIALS_VERSION}" + ) endif() -find_package_handle_standard_args(SUNDIALS +find_package_handle_standard_args( + SUNDIALS REQUIRED_VARS SUNDIALS_LIBRARIES SUNDIALS_INCLUDE_DIR SUNDIALS_INCLUDE_DIRS VERSION_VAR SUNDIALS_VERSION - ) +) -set(SUNDIALS_LIBRARIES "${SUNDIALS_LIBRARIES}" CACHE STRING "SUNDIALS libraries") +set(SUNDIALS_LIBRARIES + "${SUNDIALS_LIBRARIES}" + CACHE STRING "SUNDIALS libraries" +) mark_as_advanced(SUNDIALS_LIBRARIES SUNDIALS_INCLUDE_DIR SUNDIALS_INCLUDE_DIRS) -if (SUNDIALS_FOUND AND NOT TARGET SUNDIALS::SUNDIALS) +if(SUNDIALS_FOUND AND NOT TARGET SUNDIALS::SUNDIALS) add_library(SUNDIALS::nvecparallel UNKNOWN IMPORTED) - set_target_properties(SUNDIALS::nvecparallel PROPERTIES - IMPORTED_LOCATION "${SUNDIALS_nvecparallel_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${SUNDIALS_INCLUDE_DIRS}") + set_target_properties( + SUNDIALS::nvecparallel + PROPERTIES IMPORTED_LOCATION "${SUNDIALS_nvecparallel_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${SUNDIALS_INCLUDE_DIRS}" + ) - foreach (LIB ${SUNDIALS_COMPONENTS}) + foreach(LIB ${SUNDIALS_COMPONENTS}) add_library(SUNDIALS::${LIB} UNKNOWN IMPORTED) - set_target_properties(SUNDIALS::${LIB} PROPERTIES - IMPORTED_LOCATION "${SUNDIALS_${LIB}_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${SUNDIALS_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES SUNDIALS::nvecparallel) + set_target_properties( + SUNDIALS::${LIB} + PROPERTIES IMPORTED_LOCATION "${SUNDIALS_${LIB}_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${SUNDIALS_INCLUDE_DIRS}" + INTERFACE_LINK_LIBRARIES SUNDIALS::nvecparallel + ) endforeach() endif() diff --git a/cmake/FindScoreP.cmake b/cmake/FindScoreP.cmake index dd119545e8..80fe102749 100644 --- a/cmake/FindScoreP.cmake +++ b/cmake/FindScoreP.cmake @@ -61,76 +61,90 @@ # IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - find_program(ScoreP_CONFIG scorep-config) mark_as_advanced(ScoreP_CONFIG) get_filename_component(ScoreP_TMP "${ScoreP_CONFIG}" DIRECTORY) get_filename_component(ScoreP_EXEC_LOCATION "${ScoreP_TMP}" DIRECTORY) -if (ScoreP_DEBUG) +if(ScoreP_DEBUG) message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " ScoreP_CONFIG = ${ScoreP_CONFIG}" - " ScoreP_EXEC_LOCATION = ${ScoreP_EXEC_LOCATION}") + " ScoreP_CONFIG = ${ScoreP_CONFIG}" + " ScoreP_EXEC_LOCATION = ${ScoreP_EXEC_LOCATION}" + ) endif() if(ScoreP_CONFIG) message(STATUS "SCOREP library found. (using ${ScoreP_CONFIG})") - execute_process(COMMAND ${ScoreP_CONFIG} "--user" "--nocompiler" "--cppflags" - OUTPUT_VARIABLE ScoreP_CONFIG_FLAGS) + execute_process( + COMMAND ${ScoreP_CONFIG} "--user" "--nocompiler" "--cppflags" + OUTPUT_VARIABLE ScoreP_CONFIG_FLAGS + ) - string(REGEX MATCHALL "-I[^ ]*" ScoreP_CONFIG_INCLUDES "${ScoreP_CONFIG_FLAGS}") + string(REGEX MATCHALL "-I[^ ]*" ScoreP_CONFIG_INCLUDES + "${ScoreP_CONFIG_FLAGS}" + ) foreach(inc ${ScoreP_CONFIG_INCLUDES}) string(SUBSTRING ${inc} 2 -1 inc) list(APPEND ScoreP_INCLUDE_DIRS ${inc}) endforeach() - if (ScoreP_DEBUG) + if(ScoreP_DEBUG) message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " ScoreP_INCLUDE_DIRS = ${ScoreP_INCLUDE_DIRS}") + " ScoreP_INCLUDE_DIRS = ${ScoreP_INCLUDE_DIRS}" + ) endif() - string(REGEX MATCHALL "(^| +)-[^I][^ ]*" ScoreP_CONFIG_CXXFLAGS "${ScoreP_CONFIG_FLAGS}") + string(REGEX MATCHALL "(^| +)-[^I][^ ]*" ScoreP_CONFIG_CXXFLAGS + "${ScoreP_CONFIG_FLAGS}" + ) foreach(flag ${ScoreP_CONFIG_CXXFLAGS}) string(STRIP ${flag} flag) list(APPEND ScoreP_CXX_FLAGS ${flag}) endforeach() - if (ScoreP_DEBUG) + if(ScoreP_DEBUG) message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " ScoreP_CXX_FLAGS = ${ScoreP_CXX_FLAGS}") + " ScoreP_CXX_FLAGS = ${ScoreP_CXX_FLAGS}" + ) endif() unset(ScoreP_CONFIG_FLAGS) unset(ScoreP_CONFIG_INCLUDES) unset(ScoreP_CONFIG_CXXFLAGS) - execute_process(COMMAND ${ScoreP_CONFIG} "--user" "--nocompiler" "--ldflags" - OUTPUT_VARIABLE _LINK_LD_ARGS) - string( REPLACE " " ";" _LINK_LD_ARGS ${_LINK_LD_ARGS} ) - foreach( _ARG ${_LINK_LD_ARGS} ) + execute_process( + COMMAND ${ScoreP_CONFIG} "--user" "--nocompiler" "--ldflags" + OUTPUT_VARIABLE _LINK_LD_ARGS + ) + string(REPLACE " " ";" _LINK_LD_ARGS ${_LINK_LD_ARGS}) + foreach(_ARG ${_LINK_LD_ARGS}) if(${_ARG} MATCHES "^-L") - STRING(REGEX REPLACE "^-L" "" _ARG ${_ARG}) - SET(ScoreP_LINK_DIRS ${ScoreP_LINK_DIRS} ${_ARG}) + string(REGEX REPLACE "^-L" "" _ARG ${_ARG}) + set(ScoreP_LINK_DIRS ${ScoreP_LINK_DIRS} ${_ARG}) endif() endforeach() - if (ScoreP_DEBUG) + if(ScoreP_DEBUG) message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " ScoreP_LINK_DIRS = ${ScoreP_LINK_DIRS}") + " ScoreP_LINK_DIRS = ${ScoreP_LINK_DIRS}" + ) endif() - execute_process(COMMAND ${ScoreP_CONFIG} "--user" "--nocompiler" "--libs" - OUTPUT_VARIABLE _LINK_LD_ARGS) - string( REPLACE " " ";" _LINK_LD_ARGS ${_LINK_LD_ARGS} ) - foreach( _ARG ${_LINK_LD_ARGS} ) + execute_process( + COMMAND ${ScoreP_CONFIG} "--user" "--nocompiler" "--libs" + OUTPUT_VARIABLE _LINK_LD_ARGS + ) + string(REPLACE " " ";" _LINK_LD_ARGS ${_LINK_LD_ARGS}) + foreach(_ARG ${_LINK_LD_ARGS}) if(${_ARG} MATCHES "^-l") string(REGEX REPLACE "^-l" "" _ARG ${_ARG}) - find_library(_SCOREP_LIB_FROM_ARG NAMES ${_ARG} - PATHS - ${ScoreP_LINK_DIRS} - ) + find_library( + _SCOREP_LIB_FROM_ARG + NAMES ${_ARG} + PATHS ${ScoreP_LINK_DIRS} + ) if(_SCOREP_LIB_FROM_ARG) set(ScoreP_LIBRARIES ${ScoreP_LIBRARIES} ${_SCOREP_LIB_FROM_ARG}) endif() @@ -138,26 +152,26 @@ if(ScoreP_CONFIG) endif() endforeach() - if (ScoreP_DEBUG) + if(ScoreP_DEBUG) message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " ScoreP_LIBRARIES = ${ScoreP_LIBRARIES}") + " ScoreP_LIBRARIES = ${ScoreP_LIBRARIES}" + ) endif() endif() -include (FindPackageHandleStandardArgs) -find_package_handle_standard_args(ScoreP DEFAULT_MSG - ScoreP_CONFIG - ScoreP_LIBRARIES - ScoreP_INCLUDE_DIRS - ) +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + ScoreP DEFAULT_MSG ScoreP_CONFIG ScoreP_LIBRARIES ScoreP_INCLUDE_DIRS +) -if (ScoreP_FOUND AND NOT TARGET ScoreP::ScoreP) +if(ScoreP_FOUND AND NOT TARGET ScoreP::ScoreP) add_library(ScoreP::ScoreP UNKNOWN IMPORTED) - set_target_properties(ScoreP::ScoreP PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${ScoreP_INCLUDE_DIRS}" - IMPORTED_LINK_INTERFACE_LIBRARIES "${ScoreP_LIBRARIES}" - INTERFACE_INCLUDE_DEFINITIONS "${ScoreP_CXX_FLAGS}" - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - ) + set_target_properties( + ScoreP::ScoreP + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${ScoreP_INCLUDE_DIRS}" + IMPORTED_LINK_INTERFACE_LIBRARIES "${ScoreP_LIBRARIES}" + INTERFACE_INCLUDE_DEFINITIONS "${ScoreP_CXX_FLAGS}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + ) endif() diff --git a/cmake/FindSphinx.cmake b/cmake/FindSphinx.cmake index fd377d0d00..a15ae8d6f9 100644 --- a/cmake/FindSphinx.cmake +++ b/cmake/FindSphinx.cmake @@ -14,13 +14,15 @@ # https://devblogs.microsoft.com/cppblog/clear-functional-c-documentation-with-sphinx-breathe-doxygen-cmake/ #Look for an executable called sphinx-build -find_program(SPHINX_EXECUTABLE - NAMES sphinx-build sphinx-build-3 - DOC "Path to sphinx-build executable") +find_program( + SPHINX_EXECUTABLE + NAMES sphinx-build sphinx-build-3 + DOC "Path to sphinx-build executable" +) include(FindPackageHandleStandardArgs) #Handle standard arguments to find_package like REQUIRED and QUIET -find_package_handle_standard_args(Sphinx - "Failed to find sphinx-build executable" - SPHINX_EXECUTABLE) +find_package_handle_standard_args( + Sphinx "Failed to find sphinx-build executable" SPHINX_EXECUTABLE +) diff --git a/cmake/FindnetCDF.cmake b/cmake/FindnetCDF.cmake index 361095954e..ca6eff8d75 100644 --- a/cmake/FindnetCDF.cmake +++ b/cmake/FindnetCDF.cmake @@ -28,119 +28,129 @@ include(BOUT++functions) include(CMakePrintHelpers) -if (NOT netCDF_ROOT AND EXISTS "${BOUT_USE_NETCDF}") +if(NOT netCDF_ROOT AND EXISTS "${BOUT_USE_NETCDF}") set(netCDF_ROOT "${BOUT_USE_NETCDF}") endif() enable_language(C) find_package(netCDF QUIET CONFIG) -if (netCDF_FOUND) +if(netCDF_FOUND) message(STATUS "netCDF CONFIG found") set(netCDF_FOUND TRUE) - if (NOT TARGET netCDF::netcdf) + if(NOT TARGET netCDF::netcdf) bout_add_library_alias(netCDF::netcdf netcdf) endif() - if (netCDF_DEBUG) - cmake_print_properties(TARGETS netcdf PROPERTIES LOCATION VERSION) - endif (netCDF_DEBUG) + if(netCDF_DEBUG) + cmake_print_properties(TARGETS netcdf PROPERTIES LOCATION VERSION) + endif(netCDF_DEBUG) return() endif() -find_program(NC_CONFIG "nc-config" +find_program( + NC_CONFIG "nc-config" PATHS "${netCDF_ROOT}" PATH_SUFFIXES bin DOC "Path to netCDF C config helper" NO_DEFAULT_PATH - ) +) -find_program(NC_CONFIG "nc-config" - DOC "Path to netCDF C config helper" - ) +find_program(NC_CONFIG "nc-config" DOC "Path to netCDF C config helper") get_filename_component(NC_CONFIG_TMP "${NC_CONFIG}" DIRECTORY) get_filename_component(NC_CONFIG_LOCATION "${NC_CONFIG_TMP}" DIRECTORY) -if (netCDF_DEBUG) +if(netCDF_DEBUG) message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " NC_CONFIG_LOCATION = ${NC_CONFIG_LOCATION}" - " netCDF_ROOT = ${netCDF_ROOT}") + " NC_CONFIG_LOCATION = ${NC_CONFIG_LOCATION}" + " netCDF_ROOT = ${netCDF_ROOT}" + ) endif() bout_inspect_netcdf_config(NC_HINTS_INCLUDE_DIR "${NC_CONFIG}" "--includedir") bout_inspect_netcdf_config(NC_HINTS_PREFIX "${NC_CONFIG}" "--prefix") -find_path(netCDF_C_INCLUDE_DIR +find_path( + netCDF_C_INCLUDE_DIR NAMES netcdf.h DOC "netCDF C include directories" - HINTS - "${NC_HINTS_INCLUDE_DIR}" - "${NC_HINTS_PREFIX}" - "${NC_CONFIG_LOCATION}" - PATH_SUFFIXES - "include" + HINTS "${NC_HINTS_INCLUDE_DIR}" "${NC_HINTS_PREFIX}" "${NC_CONFIG_LOCATION}" + PATH_SUFFIXES "include" +) +if(netCDF_DEBUG) + message( + STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " netCDF_C_INCLUDE_DIR = ${netCDF_C_INCLUDE_DIR}" + " NC_HINTS_INCLUDE_DIR = ${NC_HINTS_INCLUDE_DIR}" + " NC_HINTS_PREFIX = ${NC_HINTS_PREFIX}" ) -if (netCDF_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " netCDF_C_INCLUDE_DIR = ${netCDF_C_INCLUDE_DIR}" - " NC_HINTS_INCLUDE_DIR = ${NC_HINTS_INCLUDE_DIR}" - " NC_HINTS_PREFIX = ${NC_HINTS_PREFIX}" - ) endif() mark_as_advanced(netCDF_C_INCLUDE_DIR) -find_library(netCDF_C_LIBRARY +find_library( + netCDF_C_LIBRARY NAMES netcdf DOC "netCDF C library" - HINTS - "${NC_HINTS_INCLUDE_DIR}" - "${NC_HINTS_PREFIX}" - "${NC_CONFIG_LOCATION}" - PATH_SUFFIXES - "lib" "lib64" - ) -if (netCDF_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " netCDF_C_LIBRARY = ${netCDF_C_LIBRARY}" - " NC_HINTS_INCLUDE_DIR = ${NC_HINTS_INCLUDE_DIR}" - " NC_HINTS_PREFIX = ${NC_HINTS_PREFIX}" - ) + HINTS "${NC_HINTS_INCLUDE_DIR}" "${NC_HINTS_PREFIX}" "${NC_CONFIG_LOCATION}" + PATH_SUFFIXES "lib" "lib64" +) +if(netCDF_DEBUG) + message( + STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " netCDF_C_LIBRARY = ${netCDF_C_LIBRARY}" + " NC_HINTS_INCLUDE_DIR = ${NC_HINTS_INCLUDE_DIR}" + " NC_HINTS_PREFIX = ${NC_HINTS_PREFIX}" + ) endif() mark_as_advanced(netCDF_C_LIBRARY) -if (netCDF_C_INCLUDE_DIR) +if(netCDF_C_INCLUDE_DIR) file(STRINGS "${netCDF_C_INCLUDE_DIR}/netcdf_meta.h" _netcdf_version_lines - REGEX "#define[ \t]+NC_VERSION_(MAJOR|MINOR|PATCH|NOTE)") - string(REGEX REPLACE ".*NC_VERSION_MAJOR *\([0-9]*\).*" "\\1" _netcdf_version_major "${_netcdf_version_lines}") - string(REGEX REPLACE ".*NC_VERSION_MINOR *\([0-9]*\).*" "\\1" _netcdf_version_minor "${_netcdf_version_lines}") - string(REGEX REPLACE ".*NC_VERSION_PATCH *\([0-9]*\).*" "\\1" _netcdf_version_patch "${_netcdf_version_lines}") - string(REGEX REPLACE ".*NC_VERSION_NOTE *\"\([^\"]*\)\".*" "\\1" _netcdf_version_note "${_netcdf_version_lines}") - if (NOT _netcdf_version_note STREQUAL "") + REGEX "#define[ \t]+NC_VERSION_(MAJOR|MINOR|PATCH|NOTE)" + ) + string(REGEX REPLACE ".*NC_VERSION_MAJOR *\([0-9]*\).*" "\\1" + _netcdf_version_major "${_netcdf_version_lines}" + ) + string(REGEX REPLACE ".*NC_VERSION_MINOR *\([0-9]*\).*" "\\1" + _netcdf_version_minor "${_netcdf_version_lines}" + ) + string(REGEX REPLACE ".*NC_VERSION_PATCH *\([0-9]*\).*" "\\1" + _netcdf_version_patch "${_netcdf_version_lines}" + ) + string(REGEX REPLACE ".*NC_VERSION_NOTE *\"\([^\"]*\)\".*" "\\1" + _netcdf_version_note "${_netcdf_version_lines}" + ) + if(NOT _netcdf_version_note STREQUAL "") # Make development version compare higher than any patch level set(_netcdf_version_note ".99") endif() - set(netCDF_VERSION "${_netcdf_version_major}.${_netcdf_version_minor}.${_netcdf_version_patch}${_netcdf_version_note}") + set(netCDF_VERSION + "${_netcdf_version_major}.${_netcdf_version_minor}.${_netcdf_version_patch}${_netcdf_version_note}" + ) unset(_netcdf_version_major) unset(_netcdf_version_minor) unset(_netcdf_version_patch) unset(_netcdf_version_note) unset(_netcdf_version_lines) -endif () +endif() include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(netCDF +find_package_handle_standard_args( + netCDF REQUIRED_VARS netCDF_C_LIBRARY netCDF_C_INCLUDE_DIR - VERSION_VAR netCDF_VERSION) + VERSION_VAR netCDF_VERSION +) -if (netCDF_FOUND) +if(netCDF_FOUND) set(netCDF_INCLUDE_DIR "${netCDF_C_INCLUDE_DIR}") set(netCDF_INCLUDE_DIRS "${netCDF_C_INCLUDE_DIR}") set(netCDF_LIBRARIES "${netCDF_C_LIBRARY}") - if (NOT TARGET netCDF::netcdf) + if(NOT TARGET netCDF::netcdf) add_library(netCDF::netcdf UNKNOWN IMPORTED) - set_target_properties(netCDF::netcdf PROPERTIES - IMPORTED_LOCATION "${netCDF_C_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${netCDF_C_INCLUDE_DIR}" - ) - endif () -endif () + set_target_properties( + netCDF::netcdf + PROPERTIES IMPORTED_LOCATION "${netCDF_C_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${netCDF_C_INCLUDE_DIR}" + ) + endif() +endif() diff --git a/cmake/FindnetCDFCxx.cmake b/cmake/FindnetCDFCxx.cmake index d4155dc760..78bd93851e 100644 --- a/cmake/FindnetCDFCxx.cmake +++ b/cmake/FindnetCDFCxx.cmake @@ -27,16 +27,16 @@ include(BOUT++functions) -if (NOT netCDFCxx_ROOT AND EXISTS "${BOUT_USE_NETCDF}") +if(NOT netCDFCxx_ROOT AND EXISTS "${BOUT_USE_NETCDF}") set(netCDFCxx_ROOT "${BOUT_USE_NETCDF}") endif() -if (NOT EXISTS ${NCXX4_CONFIG}) +if(NOT EXISTS ${NCXX4_CONFIG}) # Only search if NCXX4_CONFIG was not set explicitly find_package(netCDFCxx QUIET CONFIG) - if (netCDFCxx_FOUND) + if(netCDFCxx_FOUND) set(netCDFCxx_FOUND TRUE) - if (NOT TARGET netCDF::netcdf-cxx4) + if(NOT TARGET netCDF::netcdf-cxx4) bout_add_library_alias(netCDF::netcdf-cxx4 netcdf-cxx4) endif() return() @@ -45,73 +45,74 @@ endif() find_package(netCDF REQUIRED) -find_program(NCXX4_CONFIG "ncxx4-config" +find_program( + NCXX4_CONFIG "ncxx4-config" PATHS "${netCDFCxx_ROOT}" PATH_SUFFIXES bin DOC "Path to netCDF C++ config helper" NO_DEFAULT_PATH - ) +) -find_program(NCXX4_CONFIG "ncxx4-config" - DOC "Path to netCDF C++ config helper" - ) +find_program(NCXX4_CONFIG "ncxx4-config" DOC "Path to netCDF C++ config helper") get_filename_component(NCXX4_CONFIG_TMP "${NCXX4_CONFIG}" DIRECTORY) get_filename_component(NCXX4_CONFIG_LOCATION "${NCXX4_CONFIG_TMP}" DIRECTORY) -if (netCDFCxx_DEBUG) +if(netCDFCxx_DEBUG) message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " NCXX4_CONFIG_LOCATION = ${NCXX4_CONFIG_LOCATION}") + " NCXX4_CONFIG_LOCATION = ${NCXX4_CONFIG_LOCATION}" + ) endif() -bout_inspect_netcdf_config(NCXX4_HINTS_INCLUDE_DIR "${NCXX4_CONFIG}" "--includedir") +bout_inspect_netcdf_config( + NCXX4_HINTS_INCLUDE_DIR "${NCXX4_CONFIG}" "--includedir" +) bout_inspect_netcdf_config(NCXX4_HINTS_PREFIX "${NCXX4_CONFIG}" "--prefix") -find_path(netCDF_CXX_INCLUDE_DIR +find_path( + netCDF_CXX_INCLUDE_DIR NAMES netcdf DOC "netCDF C++ include directories" - HINTS - "${netCDF_C_INCLUDE_DIR}" - "${NCXX4_HINTS_INCLUDE_DIR}" - "${NCXX4_HINTS_PREFIX}" - "${NCXX4_CONFIG_LOCATION}" - PATH_SUFFIXES - "include" + HINTS "${netCDF_C_INCLUDE_DIR}" "${NCXX4_HINTS_INCLUDE_DIR}" + "${NCXX4_HINTS_PREFIX}" "${NCXX4_CONFIG_LOCATION}" + PATH_SUFFIXES "include" +) +if(netCDFCxx_DEBUG) + message( + STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " netCDF_CXX_INCLUDE_DIR = ${netCDF_CXX_INCLUDE_DIR}" + " NCXX4_HINTS_INCLUDE_DIR = ${NCXX4_HINTS_INCLUDE_DIR}" + " NCXX4_HINTS_PREFIX = ${NCXX4_HINTS_PREFIX}" ) -if (netCDFCxx_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " netCDF_CXX_INCLUDE_DIR = ${netCDF_CXX_INCLUDE_DIR}" - " NCXX4_HINTS_INCLUDE_DIR = ${NCXX4_HINTS_INCLUDE_DIR}" - " NCXX4_HINTS_PREFIX = ${NCXX4_HINTS_PREFIX}" - ) endif() mark_as_advanced(netCDF_CXX_INCLUDE_DIR) -find_library(netCDF_CXX_LIBRARY +find_library( + netCDF_CXX_LIBRARY NAMES netcdf_c++4 netcdf-cxx4 DOC "netCDF C++ library" - HINTS - "${NCXX4_HINTS_INCLUDE_DIR}" - "${NCXX4_HINTS_PREFIX}" - "${NCXX4_CONFIG_LOCATION}" - PATH_SUFFIXES - "lib" "lib64" + HINTS "${NCXX4_HINTS_INCLUDE_DIR}" "${NCXX4_HINTS_PREFIX}" + "${NCXX4_CONFIG_LOCATION}" + PATH_SUFFIXES "lib" "lib64" +) +if(netCDFCxx_DEBUG) + message( + STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " netCDF_CXX_LIBRARY = ${netCDF_CXX_LIBRARY}" + " NCXX4_HINTS_INCLUDE_DIR = ${NCXX4_HINTS_INCLUDE_DIR}" + " NCXX4_HINTS_PREFIX = ${NCXX4_HINTS_PREFIX}" ) -if (netCDFCxx_DEBUG) - message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " - " netCDF_CXX_LIBRARY = ${netCDF_CXX_LIBRARY}" - " NCXX4_HINTS_INCLUDE_DIR = ${NCXX4_HINTS_INCLUDE_DIR}" - " NCXX4_HINTS_PREFIX = ${NCXX4_HINTS_PREFIX}" - ) endif() mark_as_advanced(netCDF_CXX_LIBRARY) bout_inspect_netcdf_config(_ncxx4_version "${NCXX4_CONFIG}" "--version") -if (_ncxx4_version) +if(_ncxx4_version) # Change to lower case before matching, to avoid case problems string(TOLOWER "${_ncxx4_version}" _ncxx4_version_lower) - string(REGEX REPLACE "netcdf-cxx4 \([0-9]+\\.[0-9]+\\.[0-9]+\).*" "\\1" netCDFCxx_VERSION "${_ncxx4_version_lower}") + string(REGEX REPLACE "netcdf-cxx4 \([0-9]+\\.[0-9]+\\.[0-9]+\).*" "\\1" + netCDFCxx_VERSION "${_ncxx4_version_lower}" + ) message(STATUS "Found netCDFCxx version ${netCDFCxx_VERSION}") -else () +else() message(WARNING "Couldn't get NetCDF version") endif() @@ -121,19 +122,23 @@ unset(_netcdf_version_minor) unset(_netcdf_version_patch) include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(netCDFCxx +find_package_handle_standard_args( + netCDFCxx REQUIRED_VARS netCDF_CXX_LIBRARY netCDF_CXX_INCLUDE_DIR - VERSION_VAR netCDFCxx_VERSION) + VERSION_VAR netCDFCxx_VERSION +) -if (netCDFCxx_FOUND) +if(netCDFCxx_FOUND) set(netCDFCxx_INCLUDE_DIRS "${netCDF_CXX_INCLUDE_DIR}") set(netCDFCxx_LIBRARIES "${netCDF_CXX_LIBRARY}") - if (NOT TARGET netCDF::netcdf-cxx4) + if(NOT TARGET netCDF::netcdf-cxx4) add_library(netCDF::netcdf-cxx4 UNKNOWN IMPORTED) - set_target_properties(netCDF::netcdf-cxx4 PROPERTIES - IMPORTED_LINK_INTERFACE_LIBRARIES netCDF::netcdf - IMPORTED_LOCATION "${netCDF_CXX_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${netCDF_CXX_INCLUDE_DIR}") - endif () -endif () + set_target_properties( + netCDF::netcdf-cxx4 + PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES netCDF::netcdf + IMPORTED_LOCATION "${netCDF_CXX_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${netCDF_CXX_INCLUDE_DIR}" + ) + endif() +endif() diff --git a/cmake/GenerateDateTimeFile.cmake b/cmake/GenerateDateTimeFile.cmake index ef48fc4638..8ed742b31a 100644 --- a/cmake/GenerateDateTimeFile.cmake +++ b/cmake/GenerateDateTimeFile.cmake @@ -2,6 +2,9 @@ # compilation date and time as variables set(bout_date_time_file - "const char* boutcompiledate{__DATE__}; const char* boutcompiletime{__TIME__};") + "const char* boutcompiledate{__DATE__}; const char* boutcompiletime{__TIME__};" +) -file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/bout++-time.cxx" "${bout_date_time_file}") +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/bout++-time.cxx" + "${bout_date_time_file}" +) diff --git a/cmake/GetGitRevisionDescription.cmake b/cmake/GetGitRevisionDescription.cmake index 8ab03bc5f0..33f95dd589 100644 --- a/cmake/GetGitRevisionDescription.cmake +++ b/cmake/GetGitRevisionDescription.cmake @@ -37,7 +37,7 @@ # http://www.boost.org/LICENSE_1_0.txt) if(__get_git_revision_description) - return() + return() endif() set(__get_git_revision_description YES) @@ -46,123 +46,152 @@ set(__get_git_revision_description YES) get_filename_component(_gitdescmoddir ${CMAKE_CURRENT_LIST_FILE} PATH) function(get_git_head_revision _refspecvar _hashvar) - set(GIT_PARENT_DIR "${CMAKE_CURRENT_SOURCE_DIR}") - set(GIT_DIR "${GIT_PARENT_DIR}/.git") - while(NOT EXISTS "${GIT_DIR}") # .git dir not found, search parent directories - set(GIT_PREVIOUS_PARENT "${GIT_PARENT_DIR}") - get_filename_component(GIT_PARENT_DIR ${GIT_PARENT_DIR} PATH) - if(GIT_PARENT_DIR STREQUAL GIT_PREVIOUS_PARENT) - # We have reached the root directory, we are not in git - set(${_refspecvar} "GITDIR-NOTFOUND" PARENT_SCOPE) - set(${_hashvar} "GITDIR-NOTFOUND" PARENT_SCOPE) - return() - endif() - set(GIT_DIR "${GIT_PARENT_DIR}/.git") - endwhile() - # check if this is a submodule - if(NOT IS_DIRECTORY ${GIT_DIR}) - file(READ ${GIT_DIR} submodule) - string(REGEX REPLACE "gitdir: (.*)\n$" "\\1" GIT_DIR_RELATIVE ${submodule}) - get_filename_component(SUBMODULE_DIR ${GIT_DIR} PATH) - get_filename_component(GIT_DIR ${SUBMODULE_DIR}/${GIT_DIR_RELATIVE} ABSOLUTE) - endif() - set(GIT_DATA "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/git-data") - if(NOT EXISTS "${GIT_DATA}") - file(MAKE_DIRECTORY "${GIT_DATA}") - endif() + set(GIT_PARENT_DIR "${CMAKE_CURRENT_SOURCE_DIR}") + set(GIT_DIR "${GIT_PARENT_DIR}/.git") + while(NOT EXISTS "${GIT_DIR}") # .git dir not found, search parent directories + set(GIT_PREVIOUS_PARENT "${GIT_PARENT_DIR}") + get_filename_component(GIT_PARENT_DIR ${GIT_PARENT_DIR} PATH) + if(GIT_PARENT_DIR STREQUAL GIT_PREVIOUS_PARENT) + # We have reached the root directory, we are not in git + set(${_refspecvar} + "GITDIR-NOTFOUND" + PARENT_SCOPE + ) + set(${_hashvar} + "GITDIR-NOTFOUND" + PARENT_SCOPE + ) + return() + endif() + set(GIT_DIR "${GIT_PARENT_DIR}/.git") + endwhile() + # check if this is a submodule + if(NOT IS_DIRECTORY ${GIT_DIR}) + file(READ ${GIT_DIR} submodule) + string(REGEX REPLACE "gitdir: (.*)\n$" "\\1" GIT_DIR_RELATIVE ${submodule}) + get_filename_component(SUBMODULE_DIR ${GIT_DIR} PATH) + get_filename_component( + GIT_DIR ${SUBMODULE_DIR}/${GIT_DIR_RELATIVE} ABSOLUTE + ) + endif() + set(GIT_DATA "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/git-data") + if(NOT EXISTS "${GIT_DATA}") + file(MAKE_DIRECTORY "${GIT_DATA}") + endif() - if(NOT EXISTS "${GIT_DIR}/HEAD") - return() - endif() - set(HEAD_FILE "${GIT_DATA}/HEAD") - configure_file("${GIT_DIR}/HEAD" "${HEAD_FILE}" COPYONLY) + if(NOT EXISTS "${GIT_DIR}/HEAD") + return() + endif() + set(HEAD_FILE "${GIT_DATA}/HEAD") + configure_file("${GIT_DIR}/HEAD" "${HEAD_FILE}" COPYONLY) - configure_file("${_gitdescmoddir}/GetGitRevisionDescription.cmake.in" - "${GIT_DATA}/grabRef.cmake" - @ONLY) - include("${GIT_DATA}/grabRef.cmake") + configure_file( + "${_gitdescmoddir}/GetGitRevisionDescription.cmake.in" + "${GIT_DATA}/grabRef.cmake" @ONLY + ) + include("${GIT_DATA}/grabRef.cmake") - set(${_refspecvar} "${HEAD_REF}" PARENT_SCOPE) - set(${_hashvar} "${HEAD_HASH}" PARENT_SCOPE) + set(${_refspecvar} + "${HEAD_REF}" + PARENT_SCOPE + ) + set(${_hashvar} + "${HEAD_HASH}" + PARENT_SCOPE + ) endfunction() function(git_describe _var) - if(NOT GIT_FOUND) - find_package(Git QUIET) - endif() - get_git_head_revision(refspec hash) - if(NOT GIT_FOUND) - set(${_var} "GIT-NOTFOUND" PARENT_SCOPE) - return() - endif() - if(NOT hash) - set(${_var} "HEAD-HASH-NOTFOUND" PARENT_SCOPE) - return() - endif() + if(NOT GIT_FOUND) + find_package(Git QUIET) + endif() + get_git_head_revision(refspec hash) + if(NOT GIT_FOUND) + set(${_var} + "GIT-NOTFOUND" + PARENT_SCOPE + ) + return() + endif() + if(NOT hash) + set(${_var} + "HEAD-HASH-NOTFOUND" + PARENT_SCOPE + ) + return() + endif() - # TODO sanitize - #if((${ARGN}" MATCHES "&&") OR - # (ARGN MATCHES "||") OR - # (ARGN MATCHES "\\;")) - # message("Please report the following error to the project!") - # message(FATAL_ERROR "Looks like someone's doing something nefarious with git_describe! Passed arguments ${ARGN}") - #endif() + # TODO sanitize + #if((${ARGN}" MATCHES "&&") OR + # (ARGN MATCHES "||") OR + # (ARGN MATCHES "\\;")) + # message("Please report the following error to the project!") + # message(FATAL_ERROR "Looks like someone's doing something nefarious with git_describe! Passed arguments ${ARGN}") + #endif() - #message(STATUS "Arguments to execute_process: ${ARGN}") + #message(STATUS "Arguments to execute_process: ${ARGN}") - execute_process(COMMAND - "${GIT_EXECUTABLE}" - describe - ${hash} - ${ARGN} - WORKING_DIRECTORY - "${CMAKE_CURRENT_SOURCE_DIR}" - RESULT_VARIABLE - res - OUTPUT_VARIABLE - out - ERROR_QUIET - OUTPUT_STRIP_TRAILING_WHITESPACE) - if(NOT res EQUAL 0) - set(out "${out}-${res}-NOTFOUND") - endif() + execute_process( + COMMAND "${GIT_EXECUTABLE}" describe ${hash} ${ARGN} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + RESULT_VARIABLE res + OUTPUT_VARIABLE out + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(NOT res EQUAL 0) + set(out "${out}-${res}-NOTFOUND") + endif() - set(${_var} "${out}" PARENT_SCOPE) + set(${_var} + "${out}" + PARENT_SCOPE + ) endfunction() function(git_get_exact_tag _var) - git_describe(out --exact-match ${ARGN}) - set(${_var} "${out}" PARENT_SCOPE) + git_describe(out --exact-match ${ARGN}) + set(${_var} + "${out}" + PARENT_SCOPE + ) endfunction() function(git_local_changes _var) - if(NOT GIT_FOUND) - find_package(Git QUIET) - endif() - get_git_head_revision(refspec hash) - if(NOT GIT_FOUND) - set(${_var} "GIT-NOTFOUND" PARENT_SCOPE) - return() - endif() - if(NOT hash) - set(${_var} "HEAD-HASH-NOTFOUND" PARENT_SCOPE) - return() - endif() + if(NOT GIT_FOUND) + find_package(Git QUIET) + endif() + get_git_head_revision(refspec hash) + if(NOT GIT_FOUND) + set(${_var} + "GIT-NOTFOUND" + PARENT_SCOPE + ) + return() + endif() + if(NOT hash) + set(${_var} + "HEAD-HASH-NOTFOUND" + PARENT_SCOPE + ) + return() + endif() - execute_process(COMMAND - "${GIT_EXECUTABLE}" - diff-index --quiet HEAD -- - WORKING_DIRECTORY - "${CMAKE_CURRENT_SOURCE_DIR}" - RESULT_VARIABLE - res - OUTPUT_VARIABLE - out - ERROR_QUIET - OUTPUT_STRIP_TRAILING_WHITESPACE) - if(res EQUAL 0) - set(${_var} "CLEAN" PARENT_SCOPE) - else() - set(${_var} "DIRTY" PARENT_SCOPE) - endif() + execute_process( + COMMAND "${GIT_EXECUTABLE}" diff-index --quiet HEAD -- + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + RESULT_VARIABLE res + OUTPUT_VARIABLE out + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(res EQUAL 0) + set(${_var} + "CLEAN" + PARENT_SCOPE + ) + else() + set(${_var} + "DIRTY" + PARENT_SCOPE + ) + endif() endfunction() diff --git a/cmake/ResolveCompilerPaths.cmake b/cmake/ResolveCompilerPaths.cmake index 54787fa38f..d4274496df 100644 --- a/cmake/ResolveCompilerPaths.cmake +++ b/cmake/ResolveCompilerPaths.cmake @@ -38,68 +38,77 @@ # # assuming both directories exist. # Note: as currently implemented, the -I/string will be picked up mistakenly (cry, cry) -include (CorrectWindowsPaths) +include(CorrectWindowsPaths) -macro (RESOLVE_LIBRARIES LIBS LINK_LINE) - string (REGEX MATCHALL "((-L|-l|-Wl)([^\" ]+|\"[^\"]+\")|[^\" ]+\\.(a|so|dll|lib))" _all_tokens "${LINK_LINE}") - set (_libs_found "") - set (_directory_list "") - foreach (token ${_all_tokens}) - if (token MATCHES "-L([^\" ]+|\"[^\"]+\")") +macro(RESOLVE_LIBRARIES LIBS LINK_LINE) + string(REGEX MATCHALL + "((-L|-l|-Wl)([^\" ]+|\"[^\"]+\")|[^\" ]+\\.(a|so|dll|lib))" + _all_tokens "${LINK_LINE}" + ) + set(_libs_found "") + set(_directory_list "") + foreach(token ${_all_tokens}) + if(token MATCHES "-L([^\" ]+|\"[^\"]+\")") # If it's a library path, add it to the list - string (REGEX REPLACE "^-L" "" token ${token}) - string (REGEX REPLACE "//" "/" token ${token}) + string(REGEX REPLACE "^-L" "" token ${token}) + string(REGEX REPLACE "//" "/" token ${token}) convert_cygwin_path(token) - list (APPEND _directory_list ${token}) - elseif (token MATCHES "^(-l([^\" ]+|\"[^\"]+\")|[^\" ]+\\.(a|so|dll|lib))") + list(APPEND _directory_list ${token}) + elseif(token MATCHES "^(-l([^\" ]+|\"[^\"]+\")|[^\" ]+\\.(a|so|dll|lib))") # It's a library, resolve the path by looking in the list and then (by default) in system directories - if (WIN32) #windows expects "libfoo", linux expects "foo" - string (REGEX REPLACE "^-l" "lib" token ${token}) - else (WIN32) - string (REGEX REPLACE "^-l" "" token ${token}) - endif (WIN32) - set (_root "") - if (token MATCHES "^/") # We have an absolute path + if(WIN32) #windows expects "libfoo", linux expects "foo" + string(REGEX REPLACE "^-l" "lib" token ${token}) + else(WIN32) + string(REGEX REPLACE "^-l" "" token ${token}) + endif(WIN32) + set(_root "") + if(token MATCHES "^/") # We have an absolute path #separate into a path and a library name: - string (REGEX MATCH "[^/]*\\.(a|so|dll|lib)$" libname ${token}) - string (REGEX MATCH ".*[^${libname}$]" libpath ${token}) + string(REGEX MATCH "[^/]*\\.(a|so|dll|lib)$" libname ${token}) + string(REGEX MATCH ".*[^${libname}$]" libpath ${token}) convert_cygwin_path(libpath) - set (_directory_list ${_directory_list} ${libpath}) - set (token ${libname}) - endif (token MATCHES "^/") - set (_lib "NOTFOUND" CACHE FILEPATH "Cleared" FORCE) - find_library (_lib ${token} HINTS ${_directory_list} ${_root}) - if (_lib) - string (REPLACE "//" "/" _lib ${_lib}) - list (APPEND _libs_found ${_lib}) - else (_lib) - message (STATUS "Unable to find library ${token}") - endif (_lib) - endif (token MATCHES "-L([^\" ]+|\"[^\"]+\")") - endforeach (token) - set (_lib "NOTFOUND" CACHE INTERNAL "Scratch variable" FORCE) + set(_directory_list ${_directory_list} ${libpath}) + set(token ${libname}) + endif(token MATCHES "^/") + set(_lib + "NOTFOUND" + CACHE FILEPATH "Cleared" FORCE + ) + find_library(_lib ${token} HINTS ${_directory_list} ${_root}) + if(_lib) + string(REPLACE "//" "/" _lib ${_lib}) + list(APPEND _libs_found ${_lib}) + else(_lib) + message(STATUS "Unable to find library ${token}") + endif(_lib) + endif(token MATCHES "-L([^\" ]+|\"[^\"]+\")") + endforeach(token) + set(_lib + "NOTFOUND" + CACHE INTERNAL "Scratch variable" FORCE + ) # only the LAST occurence of each library is required since there should be no circular dependencies - if (_libs_found) - list (REVERSE _libs_found) - list (REMOVE_DUPLICATES _libs_found) - list (REVERSE _libs_found) - endif (_libs_found) - set (${LIBS} "${_libs_found}") -endmacro (RESOLVE_LIBRARIES) + if(_libs_found) + list(REVERSE _libs_found) + list(REMOVE_DUPLICATES _libs_found) + list(REVERSE _libs_found) + endif(_libs_found) + set(${LIBS} "${_libs_found}") +endmacro(RESOLVE_LIBRARIES) -macro (RESOLVE_INCLUDES INCS COMPILE_LINE) - string (REGEX MATCHALL "-I([^\" ]+|\"[^\"]+\")" _all_tokens "${COMPILE_LINE}") - set (_incs_found "") - foreach (token ${_all_tokens}) - string (REGEX REPLACE "^-I" "" token ${token}) - string (REGEX REPLACE "//" "/" token ${token}) +macro(RESOLVE_INCLUDES INCS COMPILE_LINE) + string(REGEX MATCHALL "-I([^\" ]+|\"[^\"]+\")" _all_tokens "${COMPILE_LINE}") + set(_incs_found "") + foreach(token ${_all_tokens}) + string(REGEX REPLACE "^-I" "" token ${token}) + string(REGEX REPLACE "//" "/" token ${token}) convert_cygwin_path(token) - if (EXISTS ${token}) - list (APPEND _incs_found ${token}) - else (EXISTS ${token}) - message (STATUS "Include directory ${token} does not exist") - endif (EXISTS ${token}) - endforeach (token) - list (REMOVE_DUPLICATES _incs_found) - set (${INCS} "${_incs_found}") -endmacro (RESOLVE_INCLUDES) + if(EXISTS ${token}) + list(APPEND _incs_found ${token}) + else(EXISTS ${token}) + message(STATUS "Include directory ${token} does not exist") + endif(EXISTS ${token}) + endforeach(token) + list(REMOVE_DUPLICATES _incs_found) + set(${INCS} "${_incs_found}") +endmacro(RESOLVE_INCLUDES) diff --git a/cmake/Sanitizers.cmake b/cmake/Sanitizers.cmake index 715a08ab88..aab58d6f70 100644 --- a/cmake/Sanitizers.cmake +++ b/cmake/Sanitizers.cmake @@ -4,7 +4,9 @@ function(enable_sanitizers target_name) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES + ".*Clang" + ) option(ENABLE_COVERAGE "Enable coverage reporting for gcc/clang" FALSE) message(STATUS "Enable coverage: ${ENABLE_COVERAGE}") @@ -17,33 +19,40 @@ function(enable_sanitizers target_name) find_program(genhtml_FOUND genhtml) message(STATUS "Looking for genhtml: ${genhtml_FOUND}") - if (lcov_FOUND AND genhtml_FOUND) - set(COVERAGE_NAME coverage CACHE STRING "Name of coverage output file") + if(lcov_FOUND AND genhtml_FOUND) + set(COVERAGE_NAME + coverage + CACHE STRING "Name of coverage output file" + ) set(COVERAGE_FILE "${COVERAGE_NAME}.info") - set(COVERAGE_MSG "Open file://${PROJECT_SOURCE_DIR}/${COVERAGE_NAME}/index.html in your browser to view coverage HTML output") + set(COVERAGE_MSG + "Open file://${PROJECT_SOURCE_DIR}/${COVERAGE_NAME}/index.html in your browser to view coverage HTML output" + ) - add_custom_target(code-coverage-capture + add_custom_target( + code-coverage-capture COMMAND - lcov -c --directory "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/bout++.dir/src" - --output-file "${COVERAGE_FILE}" - COMMAND - genhtml --output-directory "${COVERAGE_NAME}" --demangle-cpp --legend --show-details "${COVERAGE_FILE}" - COMMAND - "${CMAKE_COMMAND}" -E echo ${COVERAGE_MSG} + lcov -c --directory + "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/bout++.dir/src" + --output-file "${COVERAGE_FILE}" + COMMAND genhtml --output-directory "${COVERAGE_NAME}" --demangle-cpp + --legend --show-details "${COVERAGE_FILE}" + COMMAND "${CMAKE_COMMAND}" -E echo ${COVERAGE_MSG} WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" COMMENT "Capturing coverage information" - BYPRODUCTS - "${COVERAGE_FILE}" - "${COVERAGE_NAME}/index.html" - ) + BYPRODUCTS "${COVERAGE_FILE}" "${COVERAGE_NAME}/index.html" + ) - add_custom_target(code-coverage-clean - COMMAND - lcov --zerocounters + add_custom_target( + code-coverage-clean + COMMAND lcov --zerocounters COMMENT "Cleaning coverage information" - ) + ) else() - message(FATAL_ERROR "Coverage enabled, but coverage-capture not available. Please install lcov") + message( + FATAL_ERROR + "Coverage enabled, but coverage-capture not available. Please install lcov" + ) endif() endif() @@ -60,7 +69,9 @@ function(enable_sanitizers target_name) list(APPEND SANITIZERS "leak") endif() - option(ENABLE_SANITIZER_UNDEFINED_BEHAVIOR "Enable undefined behavior sanitizer" FALSE) + option(ENABLE_SANITIZER_UNDEFINED_BEHAVIOR + "Enable undefined behavior sanitizer" FALSE + ) if(ENABLE_SANITIZER_UNDEFINED_BEHAVIOR) list(APPEND SANITIZERS "undefined") endif() @@ -68,7 +79,10 @@ function(enable_sanitizers target_name) option(ENABLE_SANITIZER_THREAD "Enable thread sanitizer" FALSE) if(ENABLE_SANITIZER_THREAD) if("address" IN_LIST SANITIZERS OR "leak" IN_LIST SANITIZERS) - message(WARNING "Thread sanitizer does not work with Address and Leak sanitizer enabled") + message( + WARNING + "Thread sanitizer does not work with Address and Leak sanitizer enabled" + ) else() list(APPEND SANITIZERS "thread") endif() @@ -78,32 +92,40 @@ function(enable_sanitizers target_name) if(ENABLE_SANITIZER_MEMORY AND CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") if("address" IN_LIST SANITIZERS OR "thread" IN_LIST SANITIZERS - OR "leak" IN_LIST SANITIZERS) - message(WARNING "Memory sanitizer does not work with Address, Thread and Leak sanitizer enabled") + OR "leak" IN_LIST SANITIZERS + ) + message( + WARNING + "Memory sanitizer does not work with Address, Thread and Leak sanitizer enabled" + ) else() list(APPEND SANITIZERS "memory") endif() endif() - list( - JOIN - SANITIZERS - "," - LIST_OF_SANITIZERS) + list(JOIN SANITIZERS "," LIST_OF_SANITIZERS) endif() # Default value gets overridden below - set(BOUT_USE_SANITIZERS "None" PARENT_SCOPE) + set(BOUT_USE_SANITIZERS + "None" + PARENT_SCOPE + ) if(LIST_OF_SANITIZERS) - if(NOT - "${LIST_OF_SANITIZERS}" - STREQUAL - "") - set(BOUT_USE_SANITIZERS ${LIST_OF_SANITIZERS} PARENT_SCOPE) - target_compile_options(${target_name} PUBLIC -fsanitize=${LIST_OF_SANITIZERS} -fno-omit-frame-pointer) - target_link_options(${target_name} PUBLIC -fsanitize=${LIST_OF_SANITIZERS}) + if(NOT "${LIST_OF_SANITIZERS}" STREQUAL "") + set(BOUT_USE_SANITIZERS + ${LIST_OF_SANITIZERS} + PARENT_SCOPE + ) + target_compile_options( + ${target_name} PUBLIC -fsanitize=${LIST_OF_SANITIZERS} + -fno-omit-frame-pointer + ) + target_link_options( + ${target_name} PUBLIC -fsanitize=${LIST_OF_SANITIZERS} + ) endif() endif() diff --git a/cmake/SetupBOUTThirdParty.cmake b/cmake/SetupBOUTThirdParty.cmake index 53ceb4351c..9e6bdc6d2a 100644 --- a/cmake/SetupBOUTThirdParty.cmake +++ b/cmake/SetupBOUTThirdParty.cmake @@ -1,12 +1,12 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake/") # determined in SetupCompilers.cmake -if (BOUT_USE_MPI) +if(BOUT_USE_MPI) target_link_libraries(bout++ PUBLIC MPI::MPI_CXX) -endif () +endif() # determined in SetupCompilers.cmake -if (BOUT_USE_OPENMP) +if(BOUT_USE_OPENMP) target_link_libraries(bout++ PUBLIC OpenMP::OpenMP_CXX) set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} -fopenmp") set(CONFIG_LDFLAGS_SHARED "${CONFIG_LDFLAGS_SHARED} -fopenmp") @@ -14,7 +14,7 @@ if (BOUT_USE_OPENMP) endif() # determined in SetupCompilers.cmake -if (BOUT_HAS_CUDA) +if(BOUT_HAS_CUDA) enable_language(CUDA) message(STATUS "BOUT_HAS_CUDA ${CMAKE_CUDA_COMPILER}") @@ -29,58 +29,65 @@ if (BOUT_HAS_CUDA) set_target_properties(bout++ PROPERTIES CUDA_SEPARABLE_COMPILATION ON) set_target_properties(bout++ PROPERTIES POSITION_INDEPENDENT_CODE ON) set_target_properties(bout++ PROPERTIES LINKER_LANGUAGE CUDA) -endif () +endif() # Caliper option(BOUT_ENABLE_CALIPER "Enable Caliper" OFF) -if (BOUT_ENABLE_CALIPER) +if(BOUT_ENABLE_CALIPER) find_package(caliper REQUIRED) target_include_directories(bout++ PUBLIC ${caliper_INCLUDE_DIR}) target_link_libraries(bout++ PUBLIC caliper) -endif () +endif() set(BOUT_HAS_CALIPER ${BOUT_ENABLE_CALIPER}) # UMPIRE option(BOUT_ENABLE_UMPIRE "Enable UMPIRE memory management" OFF) -if (BOUT_ENABLE_UMPIRE) +if(BOUT_ENABLE_UMPIRE) find_package(UMPIRE REQUIRED) target_include_directories(bout++ PUBLIC ${UMPIRE_INCLUDE_DIRS}/include) target_link_libraries(bout++ PUBLIC umpire) -endif () +endif() set(BOUT_HAS_UMPIRE ${BOUT_ENABLE_UMPIRE}) # RAJA option(BOUT_ENABLE_RAJA "Enable RAJA" OFF) -if (BOUT_ENABLE_RAJA) +if(BOUT_ENABLE_RAJA) find_package(RAJA REQUIRED) - message(STATUS "RAJA_CONFIG:" ${RAJA_CONFIG}) + message(STATUS "RAJA_CONFIG:" ${RAJA_CONFIG}) string(FIND ${RAJA_CONFIG} "raja" loc) math(EXPR value "${loc} + 5" OUTPUT_FORMAT DECIMAL) - string(SUBSTRING ${RAJA_CONFIG} 0 ${value} RAJA_PATH) + string(SUBSTRING ${RAJA_CONFIG} 0 ${value} RAJA_PATH) message(STATUS "RAJA_PATH" ${RAJA_PATH}) target_include_directories(bout++ PUBLIC ${RAJA_PATH}/include) target_link_libraries(bout++ PUBLIC RAJA) -endif () +endif() set(BOUT_HAS_RAJA ${BOUT_ENABLE_RAJA}) # Hypre option(BOUT_USE_HYPRE "Enable support for Hypre solvers" OFF) -if (BOUT_USE_HYPRE) +if(BOUT_USE_HYPRE) enable_language(C) find_package(HYPRE REQUIRED) target_link_libraries(bout++ PUBLIC HYPRE::HYPRE) - if (HYPRE_WITH_CUDA AND BOUT_HAS_CUDA) - target_compile_definitions(bout++ PUBLIC "HYPRE_USING_CUDA;HYPRE_USING_UNIFIED_MEMORY") - target_link_libraries(bout++ PUBLIC CUDA::cusparse CUDA::curand CUDA::culibos CUDA::cublas CUDA::cublasLt) - endif () -endif () + if(HYPRE_WITH_CUDA AND BOUT_HAS_CUDA) + target_compile_definitions( + bout++ PUBLIC "HYPRE_USING_CUDA;HYPRE_USING_UNIFIED_MEMORY" + ) + target_link_libraries( + bout++ PUBLIC CUDA::cusparse CUDA::curand CUDA::culibos CUDA::cublas + CUDA::cublasLt + ) + endif() +endif() message(STATUS "HYPRE support: ${BOUT_USE_HYPRE}") set(BOUT_HAS_HYPRE ${BOUT_USE_HYPRE}) # PETSc -option(BOUT_USE_PETSC "Enable support for PETSc time solvers and inversions" OFF) -if (BOUT_USE_PETSC) - if (NOT CMAKE_SYSTEM_NAME STREQUAL "CrayLinuxEnvironment") +option(BOUT_USE_PETSC "Enable support for PETSc time solvers and inversions" + OFF +) +if(BOUT_USE_PETSC) + if(NOT CMAKE_SYSTEM_NAME STREQUAL "CrayLinuxEnvironment") # Cray wrappers sort this out for us find_package(PETSc REQUIRED) target_link_libraries(bout++ PUBLIC PETSc::PETSc) @@ -94,28 +101,44 @@ endif() message(STATUS "PETSc support: ${BOUT_USE_PETSC}") set(BOUT_HAS_PETSC ${BOUT_USE_PETSC}) - -cmake_dependent_option(BOUT_USE_SYSTEM_MPARK_VARIANT "Use external installation of mpark.variant" OFF - "BOUT_UPDATE_GIT_SUBMODULE OR EXISTS ${PROJECT_SOURCE_DIR}/externalpackages/mpark.variant/CMakeLists.txt" ON) +cmake_dependent_option( + BOUT_USE_SYSTEM_MPARK_VARIANT + "Use external installation of mpark.variant" + OFF + "BOUT_UPDATE_GIT_SUBMODULE OR EXISTS ${PROJECT_SOURCE_DIR}/externalpackages/mpark.variant/CMakeLists.txt" + ON +) if(BOUT_USE_SYSTEM_MPARK_VARIANT) message(STATUS "Using external mpark.variant") find_package(mpark_variant REQUIRED) - get_target_property(MPARK_VARIANT_INCLUDE_PATH mpark_variant INTERFACE_INCLUDE_DIRECTORIES) + get_target_property( + MPARK_VARIANT_INCLUDE_PATH mpark_variant INTERFACE_INCLUDE_DIRECTORIES + ) else() message(STATUS "Using mpark.variant submodule") bout_update_submodules() add_subdirectory(externalpackages/mpark.variant) if(NOT TARGET mpark_variant) - message(FATAL_ERROR "mpark_variant not found! Have you disabled the git submodules (BOUT_UPDATE_GIT_SUBMODULE)?") + message( + FATAL_ERROR + "mpark_variant not found! Have you disabled the git submodules (BOUT_UPDATE_GIT_SUBMODULE)?" + ) endif() - set(MPARK_VARIANT_INCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/externalpackages/mpark.variant/include") + set(MPARK_VARIANT_INCLUDE_PATH + "${CMAKE_CURRENT_SOURCE_DIR}/externalpackages/mpark.variant/include" + ) set(CONFIG_CFLAGS "${CONFIG_CFLAGS} -I\${MPARK_VARIANT_INCLUDE_PATH}") endif() target_link_libraries(bout++ PUBLIC mpark_variant) -cmake_dependent_option(BOUT_USE_SYSTEM_FMT "Use external installation of fmt" OFF - "BOUT_UPDATE_GIT_SUBMODULE OR EXISTS ${PROJECT_SOURCE_DIR}/externalpackages/fmt/CMakeLists.txt" ON) +cmake_dependent_option( + BOUT_USE_SYSTEM_FMT + "Use external installation of fmt" + OFF + "BOUT_UPDATE_GIT_SUBMODULE OR EXISTS ${PROJECT_SOURCE_DIR}/externalpackages/fmt/CMakeLists.txt" + ON +) if(BOUT_USE_SYSTEM_FMT) message(STATUS "Using external fmt") @@ -125,32 +148,46 @@ else() message(STATUS "Using fmt submodule") bout_update_submodules() # Need to install fmt alongside BOUT++ - set(FMT_INSTALL ON CACHE BOOL "") - set(FMT_DEBUG_POSTFIX "" CACHE STRING "") + set(FMT_INSTALL + ON + CACHE BOOL "" + ) + set(FMT_DEBUG_POSTFIX + "" + CACHE STRING "" + ) add_subdirectory(externalpackages/fmt) if(NOT TARGET fmt::fmt) - message(FATAL_ERROR "fmt not found! Have you disabled the git submodules (BOUT_UPDATE_GIT_SUBMODULE)?") + message( + FATAL_ERROR + "fmt not found! Have you disabled the git submodules (BOUT_UPDATE_GIT_SUBMODULE)?" + ) endif() # Build the library in /lib: this makes updating the path # for bout-config much easier - set_target_properties(fmt PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib" - ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib") - set(FMT_INCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/externalpackages/fmt/include") + set_target_properties( + fmt PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib" + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib" + ) + set(FMT_INCLUDE_PATH + "${CMAKE_CURRENT_SOURCE_DIR}/externalpackages/fmt/include" + ) set(CONFIG_CFLAGS "${CONFIG_CFLAGS} -I\${FMT_INCLUDE_PATH}") set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} -lfmt") endif() target_link_libraries(bout++ PUBLIC fmt::fmt) option(BOUT_USE_PVODE "Enable support for bundled PVODE" ON) -if (BOUT_USE_PVODE) +if(BOUT_USE_PVODE) add_subdirectory(externalpackages/PVODE) target_link_libraries(bout++ PUBLIC pvode pvpre) # Build the libraries in /lib: this makes updating the # path for bout-config much easier - set_target_properties(pvode pvpre PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib" - ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib") + set_target_properties( + pvode pvpre + PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib" + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib" + ) set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} -lpvode -lpvpre") endif() message(STATUS "PVODE support: ${BOUT_USE_PVODE}") @@ -158,25 +195,30 @@ set(BOUT_HAS_PVODE ${BOUT_USE_PVODE}) option(BOUT_USE_NETCDF "Enable support for NetCDF output" ON) option(BOUT_DOWNLOAD_NETCDF_CXX4 "Download and build netCDF-cxx4" OFF) -if (BOUT_USE_NETCDF) - if (BOUT_DOWNLOAD_NETCDF_CXX4) +if(BOUT_USE_NETCDF) + if(BOUT_DOWNLOAD_NETCDF_CXX4) message(STATUS "Downloading and configuring NetCDF-cxx4") include(FetchContent) FetchContent_Declare( netcdf-cxx4 - GIT_REPOSITORY https://github.com/ZedThree/netcdf-cxx4 - GIT_TAG "ad3e50953190615cb69dcc8a4652f9a88a8499cf" - ) + GIT_REPOSITORY https://github.com/Unidata/netcdf-cxx4 + GIT_TAG "a43d6d4d415d407712c246faca553bd951730dc1" + ) # Don't build the netcdf tests, they have lots of warnings - set(NCXX_ENABLE_TESTS OFF CACHE BOOL "" FORCE) + set(NCXX_ENABLE_TESTS + OFF + CACHE BOOL "" FORCE + ) # Use our own FindnetCDF module which uses nc-config find_package(netCDF REQUIRED) FetchContent_MakeAvailable(netcdf-cxx4) target_link_libraries(bout++ PUBLIC netCDF::netcdf-cxx4) else() find_package(netCDFCxx) - if (netCDFCxx_FOUND) - set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} ${netCDF_CXX_LIBRARY} ${netCDF_LIBRARIES}") + if(netCDFCxx_FOUND) + set(CONFIG_LDFLAGS + "${CONFIG_LDFLAGS} ${netCDF_CXX_LIBRARY} ${netCDF_LIBRARIES}" + ) target_link_libraries(bout++ PUBLIC netCDF::netcdf-cxx4) else() find_package(PkgConfig REQUIRED) @@ -186,17 +228,23 @@ if (BOUT_USE_NETCDF) set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} ${NETCDF_LDFLAGS_STRING}") endif() endif() + if(netCDF_DIR) + set(netCDF_ROOT "${netCDF_DIR}") + endif() + if(netCDFCxx_DIR) + set(netCDFCxx_ROOT "${netCDFCxx_DIR}") + endif() endif() message(STATUS "NetCDF support: ${BOUT_USE_NETCDF}") set(BOUT_HAS_NETCDF ${BOUT_USE_NETCDF}) option(BOUT_USE_ADIOS2 "Enable support for ADIOS output" OFF) option(BOUT_DOWNLOAD_ADIOS2 "Download and build ADIOS2" OFF) -if (BOUT_USE_ADIOS2) +if(BOUT_USE_ADIOS2) enable_language(C) find_package(MPI REQUIRED COMPONENTS C) - if (BOUT_DOWNLOAD_ADIOS2) + if(BOUT_DOWNLOAD_ADIOS2) message(STATUS "Downloading and configuring ADIOS2") include(FetchContent) FetchContent_Declare( @@ -204,15 +252,33 @@ if (BOUT_USE_ADIOS2) GIT_REPOSITORY https://github.com/ornladios/ADIOS2.git GIT_TAG origin/master GIT_SHALLOW 1 - ) - set(ADIOS2_USE_MPI ON CACHE BOOL "" FORCE) - set(ADIOS2_USE_Fortran OFF CACHE BOOL "" FORCE) - set(ADIOS2_USE_Python OFF CACHE BOOL "" FORCE) - set(ADIOS2_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + ) + set(ADIOS2_USE_MPI + ON + CACHE BOOL "" FORCE + ) + set(ADIOS2_USE_Fortran + OFF + CACHE BOOL "" FORCE + ) + set(ADIOS2_USE_Python + OFF + CACHE BOOL "" FORCE + ) + set(ADIOS2_BUILD_EXAMPLES + OFF + CACHE BOOL "" FORCE + ) # Disable testing, or ADIOS will try to find or install GTEST - set(BUILD_TESTING OFF CACHE BOOL "" FORCE) + set(BUILD_TESTING + OFF + CACHE BOOL "" FORCE + ) # Note: SST requires but doesn't check at configure time - set(ADIOS2_USE_SST OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_SST + OFF + CACHE BOOL "" FORCE + ) FetchContent_MakeAvailable(adios2) message(STATUS "ADIOS2 done configuring") else() @@ -223,9 +289,8 @@ endif() message(STATUS "ADIOS2 support: ${BOUT_USE_ADIOS2}") set(BOUT_HAS_ADIOS2 ${BOUT_USE_ADIOS2}) - option(BOUT_USE_FFTW "Enable support for FFTW" ON) -if (BOUT_USE_FFTW) +if(BOUT_USE_FFTW) find_package(FFTW REQUIRED) target_link_libraries(bout++ PUBLIC FFTW::FFTW) set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} ${FFTW_LIBRARIES}") @@ -238,15 +303,15 @@ option(BOUT_USE_LAPACK "Enable support for LAPACK" AUTO) set_property(CACHE BOUT_USE_LAPACK PROPERTY STRINGS ${ON_OFF_AUTO}) set(LAPACK_FOUND OFF) -if (BOUT_USE_LAPACK) - if (NOT CMAKE_SYSTEM_NAME STREQUAL "CrayLinuxEnvironment") +if(BOUT_USE_LAPACK) + if(NOT CMAKE_SYSTEM_NAME STREQUAL "CrayLinuxEnvironment") # Cray wrappers sort this out for us - if (BOUT_USE_LAPACK STREQUAL ON) + if(BOUT_USE_LAPACK STREQUAL ON) find_package(LAPACK REQUIRED) else() find_package(LAPACK) endif() - if (LAPACK_FOUND) + if(LAPACK_FOUND) target_link_libraries(bout++ PUBLIC "${LAPACK_LIBRARIES}") string(JOIN " " CONFIG_LAPACK_LIBRARIES ${LAPACK_LIBRARIES}) set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} ${CONFIG_LAPACK_LIBRARIES}") @@ -258,7 +323,7 @@ message(STATUS "LAPACK support: ${LAPACK_FOUND}") set(BOUT_HAS_LAPACK ${LAPACK_FOUND}) option(BOUT_USE_SLEPC "Enable support for SLEPc eigen solver" OFF) -if (BOUT_USE_SLEPC) +if(BOUT_USE_SLEPC) find_package(SLEPc REQUIRED) target_link_libraries(bout++ PUBLIC SLEPc::SLEPc) string(JOIN " " CONFIG_SLEPC_LIBRARIES ${SLEPC_LIBRARIES}) @@ -269,41 +334,69 @@ set(BOUT_HAS_SLEPC ${BOUT_USE_SLEPC}) option(BOUT_DOWNLOAD_SUNDIALS "Download and build SUNDIALS" OFF) # Force BOUT_USE_SUNDIALS if we're downloading it! -cmake_dependent_option(BOUT_USE_SUNDIALS "Enable support for SUNDIALS time solvers" OFF - "NOT BOUT_DOWNLOAD_SUNDIALS" ON) -if (BOUT_USE_SUNDIALS) +cmake_dependent_option( + BOUT_USE_SUNDIALS "Enable support for SUNDIALS time solvers" OFF + "NOT BOUT_DOWNLOAD_SUNDIALS" ON +) +if(BOUT_USE_SUNDIALS) enable_language(C) - if (BOUT_DOWNLOAD_SUNDIALS) + if(BOUT_DOWNLOAD_SUNDIALS) message(STATUS "Downloading and configuring SUNDIALS") include(FetchContent) FetchContent_Declare( sundials GIT_REPOSITORY https://github.com/LLNL/sundials - GIT_TAG v7.2.1 - ) + GIT_TAG v7.2.1 + ) # Note: These are settings for building SUNDIALS - set(EXAMPLES_ENABLE_C OFF CACHE BOOL "" FORCE) - set(EXAMPLES_INSTALL OFF CACHE BOOL "" FORCE) - set(ENABLE_MPI ${BOUT_USE_MPI} CACHE BOOL "" FORCE) - set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE) - if (BUILD_SHARED_LIBS) - set(BUILD_STATIC_LIBS OFF CACHE BOOL "" FORCE) + set(EXAMPLES_ENABLE_C + OFF + CACHE BOOL "" FORCE + ) + set(EXAMPLES_INSTALL + OFF + CACHE BOOL "" FORCE + ) + set(ENABLE_MPI + ${BOUT_USE_MPI} + CACHE BOOL "" FORCE + ) + set(ENABLE_OPENMP + ${BOUT_USE_OPENMP} + CACHE BOOL "" FORCE + ) + if(BUILD_SHARED_LIBS) + set(BUILD_STATIC_LIBS + OFF + CACHE BOOL "" FORCE + ) else() - set(BUILD_STATIC_LIBS ON CACHE BOOL "" FORCE) + set(BUILD_STATIC_LIBS + ON + CACHE BOOL "" FORCE + ) endif() FetchContent_MakeAvailable(sundials) message(STATUS "SUNDIALS done configuring") else() find_package(SUNDIALS REQUIRED) - if (SUNDIALS_VERSION VERSION_LESS 4.0.0) - message(FATAL_ERROR "SUNDIALS_VERSION 4.0.0 or newer is required. Found version ${SUNDIALS_VERSION}.") + if(SUNDIALS_VERSION VERSION_LESS 4.0.0) + message( + FATAL_ERROR + "SUNDIALS_VERSION 4.0.0 or newer is required. Found version ${SUNDIALS_VERSION}." + ) endif() endif() + if(SUNDIALS_DIR) + set(SUNDIALS_ROOT "${SUNDIALS_DIR}") + endif() target_link_libraries(bout++ PUBLIC SUNDIALS::nvecparallel) target_link_libraries(bout++ PUBLIC SUNDIALS::cvode) target_link_libraries(bout++ PUBLIC SUNDIALS::ida) target_link_libraries(bout++ PUBLIC SUNDIALS::arkode) - set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} ${SUNDIALS_cvode_LIBRARY} ${SUNDIALS_ida_LIBRARY} ${SUNDIALS_arkode_LIBRARY} ${SUNDIALS_nvecparallel_LIBRARY}") + set(CONFIG_LDFLAGS + "${CONFIG_LDFLAGS} ${SUNDIALS_cvode_LIBRARY} ${SUNDIALS_ida_LIBRARY} ${SUNDIALS_arkode_LIBRARY} ${SUNDIALS_nvecparallel_LIBRARY}" + ) endif() message(STATUS "SUNDIALS support: ${BOUT_USE_SUNDIALS}") set(BOUT_HAS_SUNDIALS ${BOUT_USE_SUNDIALS}) @@ -312,46 +405,51 @@ set(BOUT_HAS_CVODE ${BOUT_USE_SUNDIALS}) set(BOUT_HAS_IDA ${BOUT_USE_SUNDIALS}) set(ON_OFF_AUTO ON OFF AUTO) -set(BOUT_USE_NLS AUTO CACHE STRING "Enable Native Language Support") +set(BOUT_USE_NLS + AUTO + CACHE STRING "Enable Native Language Support" +) set_property(CACHE BOUT_USE_NLS PROPERTY STRINGS ${ON_OFF_AUTO}) set(BOUT_HAS_GETTEXT OFF) -if (BOUT_USE_NLS) +if(BOUT_USE_NLS) find_package(Gettext) - if (GETTEXT_FOUND) + if(GETTEXT_FOUND) find_package(Intl) - if (Intl_FOUND) - target_link_libraries(bout++ - PUBLIC ${Intl_LIBRARIES}) - target_include_directories(bout++ - PUBLIC ${Intl_INCLUDE_DIRS}) + if(Intl_FOUND) + target_link_libraries(bout++ PUBLIC ${Intl_LIBRARIES}) + target_include_directories(bout++ PUBLIC ${Intl_INCLUDE_DIRS}) set(BOUT_HAS_GETTEXT ON) else() - if (NOT BOUT_USE_NLS STREQUAL "AUTO") - message(FATAL_ERROR "Intl not found but requested!") + if(NOT BOUT_USE_NLS STREQUAL "AUTO") + message(FATAL_ERROR "Intl not found but requested!") endif() endif() else() - if (NOT BOUT_USE_NLS STREQUAL "AUTO") + if(NOT BOUT_USE_NLS STREQUAL "AUTO") message(FATAL_ERROR "GETTEXT not found but requested!") endif() endif() endif() option(BOUT_USE_SCOREP "Enable support for Score-P based instrumentation" OFF) -if (BOUT_USE_SCOREP) - message(STATUS "Score-P support enabled. Please make sure you are calling CMake like so: +if(BOUT_USE_SCOREP) + message( + STATUS + "Score-P support enabled. Please make sure you are calling CMake like so: SCOREP_WRAPPER=off cmake -DCMAKE_C_COMPILER=scorep-mpicc -DCMAKE_CXX_COMPILER=scorep-mpicxx -") +" + ) endif() set(BOUT_HAS_SCOREP ${BOUT_USE_SCOREP}) -option(BOUT_USE_UUID_SYSTEM_GENERATOR "Enable support for using a system UUID generator" ON) -if (BOUT_USE_UUID_SYSTEM_GENERATOR) +option(BOUT_USE_UUID_SYSTEM_GENERATOR + "Enable support for using a system UUID generator" ON +) +if(BOUT_USE_UUID_SYSTEM_GENERATOR) find_package(Libuuid QUIET) - if (Libuuid_FOUND) - target_link_libraries(bout++ - PUBLIC Libuuid::libuuid) + if(Libuuid_FOUND) + target_link_libraries(bout++ PUBLIC Libuuid::libuuid) set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} ${Libuuid_LIBRARIES}") else() message(STATUS "libuuid not found, using fallback UUID generator") @@ -360,3 +458,43 @@ if (BOUT_USE_UUID_SYSTEM_GENERATOR) endif() message(STATUS "UUID_SYSTEM_GENERATOR: ${BOUT_USE_UUID_SYSTEM_GENERATOR}") set(BOUT_HAS_UUID_SYSTEM_GENERATOR ${BOUT_USE_UUID_SYSTEM_GENERATOR}) + +cmake_dependent_option( + BOUT_USE_SYSTEM_CPPTRACE + "Use external installation of cpptrace" + OFF + "BOUT_UPDATE_GIT_SUBMODULE OR EXISTS ${PROJECT_SOURCE_DIR}/externalpackages/cpptrace/CMakeLists.txt" + ON +) + +if(BOUT_USE_SYSTEM_CPPTRACE) + message(STATUS "Using external cpptrace") + find_package(cpptrace REQUIRED) + get_target_property( + CPPTRACE_INCLUDE_PATH cpptrace::cpptrace INTERFACE_INCLUDE_DIRECTORIES + ) +else() + message(STATUS "Using cpptrace submodule") + bout_update_submodules() + # Need a fork with some fixes for CMake + set(CPPTRACE_LIBDWARF_REPO + "https://github.com/ZedThree/libdwarf-lite.git" + CACHE STRING "" FORCE + ) + set(CPPTRACE_LIBDWARF_TAG + "ebe10a39afd56b8247de633bfe17666ad50ab95e" + CACHE STRING "" FORCE + ) + add_subdirectory(externalpackages/cpptrace) + if(NOT TARGET cpptrace::cpptrace) + message( + FATAL_ERROR + "cpptrace not found! Have you disabled the git submodules (BOUT_UPDATE_GIT_SUBMODULE)?" + ) + endif() + set(CPPTRACE_INCLUDE_PATH + "${CMAKE_CURRENT_SOURCE_DIR}/externalpackages/cpptrace/include" + ) +endif() +set(CONFIG_CFLAGS "${CONFIG_CFLAGS} -I\${CPPTRACE_INCLUDE_PATH}") +target_link_libraries(bout++ PUBLIC cpptrace::cpptrace) diff --git a/cmake/SetupCompilers.cmake b/cmake/SetupCompilers.cmake index 647cb20f75..fbe960b0f0 100644 --- a/cmake/SetupCompilers.cmake +++ b/cmake/SetupCompilers.cmake @@ -1,44 +1,58 @@ - # Note: Currently BOUT++ always needs MPI. This option just determines # whether the find_* routines are used option(BOUT_ENABLE_MPI "Enable MPI support" ON) if(BOUT_ENABLE_MPI) - # This might not be entirely sensible, but helps CMake to find the - # correct MPI, workaround for https://gitlab.kitware.com/cmake/cmake/issues/18895 - find_program(MPIEXEC_EXECUTABLE NAMES mpiexec mpirun) - find_package(MPI REQUIRED) -endif () + # This might not be entirely sensible, but helps CMake to find the + # correct MPI, workaround for https://gitlab.kitware.com/cmake/cmake/issues/18895 + find_program(MPIEXEC_EXECUTABLE NAMES mpiexec mpirun) + find_package(MPI REQUIRED) +endif() set(BOUT_USE_MPI ${BOUT_ENABLE_MPI}) option(BOUT_ENABLE_OPENMP "Enable OpenMP support" OFF) -set(BOUT_OPENMP_SCHEDULE static CACHE STRING "Set OpenMP schedule") -set_property(CACHE BOUT_OPENMP_SCHEDULE PROPERTY STRINGS static dynamic guided auto) -if (BOUT_ENABLE_OPENMP) +set(BOUT_OPENMP_SCHEDULE + static + CACHE STRING "Set OpenMP schedule" +) +set_property( + CACHE BOUT_OPENMP_SCHEDULE PROPERTY STRINGS static dynamic guided auto +) +if(BOUT_ENABLE_OPENMP) find_package(OpenMP REQUIRED) set(possible_openmp_schedules static dynamic guided auto) - if (NOT BOUT_OPENMP_SCHEDULE IN_LIST possible_openmp_schedules) - message(FATAL_ERROR "BOUT_OPENMP_SCHEDULE must be one of ${possible_openmp_schedules}; got ${BOUT_OPENMP_SCHEDULE}") + if(NOT BOUT_OPENMP_SCHEDULE IN_LIST possible_openmp_schedules) + message( + FATAL_ERROR + "BOUT_OPENMP_SCHEDULE must be one of ${possible_openmp_schedules}; got ${BOUT_OPENMP_SCHEDULE}" + ) endif() message(STATUS "OpenMP schedule: ${BOUT_OPENMP_SCHEDULE}") -endif () +endif() set(BOUT_USE_OPENMP ${BOUT_ENABLE_OPENMP}) message(STATUS "Enable OpenMP: ${BOUT_ENABLE_OPENMP}") option(BOUT_ENABLE_CUDA "Enable CUDA support" OFF) -set(CUDA_ARCH "compute_70,code=sm_70" CACHE STRING "CUDA architecture") +set(CUDA_ARCH + "compute_70,code=sm_70" + CACHE STRING "CUDA architecture" +) if(BOUT_ENABLE_CUDA) - # Set specific options for CUDA if enabled - enable_language(CUDA) - set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode arch=${CUDA_ARCH} -ccbin ${CMAKE_CXX_COMPILER}") - if (BOUT_ENABLE_RAJA) - # RAJA uses lambda expressions - set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr") - endif () + # Set specific options for CUDA if enabled + enable_language(CUDA) + set(CMAKE_CUDA_FLAGS + "${CMAKE_CUDA_FLAGS} -gencode arch=${CUDA_ARCH} -ccbin ${CMAKE_CXX_COMPILER}" + ) + if(BOUT_ENABLE_RAJA) + # RAJA uses lambda expressions + set(CMAKE_CUDA_FLAGS + "${CMAKE_CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr" + ) + endif() -# TODO Ensure openmp flags are not enabled twice! - if (BOUT_ENABLE_OPENMP) - # CMAKE_CUDA_FLAGS does not pass OpenMP_CXX_FLAGS to the host compiler by default - set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler ${OpenMP_CXX_FLAGS}") - endif () + # TODO Ensure openmp flags are not enabled twice! + if(BOUT_ENABLE_OPENMP) + # CMAKE_CUDA_FLAGS does not pass OpenMP_CXX_FLAGS to the host compiler by default + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler ${OpenMP_CXX_FLAGS}") + endif() endif() set(BOUT_HAS_CUDA ${BOUT_ENABLE_CUDA}) diff --git a/cmake_build_defines.hxx.in b/cmake_build_defines.hxx.in index 4d63a01b7d..9c73679f29 100644 --- a/cmake_build_defines.hxx.in +++ b/cmake_build_defines.hxx.in @@ -21,7 +21,6 @@ #cmakedefine01 BOUT_HAS_SLEPC #cmakedefine01 BOUT_HAS_SUNDIALS #cmakedefine01 BOUT_HAS_UUID_SYSTEM_GENERATOR -#cmakedefine01 BOUT_USE_BACKTRACE #cmakedefine01 BOUT_USE_COLOR #cmakedefine01 BOUT_USE_OPENMP #cmakedefine01 BOUT_USE_OUTPUT_DEBUG @@ -39,4 +38,7 @@ // CMake build does not support legacy interface #define BOUT_HAS_LEGACY_NETCDF 0 +// We now always turn this on +#define BOUT_USE_BACKTRACE 1 + #endif // BOUT_BUILD_CONFIG_HXX diff --git a/examples/6field-simple/CMakeLists.txt b/examples/6field-simple/CMakeLists.txt index 6a51327cda..b6584c49e4 100644 --- a/examples/6field-simple/CMakeLists.txt +++ b/examples/6field-simple/CMakeLists.txt @@ -2,10 +2,12 @@ cmake_minimum_required(VERSION 3.13) project(6field-simple LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(elm_6f +bout_add_example( + elm_6f SOURCES elm_6f.cxx - EXTRA_FILES cbm18_dens8.grid_nx68ny64.nc) + EXTRA_FILES cbm18_dens8.grid_nx68ny64.nc +) diff --git a/examples/6field-simple/elm_6f.cxx b/examples/6field-simple/elm_6f.cxx index 40e4f7b3b3..24256333df 100644 --- a/examples/6field-simple/elm_6f.cxx +++ b/examples/6field-simple/elm_6f.cxx @@ -273,8 +273,6 @@ class Elm_6f : public PhysicsModel { * This function implements d2/dy2 where y is the poloidal coordinate theta */ - TRACE("Grad2_par2new( Field3D )"); - Field3D result = D2DY2(f); #if BOUT_USE_TRACK @@ -342,8 +340,6 @@ class Elm_6f : public PhysicsModel { // Parallel gradient along perturbed field-line Field3D Grad_parP(const Field3D& f, CELL_LOC loc = CELL_DEFAULT) { - TRACE("Grad_parP"); - Field3D result; if (parallel_lagrange || parallel_project) { diff --git a/examples/IMEX/advection-diffusion/CMakeLists.txt b/examples/IMEX/advection-diffusion/CMakeLists.txt index 334d48d767..f2008f3479 100644 --- a/examples/IMEX/advection-diffusion/CMakeLists.txt +++ b/examples/IMEX/advection-diffusion/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.13) project(advection-diffusion LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() diff --git a/examples/IMEX/advection-reaction/CMakeLists.txt b/examples/IMEX/advection-reaction/CMakeLists.txt index 03e8686371..d89ff10a78 100644 --- a/examples/IMEX/advection-reaction/CMakeLists.txt +++ b/examples/IMEX/advection-reaction/CMakeLists.txt @@ -2,10 +2,12 @@ cmake_minimum_required(VERSION 3.13) project(advection-reaction LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(split_operator +bout_add_example( + split_operator SOURCES split_operator.cxx - EXTRA_FILES simple_xz.nc) + EXTRA_FILES simple_xz.nc +) diff --git a/examples/IMEX/diffusion-nl/CMakeLists.txt b/examples/IMEX/diffusion-nl/CMakeLists.txt index 664d16e042..73c7250bb5 100644 --- a/examples/IMEX/diffusion-nl/CMakeLists.txt +++ b/examples/IMEX/diffusion-nl/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.13) project(diffusion-nl LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() diff --git a/examples/IMEX/drift-wave-constraint/CMakeLists.txt b/examples/IMEX/drift-wave-constraint/CMakeLists.txt index 5680b5367e..b72396d2f0 100644 --- a/examples/IMEX/drift-wave-constraint/CMakeLists.txt +++ b/examples/IMEX/drift-wave-constraint/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.13) project(drift-wave-constraint LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() diff --git a/examples/IMEX/drift-wave/CMakeLists.txt b/examples/IMEX/drift-wave/CMakeLists.txt index e3e2a1b8ee..44fbbd7b0f 100644 --- a/examples/IMEX/drift-wave/CMakeLists.txt +++ b/examples/IMEX/drift-wave/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.13) project(drift-wave LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() diff --git a/examples/blob2d-laplacexz/CMakeLists.txt b/examples/blob2d-laplacexz/CMakeLists.txt index 17f9ebe97a..855d86af70 100644 --- a/examples/blob2d-laplacexz/CMakeLists.txt +++ b/examples/blob2d-laplacexz/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.13) project(blob2d-laplacexz LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() diff --git a/examples/blob2d-outerloop/CMakeLists.txt b/examples/blob2d-outerloop/CMakeLists.txt index cd7187ee3f..e991b45d51 100644 --- a/examples/blob2d-outerloop/CMakeLists.txt +++ b/examples/blob2d-outerloop/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.13) project(blob2d-outerloop LANGUAGES CXX C) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() diff --git a/examples/blob2d/CMakeLists.txt b/examples/blob2d/CMakeLists.txt index a4772874d9..f3f93f0246 100644 --- a/examples/blob2d/CMakeLists.txt +++ b/examples/blob2d/CMakeLists.txt @@ -2,11 +2,13 @@ cmake_minimum_required(VERSION 3.13) project(blob2d LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(blob2d +bout_add_example( + blob2d SOURCES blob2d.cxx DATA_DIRS delta_0.25 delta_1 delta_10 two_blobs data - EXTRA_FILES blob_velocity.py) + EXTRA_FILES blob_velocity.py +) diff --git a/examples/blob2d/blob_velocity.py b/examples/blob2d/blob_velocity.py index d044946320..6de0937d0f 100644 --- a/examples/blob2d/blob_velocity.py +++ b/examples/blob2d/blob_velocity.py @@ -3,96 +3,100 @@ import pickle try: - from past.utils import old_div + from past.utils import old_div except ImportError: - def old_div(a,b): - return a/b - -def blob_velocity(n,**kwargs): - - from boututils import calculus as Calc - # Calculate blob velocity in normalized time and normalized grid spacing - # - # Input: Blob density as a 3D vector in the form n[t,x,z] where t is time and x,z are the perpendicular spatial coordinates - # - # Keywords: - # - # type='peak' -> Calculate velocity of the peak density - # type='COM' -> Calculate centre of mass velocity - # Index=True -> return indices used to create velocity - # - # Default: Peak velocity with no index returning - - size = n.shape - - try: - v_type = kwargs['type'] - except: - v_type = 'peak' #Default to peak velocity calculation - try: - return_index = kwargs['Index'] - except: - return_index = False #Default to no index returning - - - if v_type == 'peak': - x = np.zeros(size[0]) - z = np.zeros(size[0]) - for i in np.arange(size[0]): - nmax,nmin = np.amax((n[i,:,:])),np.amin((n[i,:,:])) - xpos,zpos = np.where(n[i,:,:]==nmax) - x[i] = xpos[0] - z[i] = zpos[0] - - if v_type == 'COM': - x = np.zeros(size[0]) - z = np.zeros(size[0]) - for i in np.arange(size[0]): - data = n[i,:,:] - n[0,0,0] #use corner cell rather than nmin - ntot = np.sum(data[:,:]) - - z[i] = old_div(np.sum(np.sum(data[:,:],axis=0)*(np.arange(size[2]))),ntot) - x[i] = old_div(np.sum(np.sum(data[:,:],axis=1)*(np.arange(size[1]))),ntot) - - vx = Calc.deriv(x) - vz = Calc.deriv(z) - - if return_index: - return vx,vz,x,z - else: - return vx,vz - - - -data='data' + def old_div(a, b): + return a / b + + +def blob_velocity(n, **kwargs): + from boututils import calculus as Calc + # Calculate blob velocity in normalized time and normalized grid spacing + # + # Input: Blob density as a 3D vector in the form n[t,x,z] where t is time and x,z are the perpendicular spatial coordinates + # + # Keywords: + # + # type='peak' -> Calculate velocity of the peak density + # type='COM' -> Calculate centre of mass velocity + # Index=True -> return indices used to create velocity + # + # Default: Peak velocity with no index returning + + size = n.shape + + try: + v_type = kwargs["type"] + except: + v_type = "peak" # Default to peak velocity calculation + try: + return_index = kwargs["Index"] + except: + return_index = False # Default to no index returning + + if v_type == "peak": + x = np.zeros(size[0]) + z = np.zeros(size[0]) + for i in np.arange(size[0]): + nmax, nmin = np.amax((n[i, :, :])), np.amin((n[i, :, :])) + xpos, zpos = np.where(n[i, :, :] == nmax) + x[i] = xpos[0] + z[i] = zpos[0] + + if v_type == "COM": + x = np.zeros(size[0]) + z = np.zeros(size[0]) + for i in np.arange(size[0]): + data = n[i, :, :] - n[0, 0, 0] # use corner cell rather than nmin + ntot = np.sum(data[:, :]) + + z[i] = old_div( + np.sum(np.sum(data[:, :], axis=0) * (np.arange(size[2]))), ntot + ) + x[i] = old_div( + np.sum(np.sum(data[:, :], axis=1) * (np.arange(size[1]))), ntot + ) + + vx = Calc.deriv(x) + vz = Calc.deriv(z) + + if return_index: + return vx, vz, x, z + else: + return vx, vz + + +data = "data" if True: - import sys - if len(sys.argv) > 1: - data=sys.argv[1] + import sys + + if len(sys.argv) > 1: + data = sys.argv[1] -n = collect('n', path=data, info=False) +n = collect("n", path=data, info=False) -vx,vy,xx,yy = blob_velocity(n[:,:,0,:],type='COM',Index=True) +vx, vy, xx, yy = blob_velocity(n[:, :, 0, :], type="COM", Index=True) -f = open('Velocity.dat','wb') -pickle.dump(vx,f) +f = open("Velocity.dat", "wb") +pickle.dump(vx, f) f.close() -f = open('Position.dat','wb') -pickle.dump(xx,f) +f = open("Position.dat", "wb") +pickle.dump(xx, f) f.close() -f = open('Velocity.dat','rb') +f = open("Velocity.dat", "rb") vx = pickle.load(f) f.close() try: - import matplotlib.pyplot as plt - plt.plot(vx) - plt.show() + import matplotlib.pyplot as plt + + plt.plot(vx) + plt.show() except ImportError: - pass + pass diff --git a/examples/boundary-conditions/advection/CMakeLists.txt b/examples/boundary-conditions/advection/CMakeLists.txt index a4ab73a24d..edd55d183a 100644 --- a/examples/boundary-conditions/advection/CMakeLists.txt +++ b/examples/boundary-conditions/advection/CMakeLists.txt @@ -2,13 +2,12 @@ cmake_minimum_required(VERSION 3.13) project(advection LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(advection +bout_add_example( + advection SOURCES advection.cxx - DATA_DIRS central-dirichlet - central-free - central-free-o3 - upwind) + DATA_DIRS central-dirichlet central-free central-free-o3 upwind +) diff --git a/examples/boutpp/CMakeLists.txt b/examples/boutpp/CMakeLists.txt index e46a7ae990..1cc5c9619b 100644 --- a/examples/boutpp/CMakeLists.txt +++ b/examples/boutpp/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.13) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() diff --git a/examples/boutpp/simulation.py b/examples/boutpp/simulation.py index 1b57ff8b33..9948bd9b03 100755 --- a/examples/boutpp/simulation.py +++ b/examples/boutpp/simulation.py @@ -3,12 +3,13 @@ bc.init("mesh:n=48") + class Model(bc.PhysicsModel): - def init(self,restart): + def init(self, restart): self.dens = bc.create3D("sin(x)") self.solve_for(n=self.dens) - def rhs(self,time): + def rhs(self, time): self.dens.ddt(bc.DDX(self.dens)) diff --git a/examples/conducting-wall-mode/CMakeLists.txt b/examples/conducting-wall-mode/CMakeLists.txt index 857a22038e..0e000c1ba8 100644 --- a/examples/conducting-wall-mode/CMakeLists.txt +++ b/examples/conducting-wall-mode/CMakeLists.txt @@ -2,10 +2,12 @@ cmake_minimum_required(VERSION 3.13) project(conducting-wall-mode LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(conducting-wall-mode +bout_add_example( + conducting-wall-mode SOURCES cwm.cxx - EXTRA_FILES cwm_grid.nc) + EXTRA_FILES cwm_grid.nc +) diff --git a/examples/conduction-snb/CMakeLists.txt b/examples/conduction-snb/CMakeLists.txt index 45072dbe59..3de25718d1 100644 --- a/examples/conduction-snb/CMakeLists.txt +++ b/examples/conduction-snb/CMakeLists.txt @@ -2,11 +2,20 @@ cmake_minimum_required(VERSION 3.13) project(conduction-snb LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(conduction-snb +bout_add_example( + conduction-snb SOURCES conduction-snb.cxx - EXTRA_FILES fit_temperature.py sinusoid.py snb.csv spitzer-harm.csv step.py temperature.csv vfp.csv - DATA_DIRS data step) + EXTRA_FILES + fit_temperature.py + sinusoid.py + snb.csv + spitzer-harm.csv + step.py + temperature.csv + vfp.csv + DATA_DIRS data step +) diff --git a/examples/conduction-snb/fit_temperature.py b/examples/conduction-snb/fit_temperature.py index 1e7f3ddf94..81faa9cf93 100644 --- a/examples/conduction-snb/fit_temperature.py +++ b/examples/conduction-snb/fit_temperature.py @@ -3,23 +3,31 @@ import matplotlib.pyplot as plt Te_ref = np.loadtxt("temperature.csv", delimiter=",") -Te_ref[:,0] *= 1e-4 # Convert X axis to m +Te_ref[:, 0] *= 1e-4 # Convert X axis to m + def te_function(ypos, mid, wwid, w0, w1, w2, Tmax, Tmin, clip=False): - width = w0 + ((ypos - mid)*w1 + (ypos - mid)**2 * w2) * np.exp(-((ypos - mid)/wwid)**2) + width = w0 + ((ypos - mid) * w1 + (ypos - mid) ** 2 * w2) * np.exp( + -(((ypos - mid) / wwid) ** 2) + ) if clip: width = np.clip(width, 1e-10, None) - return Tmax - 0.5 * (1 + np.tanh((ypos - mid)/width)) * (Tmax - Tmin) + return Tmax - 0.5 * (1 + np.tanh((ypos - mid) / width)) * (Tmax - Tmin) + -popt, pcov = optimize.curve_fit(te_function, Te_ref[:,0], Te_ref[:,1], - p0 = [2.2e-4, 1e-4, 1e-4, 0.0, 0.0, 0.960, 0.190]) +popt, pcov = optimize.curve_fit( + te_function, + Te_ref[:, 0], + Te_ref[:, 1], + p0=[2.2e-4, 1e-4, 1e-4, 0.0, 0.0, 0.960, 0.190], +) print(popt) -xfit = np.linspace(Te_ref[0,0], Te_ref[-1,0], 100) +xfit = np.linspace(Te_ref[0, 0], Te_ref[-1, 0], 100) -plt.plot(xfit, te_function(xfit, *popt, clip=True), '-k') -plt.plot(Te_ref[:,0], Te_ref[:,1], 'or') +plt.plot(xfit, te_function(xfit, *popt, clip=True), "-k") +plt.plot(Te_ref[:, 0], Te_ref[:, 1], "or") plt.show() diff --git a/examples/conduction-snb/sinusoid.py b/examples/conduction-snb/sinusoid.py index 10c81923a3..439530383e 100644 --- a/examples/conduction-snb/sinusoid.py +++ b/examples/conduction-snb/sinusoid.py @@ -11,7 +11,7 @@ build_and_log("Sinusoidal SNB") # Electron temperature in eV -Telist = 10 ** np.linspace(0,3,20) +Telist = 10 ** np.linspace(0, 3, 20) # Electron density in m^-3 Ne = 1e20 @@ -19,29 +19,30 @@ # Length of the domain in m length = 1.0 -c = 299792458 -mu0 = 4.e-7*np.pi -e0 = 1/(c*c*mu0) +c = 299792458 +mu0 = 4.0e-7 * np.pi +e0 = 1 / (c * c * mu0) qe = 1.602176634e-19 me = 9.10938356e-31 -thermal_speed = np.sqrt(2.*qe * Telist / me) -Y = (qe**2 / (e0 * me))**2 / (4 * np.pi) +thermal_speed = np.sqrt(2.0 * qe * Telist / me) +Y = (qe**2 / (e0 * me)) ** 2 / (4 * np.pi) coulomb_log = 6.6 - 0.5 * np.log(Ne * 1e-20) + 1.5 * np.log(Telist) -lambda_ee_T = thermal_speed**4 / (Y * Ne * coulomb_log) +lambda_ee_T = thermal_speed**4 / (Y * Ne * coulomb_log) beta_max_list = [5, 10, 20, 40] -colors = ['k','b','g','r'] +colors = ["k", "b", "g", "r"] ngroups_list = [20, 40, 80] -syms = ['x', 'o', 'D'] +syms = ["x", "o", "D"] for beta_max, color in zip(beta_max_list, colors): for ngroups, sym in zip(ngroups_list, syms): - flux_ratio = [] for Te in Telist: - cmd = "./conduction-snb \"Te={0}+0.01*sin(y)\" Ne={1} mesh:length={2} snb:beta_max={3} snb:ngroups={4}".format(Te, Ne, length, beta_max, ngroups) + cmd = './conduction-snb "Te={0}+0.01*sin(y)" Ne={1} mesh:length={2} snb:beta_max={3} snb:ngroups={4}'.format( + Te, Ne, length, beta_max, ngroups + ) # Run the case s, out = launch_safe(cmd, nproc=1, mthread=1, pipe=True) @@ -54,7 +55,12 @@ flux_ratio.append(div_q[ind] / div_q_SH[ind]) - plt.plot(lambda_ee_T / length, flux_ratio, '-'+sym+color, label=r"$\beta_{{max}}={0}, N_g={1}$".format(beta_max,ngroups)) + plt.plot( + lambda_ee_T / length, + flux_ratio, + "-" + sym + color, + label=r"$\beta_{{max}}={0}, N_g={1}$".format(beta_max, ngroups), + ) plt.legend() plt.xlabel(r"$\lambda_{ee,T} / L$") diff --git a/examples/conduction-snb/step.py b/examples/conduction-snb/step.py index 1f8933e66b..63a13149f4 100644 --- a/examples/conduction-snb/step.py +++ b/examples/conduction-snb/step.py @@ -3,7 +3,7 @@ # # Uses a step in the temperature, intended for comparison to VFP results -length = 6e-4 # Domain length in m +length = 6e-4 # Domain length in m qe = 1.602176634e-19 @@ -40,37 +40,37 @@ # Read reference values Te_ref = np.loadtxt("temperature.csv", delimiter=",") -Te_ref[:,0] *= 1e-4 # Convert X axis to m +Te_ref[:, 0] *= 1e-4 # Convert X axis to m SH_ref = np.loadtxt("spitzer-harm.csv", delimiter=",") -SH_ref[:,0] *= 1e-4 +SH_ref[:, 0] *= 1e-4 SNB_ref = np.loadtxt("snb.csv", delimiter=",") -SNB_ref[:,0] *= 1e-4 +SNB_ref[:, 0] *= 1e-4 VFP_ref = np.loadtxt("vfp.csv", delimiter=",") -VFP_ref[:,0] *= 1e-4 +VFP_ref[:, 0] *= 1e-4 ######################################### fig, ax1 = plt.subplots() -color='tab:red' +color = "tab:red" ax1.plot(position, Te * 1e-3, color=color, label="Te") -ax1.plot(Te_ref[:,0], Te_ref[:,1], color=color, marker="o", label="Reference Te") +ax1.plot(Te_ref[:, 0], Te_ref[:, 1], color=color, marker="o", label="Reference Te") ax1.set_xlabel("position [m]") ax1.set_ylabel("Electron temperature [keV]", color=color) -ax1.set_ylim(0,1) -ax1.tick_params(axis='y', colors=color) +ax1.set_ylim(0, 1) +ax1.tick_params(axis="y", colors=color) ax2 = ax1.twinx() -ax2.plot(position, q_SH * 1e-4, '-k', label="Spitzer-Harm") -ax2.plot(SH_ref[:,0], SH_ref[:,1], '--k', label="Reference SH") +ax2.plot(position, q_SH * 1e-4, "-k", label="Spitzer-Harm") +ax2.plot(SH_ref[:, 0], SH_ref[:, 1], "--k", label="Reference SH") -ax2.plot(position, q * 1e-4, '-b', label="SNB") -ax2.plot(SNB_ref[:,0], SNB_ref[:,1], '--b', label="Reference SNB") +ax2.plot(position, q * 1e-4, "-b", label="SNB") +ax2.plot(SNB_ref[:, 0], SNB_ref[:, 1], "--b", label="Reference SNB") -ax2.plot(VFP_ref[:,0], VFP_ref[:,1], '--g', label="Reference VFP") +ax2.plot(VFP_ref[:, 0], VFP_ref[:, 1], "--g", label="Reference VFP") ax2.set_ylabel("Heat flux W/cm^2") ax2.set_ylim(bottom=0.0) @@ -78,8 +78,7 @@ plt.legend() fig.tight_layout() -plt.xlim(0,3.5e-4) +plt.xlim(0, 3.5e-4) plt.savefig("snb-step.png") plt.show() - diff --git a/examples/conduction/CMakeLists.txt b/examples/conduction/CMakeLists.txt index f26b838621..b81164f614 100644 --- a/examples/conduction/CMakeLists.txt +++ b/examples/conduction/CMakeLists.txt @@ -2,11 +2,13 @@ cmake_minimum_required(VERSION 3.13) project(conduction LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(conduction +bout_add_example( + conduction SOURCES conduction.cxx DATA_DIRS data fromfile - EXTRA_FILES generate.py) + EXTRA_FILES generate.py +) diff --git a/examples/conduction/generate.py b/examples/conduction/generate.py index 7138026898..4a2a2acfc0 100755 --- a/examples/conduction/generate.py +++ b/examples/conduction/generate.py @@ -4,9 +4,9 @@ # Generate an input mesh # -from boututils.datafile import DataFile # Wrapper around NetCDF4 libraries +from boututils.datafile import DataFile # Wrapper around NetCDF4 libraries -nx = 5 # Minimum is 5: 2 boundary, one evolved +nx = 5 # Minimum is 5: 2 boundary, one evolved ny = 64 # Minimum 5. Should be divisible by number of processors (so powers of 2 nice) f = DataFile() diff --git a/examples/constraints/alfven-wave/CMakeLists.txt b/examples/constraints/alfven-wave/CMakeLists.txt index a95ace4086..375415ab3c 100644 --- a/examples/constraints/alfven-wave/CMakeLists.txt +++ b/examples/constraints/alfven-wave/CMakeLists.txt @@ -2,12 +2,13 @@ cmake_minimum_required(VERSION 3.13) project(constraints-alfven-wave LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(constraints-alfven-wave +bout_add_example( + constraints-alfven-wave SOURCES alfven.cxx DATA_DIRS cbm18 data - EXTRA_FILES cbm18_dens8.grid_nx68ny64.nc d3d_119919.nc) - + EXTRA_FILES cbm18_dens8.grid_nx68ny64.nc d3d_119919.nc +) diff --git a/examples/constraints/laplace-dae/CMakeLists.txt b/examples/constraints/laplace-dae/CMakeLists.txt index e487bd6a0a..d515fc3515 100644 --- a/examples/constraints/laplace-dae/CMakeLists.txt +++ b/examples/constraints/laplace-dae/CMakeLists.txt @@ -2,10 +2,12 @@ cmake_minimum_required(VERSION 3.13) project(constraints-laplace-dae LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(constraints-laplace-dae +bout_add_example( + constraints-laplace-dae SOURCES laplace_dae.cxx - EXTRA_FILES simple_xz.nc) + EXTRA_FILES simple_xz.nc +) diff --git a/examples/dalf3/CMakeLists.txt b/examples/dalf3/CMakeLists.txt index 5f5b3d701d..0b6f8461f5 100644 --- a/examples/dalf3/CMakeLists.txt +++ b/examples/dalf3/CMakeLists.txt @@ -2,11 +2,12 @@ cmake_minimum_required(VERSION 3.13) project(dalf3 LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() - -bout_add_example(dalf3 +bout_add_example( + dalf3 SOURCES dalf3.cxx - EXTRA_FILES cbm18_8_y064_x516_090309.nc) + EXTRA_FILES cbm18_8_y064_x516_090309.nc +) diff --git a/examples/dalf3/dalf3.cxx b/examples/dalf3/dalf3.cxx index 46a9a4286d..b27ae97113 100644 --- a/examples/dalf3/dalf3.cxx +++ b/examples/dalf3/dalf3.cxx @@ -71,8 +71,6 @@ class DALF3 : public PhysicsModel { BoutReal viscosity, hyper_viscosity; - bool smooth_separatrix; - FieldGroup comms; std::unique_ptr phiSolver{nullptr}; // Laplacian solver in X-Z @@ -132,7 +130,6 @@ class DALF3 : public PhysicsModel { viscosity = options["viscosity"].withDefault(-1.0); hyper_viscosity = options["hyper_viscosity"].withDefault(-1.0); viscosity_par = options["viscosity_par"].withDefault(-1.0); - smooth_separatrix = options["smooth_separatrix"].withDefault(false); filter_z = options["filter_z"].withDefault(false); @@ -466,11 +463,6 @@ class DALF3 : public PhysicsModel { ddt(Pe) = -bracket(phi, Pet, bm) + Pet * (Kappa(phi - Pe) + B0 * Grad_parP(jpar - Vpar) / B0); - if (smooth_separatrix) { - // Experimental smoothing across separatrix - ddt(Vort) += mesh->smoothSeparatrix(Vort); - } - if (filter_z) { ddt(Pe) = filter(ddt(Pe), 1); } diff --git a/examples/eigen-box/CMakeLists.txt b/examples/eigen-box/CMakeLists.txt index 76af4fbaa6..305e330652 100644 --- a/examples/eigen-box/CMakeLists.txt +++ b/examples/eigen-box/CMakeLists.txt @@ -2,10 +2,12 @@ cmake_minimum_required(VERSION 3.13) project(eigen-box LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(eigen-box +bout_add_example( + eigen-box SOURCES eigen-box.cxx - EXTRA_FILES eigenvals.py) + EXTRA_FILES eigenvals.py +) diff --git a/examples/eigen-box/eigenvals.py b/examples/eigen-box/eigenvals.py index 7b52fc60f2..5a71aabfc6 100755 --- a/examples/eigen-box/eigenvals.py +++ b/examples/eigen-box/eigenvals.py @@ -27,7 +27,8 @@ def plot_eigenvals(eigenvalues, eigenvectors=None): raise ValueError("Expecting eigenvectors to be 2D") if eigenvectors.shape[0] != len(eigenvalues): raise ValueError( - "First dimension of eigenvectors must match length of eigenvalues") + "First dimension of eigenvectors must match length of eigenvalues" + ) # If no eigenvectors supplied, only plot eigenvalues, otherwise # eigenvalues and eigenvectors @@ -44,18 +45,18 @@ def plot_eigenvals(eigenvalues, eigenvectors=None): range_r = amax(eigs_r) - amin(eigs_r) range_i = amax(eigs_i) - amin(eigs_i) - ax[0].plot(eigs_r, eigs_i, 'x') + ax[0].plot(eigs_r, eigs_i, "x") ax[0].set_xlabel("Real component") ax[0].set_ylabel("Imaginary component") ax[0].set_title("Eigenvalue") - overplot, = ax[0].plot([], [], 'ok') + (overplot,) = ax[0].plot([], [], "ok") if eigenvectors is not None: # Add a eigenvectors plot - vector_r, = ax[1].plot([], [], '-k', label="Real") - vector_i, = ax[1].plot([], [], '-r', label="Imag") - ax[1].legend(loc='upper right') + (vector_r,) = ax[1].plot([], [], "-k", label="Real") + (vector_i,) = ax[1].plot([], [], "-r", label="Imag") + ax[1].legend(loc="upper right") ax[1].set_xlabel("X") ax[1].set_ylabel("Amplitude") ax[1].set_title("Eigenvector") @@ -68,33 +69,33 @@ def onclick(event): # Find closest eigenvectors point, but stretch axes so # real and imaginary components are weighted equally - if(range_r == 0): - dist = ((eigs_i - event.ydata)/range_i)**2 - elif(range_i == 0): - dist = ((eigs_r - event.xdata)/range_r)**2 + if range_r == 0: + dist = ((eigs_i - event.ydata) / range_i) ** 2 + elif range_i == 0: + dist = ((eigs_r - event.xdata) / range_r) ** 2 else: - dist = ((eigs_r - event.xdata)/range_r)**2 + \ - ((eigs_i - event.ydata)/range_i)**2 + dist = ((eigs_r - event.xdata) / range_r) ** 2 + ( + (eigs_i - event.ydata) / range_i + ) ** 2 ind = argmin(dist) # Update the highlight plot overplot.set_data([eigs_r[ind]], [eigs_i[ind]]) - print("Eigenvalue number: %d (%e,%e)" % - (ind, eigs_r[ind], eigs_i[ind])) + print("Eigenvalue number: %d (%e,%e)" % (ind, eigs_r[ind], eigs_i[ind])) if eigenvectors is not None: # Update plots nx = eigenvectors.shape[1] - vector_r.set_data(arange(nx), eigenvectors[2*ind, :]) - vector_i.set_data(arange(nx), eigenvectors[2*ind+1, :]) + vector_r.set_data(arange(nx), eigenvectors[2 * ind, :]) + vector_i.set_data(arange(nx), eigenvectors[2 * ind + 1, :]) ax[1].relim() ax[1].autoscale_view() fig.canvas.draw() - fig.canvas.mpl_connect('button_press_event', onclick) + fig.canvas.mpl_connect("button_press_event", onclick) plt.show() diff --git a/examples/elm-pb-outerloop/CMakeLists.txt b/examples/elm-pb-outerloop/CMakeLists.txt index 008918a87f..12dc2f7696 100644 --- a/examples/elm-pb-outerloop/CMakeLists.txt +++ b/examples/elm-pb-outerloop/CMakeLists.txt @@ -2,16 +2,16 @@ cmake_minimum_required(VERSION 3.13) project(elm_pb LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(elm_pb_outerloop +bout_add_example( + elm_pb_outerloop SOURCES elm_pb_outerloop.cxx EXTRA_FILES cbm18_dens8.grid_nx68ny64.nc ) if(BOUT_HAS_CUDA) - set_source_files_properties(elm_pb_outerloop.cxx PROPERTIES LANGUAGE CUDA ) + set_source_files_properties(elm_pb_outerloop.cxx PROPERTIES LANGUAGE CUDA) endif() - diff --git a/examples/elm-pb-outerloop/elm_pb_outerloop.cxx b/examples/elm-pb-outerloop/elm_pb_outerloop.cxx index 4ff8e90dcb..3cc5227b30 100644 --- a/examples/elm-pb-outerloop/elm_pb_outerloop.cxx +++ b/examples/elm-pb-outerloop/elm_pb_outerloop.cxx @@ -39,7 +39,6 @@ #include #include #include -#include #include #include // Defines BOUT_FOR_RAJA #include diff --git a/examples/elm-pb/CMakeLists.txt b/examples/elm-pb/CMakeLists.txt index 8c672822cf..6cdd0f9cd9 100644 --- a/examples/elm-pb/CMakeLists.txt +++ b/examples/elm-pb/CMakeLists.txt @@ -2,14 +2,13 @@ cmake_minimum_required(VERSION 3.13) project(elm_pb LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(elm_pb +bout_add_example( + elm_pb SOURCES elm_pb.cxx - EXTRA_FILES cbm18_dens8.grid_nx68ny64.nc - data/BOUT.inp - data-hypre/BOUT.inp - data-nonlinear/BOUT.inp) - + EXTRA_FILES cbm18_dens8.grid_nx68ny64.nc data/BOUT.inp data-hypre/BOUT.inp + data-nonlinear/BOUT.inp +) diff --git a/examples/elm-pb/Python/2dprofile.py b/examples/elm-pb/Python/2dprofile.py index 5bb01972f9..e95709ab5b 100644 --- a/examples/elm-pb/Python/2dprofile.py +++ b/examples/elm-pb/Python/2dprofile.py @@ -4,81 +4,92 @@ import numpy as np import matplotlib.pyplot as plt from boututils.datafile import DataFile -from matplotlib.ticker import FixedFormatter, FormatStrFormatter, AutoLocator, AutoMinorLocator +from matplotlib.ticker import ( + FixedFormatter, + FormatStrFormatter, + AutoLocator, + AutoMinorLocator, +) with DataFile("./cbm18_dens8.grid_nx68ny64.nc") as f: g = {v: f.read(v) for v in f.keys()} -majorLocator = AutoLocator() -majorFormatter = FormatStrFormatter('%3.0e') -minorLocator = AutoMinorLocator() -Fm = FixedFormatter(['0','$1 \\times 10^4$','$2 \\times 10^4$','$3 \\times 10^4$','$4 \\times 10^4$']) -Fm2 = FixedFormatter(['0','$2 \\times 10^5$','$4 \\times 10^5$','$6 \\times 10^5$']) - -bxy=g.get('Bxy') -p=g.get('pressure') -jpar0=g.get('Jpar0') -psixy=g.get('psixy') -btxy=g.get('Btxy') -shiftangle=g.get('ShiftAngle') - -nx=g.get('nx') -ny=g.get('ny') +majorLocator = AutoLocator() +majorFormatter = FormatStrFormatter("%3.0e") +minorLocator = AutoMinorLocator() +Fm = FixedFormatter( + [ + "0", + "$1 \\times 10^4$", + "$2 \\times 10^4$", + "$3 \\times 10^4$", + "$4 \\times 10^4$", + ] +) +Fm2 = FixedFormatter(["0", "$2 \\times 10^5$", "$4 \\times 10^5$", "$6 \\times 10^5$"]) + +bxy = g.get("Bxy") +p = g.get("pressure") +jpar0 = g.get("Jpar0") +psixy = g.get("psixy") +btxy = g.get("Btxy") +shiftangle = g.get("ShiftAngle") + +nx = g.get("nx") +ny = g.get("ny") q = np.zeros((nx, ny)) for i in range(ny): - q[:,i] = old_div(- shiftangle, (2 * np.pi)) - - + q[:, i] = old_div(-shiftangle, (2 * np.pi)) -xarr = psixy[:,0] +xarr = psixy[:, 0] xarr = old_div((xarr + 0.854856), (0.854856 + 0.0760856)) -fig=plt.figure() -plt.plot(xarr,q[:,32]) -plt.xlabel('normalized $\psi$', fontsize=25) -plt.ylabel('$q$',rotation='horizontal',fontsize=25) +fig = plt.figure() +plt.plot(xarr, q[:, 32]) +plt.xlabel("normalized $\psi$", fontsize=25) +plt.ylabel("$q$", rotation="horizontal", fontsize=25) fig.set_tight_layout(True) fig, ax1 = plt.subplots() -ax1.plot(xarr, p[:,32], 'r-', markevery=1, linewidth=3) -ax1.set_xlabel('normalized $\psi$',fontsize=25) +ax1.plot(xarr, p[:, 32], "r-", markevery=1, linewidth=3) +ax1.set_xlabel("normalized $\psi$", fontsize=25) # Make the y-axis label and tick labels match the line color. -ax1.set_ylabel('Pressure [Pa]', color='k',fontsize=25) +ax1.set_ylabel("Pressure [Pa]", color="k", fontsize=25) -#set y limit -ax1.set_ylim(0,40000,10000) +# set y limit +ax1.set_ylim(0, 40000, 10000) -#define ticks# +# define ticks# ax1.yaxis.set_ticks(np.arange(0, 40000, 10000)) -#ax1.yaxis.set_major_locator(majorLocator) -#ax1.yaxis.set_major_formatter(majorFormatter) +# ax1.yaxis.set_major_locator(majorLocator) +# ax1.yaxis.set_major_formatter(majorFormatter) ax1.yaxis.set_major_formatter(Fm) -#for the minor ticks, use no labels; default NullFormatter +# for the minor ticks, use no labels; default NullFormatter ax1.xaxis.set_minor_locator(AutoMinorLocator()) ax1.yaxis.set_minor_locator(AutoMinorLocator(10)) -#format tick labels +# format tick labels for tl in ax1.get_yticklabels(): - tl.set_color('k') + tl.set_color("k") ax2 = ax1.twinx() s2 = -jpar0 -ax2.plot(xarr, s2[:,32], 'r-',markevery=1,linewidth=3) -ax2.set_ylabel('$J_\parallel [A/m^2]$', color='k',fontsize=25) -ax2.set_ylim(0,600000) +ax2.plot(xarr, s2[:, 32], "r-", markevery=1, linewidth=3) +ax2.set_ylabel("$J_\parallel [A/m^2]$", color="k", fontsize=25) +ax2.set_ylim(0, 600000) ax2.yaxis.set_ticks(np.arange(0, 600000, 200000)) ax2.yaxis.set_major_formatter(Fm2) for tl in ax2.get_yticklabels(): - tl.set_color('k') + tl.set_color("k") fig.set_tight_layout(True) @@ -86,4 +97,4 @@ plt.show() -#plt.savefig('2d.png', transparent=True) +# plt.savefig('2d.png', transparent=True) diff --git a/examples/elm-pb/Python/analysis.py b/examples/elm-pb/Python/analysis.py index 4faf85f8ce..0e4a3acfdd 100644 --- a/examples/elm-pb/Python/analysis.py +++ b/examples/elm-pb/Python/analysis.py @@ -6,24 +6,24 @@ import pylab as plt from boutdata.collect import collect -path='./data/' -var=collect('P', path=path) +path = "./data/" +var = collect("P", path=path) -dcvar=np.mean(var, axis=3) -rmsvar=np.sqrt(np.mean(var**2,axis=3)-dcvar**2) +dcvar = np.mean(var, axis=3) +rmsvar = np.sqrt(np.mean(var**2, axis=3) - dcvar**2) plt.figure() -plt.plot(rmsvar[:,34,32]) +plt.plot(rmsvar[:, 34, 32]) plt.show(block=False) -fvar=np.fft.rfft(var,axis=3) +fvar = np.fft.rfft(var, axis=3) plt.figure() -plt.plot(abs(fvar[:,34,32,1:10])) +plt.plot(abs(fvar[:, 34, 32, 1:10])) plt.show(block=False) plt.figure() -plt.semilogy(abs(fvar[:,34,32,1:7])) +plt.semilogy(abs(fvar[:, 34, 32, 1:7])) plt.show(block=False) plt.show() diff --git a/examples/elm-pb/Python/elm_size.py b/examples/elm-pb/Python/elm_size.py index 0d5c105870..6ad2ed6a0d 100644 --- a/examples/elm-pb/Python/elm_size.py +++ b/examples/elm-pb/Python/elm_size.py @@ -4,87 +4,126 @@ from past.utils import old_div import numpy as np -def elm_size(dcp,p0,uedge,xmin=None,xmax=None,yind=None,Bbar=None): - - lis=[dcp,p0,uedge] - if np.size(lis) != 3 : + +def elm_size(dcp, p0, uedge, xmin=None, xmax=None, yind=None, Bbar=None): + lis = [dcp, p0, uedge] + if np.size(lis) != 3: print("lack of parameters") return 0 - - if xmin == None : xmin=0 - if xmax == None : xmax=327 - if yind == None : yind=63 # choose the poloidal location for 1D size - if Bbar == None : Bbar=1.992782 # the normalized magnetic field + if xmin == None: + xmin = 0 + if xmax == None: + xmax = 327 + if yind == None: + yind = 63 # choose the poloidal location for 1D size + if Bbar == None: + Bbar = 1.992782 # the normalized magnetic field - mydcp=dcp - myp0=p0 - g=uedge + mydcp = dcp + myp0 = p0 + g = uedge PI = 3.1415926 - MU0 = 4.0e-7*PI + MU0 = 4.0e-7 * PI - s=np.shape(mydcp) + s = np.shape(mydcp) - if np.ndim(mydcp) != 3 : + if np.ndim(mydcp) != 3: print("dcp should be 3D(t,x,y)") - - - nt=s[0] - nx=s[1] - ny=s[2] - - Dtheta=g['dy'] #using correct poloidal angle - psixy=g['psixy'] - R=g['Rxy'] - Bp=g['Bpxy'] - hthe=g['hthe'] - - Dpsi=np.zeros((nx,ny)) - Dpsi[0,:]=psixy[1,:]-psixy[0,:] - Dpsi[nx-1,:]=psixy[nx-1,:]-psixy[nx-2,:] - for i in range(1,nx-2): - Dpsi[i,:]=old_div((psixy[i+1,:]-psixy[i-1,:]),2) - - - Ddcp1=np.zeros(nt) - Ddcp2=np.zeros(nt) - Ddcp3=np.zeros(nt) - Tp01=0. - Tp02=0. - Tp03=0. - - for t in range(nt) : - Ddcp3[t]=2.0*PI*np.sum(mydcp[t,xmin:xmax,:]*hthe[xmin:xmax,:]*Dtheta[xmin:xmax,:]*Dpsi[xmin:xmax,:]/Bp[xmin:xmax,:]) - Ddcp2[t]=np.sum(mydcp[t,xmin:xmax,:]*hthe[xmin:xmax,:]*Dtheta[xmin:xmax,:]*Dpsi[xmin:xmax,:]/(R[xmin:xmax,:]*Bp[xmin:xmax,:])) - Ddcp1[t]=np.sum(mydcp[t,xmin:xmax,yind]*Dpsi[xmin:xmax,yind]/(R[xmin:xmax,yind]*Bp[xmin:xmax,yind])) - - - Tp03=2.0*PI*np.sum(myp0[xmin:xmax,:]*hthe[xmin:xmax,:]*Dtheta[xmin:xmax,:]*Dpsi[xmin:xmax,:]/Bp[xmin:xmax,:]) - Tp02=np.sum(myp0[xmin:xmax,:]*hthe[xmin:xmax,:]*Dtheta[xmin:xmax,:]*Dpsi[xmin:xmax,:]/(R[xmin:xmax,:]*Bp[xmin:xmax,:])) - Tp01=np.sum(myp0[xmin:xmax,yind]*Dpsi[xmin:xmax,yind]/(R[xmin:xmax,yind]*Bp[xmin:xmax,yind])) - - s1=np.zeros(nt) - s2=np.zeros(nt) - s3=np.zeros(nt) - E_loss=np.zeros(nt) - - s1=old_div(-Ddcp1,Tp01) #1D elm size - s2=old_div(-Ddcp2,Tp02) #2D elm size - s3=old_div(-Ddcp3,Tp03) #3D elm size - - E_loss=-Ddcp3*(0.5*Bbar*Bbar/MU0) #energy loss, unit J - E_total=Tp03*(0.5*Bbar*Bbar/MU0) #total energy, unit J + + nt = s[0] + nx = s[1] + ny = s[2] + + Dtheta = g["dy"] # using correct poloidal angle + psixy = g["psixy"] + R = g["Rxy"] + Bp = g["Bpxy"] + hthe = g["hthe"] + + Dpsi = np.zeros((nx, ny)) + Dpsi[0, :] = psixy[1, :] - psixy[0, :] + Dpsi[nx - 1, :] = psixy[nx - 1, :] - psixy[nx - 2, :] + for i in range(1, nx - 2): + Dpsi[i, :] = old_div((psixy[i + 1, :] - psixy[i - 1, :]), 2) + + Ddcp1 = np.zeros(nt) + Ddcp2 = np.zeros(nt) + Ddcp3 = np.zeros(nt) + Tp01 = 0.0 + Tp02 = 0.0 + Tp03 = 0.0 + + for t in range(nt): + Ddcp3[t] = ( + 2.0 + * PI + * np.sum( + mydcp[t, xmin:xmax, :] + * hthe[xmin:xmax, :] + * Dtheta[xmin:xmax, :] + * Dpsi[xmin:xmax, :] + / Bp[xmin:xmax, :] + ) + ) + Ddcp2[t] = np.sum( + mydcp[t, xmin:xmax, :] + * hthe[xmin:xmax, :] + * Dtheta[xmin:xmax, :] + * Dpsi[xmin:xmax, :] + / (R[xmin:xmax, :] * Bp[xmin:xmax, :]) + ) + Ddcp1[t] = np.sum( + mydcp[t, xmin:xmax, yind] + * Dpsi[xmin:xmax, yind] + / (R[xmin:xmax, yind] * Bp[xmin:xmax, yind]) + ) + + Tp03 = ( + 2.0 + * PI + * np.sum( + myp0[xmin:xmax, :] + * hthe[xmin:xmax, :] + * Dtheta[xmin:xmax, :] + * Dpsi[xmin:xmax, :] + / Bp[xmin:xmax, :] + ) + ) + Tp02 = np.sum( + myp0[xmin:xmax, :] + * hthe[xmin:xmax, :] + * Dtheta[xmin:xmax, :] + * Dpsi[xmin:xmax, :] + / (R[xmin:xmax, :] * Bp[xmin:xmax, :]) + ) + Tp01 = np.sum( + myp0[xmin:xmax, yind] + * Dpsi[xmin:xmax, yind] + / (R[xmin:xmax, yind] * Bp[xmin:xmax, yind]) + ) + + s1 = np.zeros(nt) + s2 = np.zeros(nt) + s3 = np.zeros(nt) + E_loss = np.zeros(nt) + + s1 = old_div(-Ddcp1, Tp01) # 1D elm size + s2 = old_div(-Ddcp2, Tp02) # 2D elm size + s3 = old_div(-Ddcp3, Tp03) # 3D elm size + + E_loss = -Ddcp3 * (0.5 * Bbar * Bbar / MU0) # energy loss, unit J + E_total = Tp03 * (0.5 * Bbar * Bbar / MU0) # total energy, unit J class ELM: pass - elmsize=ELM() - elmsize.s1=s1 - elmsize.s2=s2 - elmsize.s3=s3 - elmsize.E_loss=E_loss - elmsize.E_total=E_total + + elmsize = ELM() + elmsize.s1 = s1 + elmsize.s2 = s2 + elmsize.s3 = s3 + elmsize.E_loss = E_loss + elmsize.E_total = E_total return elmsize - - diff --git a/examples/elm-pb/Python/fftall.py b/examples/elm-pb/Python/fftall.py index e0a4385764..c6d5b87ded 100644 --- a/examples/elm-pb/Python/fftall.py +++ b/examples/elm-pb/Python/fftall.py @@ -2,8 +2,7 @@ from numpy import * from scipy.io import readsav -print('Calculating P..') -a=transpose(readsav('phi.idl.dat')['phi']) -fa=fft.fft(a,axis=2) -save('fp',fa) - +print("Calculating P..") +a = transpose(readsav("phi.idl.dat")["phi"]) +fa = fft.fft(a, axis=2) +save("fp", fa) diff --git a/examples/elm-pb/Python/fftall2.py b/examples/elm-pb/Python/fftall2.py index d864f356d0..64b08c437a 100644 --- a/examples/elm-pb/Python/fftall2.py +++ b/examples/elm-pb/Python/fftall2.py @@ -2,9 +2,9 @@ from numpy import * from boutdata.collect import collect -path='./data/' -data=collect('P',path=path) +path = "./data/" +data = collect("P", path=path) -print('Saving P..') -fa=fft.fft(data,axis=3) -save('fp',rollaxis(fa,0,4)) +print("Saving P..") +fa = fft.fft(data, axis=3) +save("fp", rollaxis(fa, 0, 4)) diff --git a/examples/elm-pb/Python/grate.py b/examples/elm-pb/Python/grate.py index 003f38ff8e..65d6da6449 100644 --- a/examples/elm-pb/Python/grate.py +++ b/examples/elm-pb/Python/grate.py @@ -8,12 +8,11 @@ from boututils.moment_xyzt import moment_xyzt +path = "./data/" -path='./data/' +p = collect("P", path=path) +rmsp_f = moment_xyzt(p[:, 34:35, 32:33, :], "RMS").rms -p=collect('P',path=path) -rmsp_f=moment_xyzt(p[:,34:35,32:33,:], 'RMS').rms - -print(np.gradient(np.log(rmsp_f[:,0,0]))[-1]) +print(np.gradient(np.log(rmsp_f[:, 0, 0]))[-1]) diff --git a/examples/elm-pb/Python/grate2.py b/examples/elm-pb/Python/grate2.py index f8172bcf08..af0768803a 100644 --- a/examples/elm-pb/Python/grate2.py +++ b/examples/elm-pb/Python/grate2.py @@ -2,6 +2,7 @@ from __future__ import division from builtins import range from past.utils import old_div + ### # computes average growth rate for all points at the final timestep # computes average growth rate for points in the mead plane at the final timestep @@ -10,40 +11,44 @@ from boutdata.collect import collect from boututils.moment_xyzt import moment_xyzt -path='./data/' +path = "./data/" + +p = collect("P", path=path) -p=collect('P',path=path) +nmpy = old_div(p.shape[2], 2) # define mead plane -nmpy=old_div(p.shape[2],2) # define mead plane +ik = 50 # disregard the first ik timesteps -ik = 50 # disregard the first ik timesteps def gr(p): - rmsp_f=moment_xyzt(p, 'RMS').rms + rmsp_f = moment_xyzt(p, "RMS").rms - ni=np.shape(rmsp_f)[1] - nj=np.shape(rmsp_f)[2] + ni = np.shape(rmsp_f)[1] + nj = np.shape(rmsp_f)[2] - growth=np.zeros((ni,nj)) + growth = np.zeros((ni, nj)) - for i in range(ni): - for j in range(nj): - growth[i,j]=np.gradient(np.log(rmsp_f[ik::,i,j]))[-1] + for i in range(ni): + for j in range(nj): + growth[i, j] = np.gradient(np.log(rmsp_f[ik::, i, j]))[-1] - return growth + return growth -growth=gr(p) +growth = gr(p) -d=np.ma.masked_array(growth,np.isnan(growth)) +d = np.ma.masked_array(growth, np.isnan(growth)) # masked arrays # http://stackoverflow.com/questions/5480694/numpy-calculate-averages-with-nans-removed -print('Total mean value = ', np.mean(np.ma.masked_array(d,np.isinf(d)))) -mm=np.ma.masked_array(growth[:,nmpy],np.isnan(growth[:,nmpy])) -if np.isinf(np.mean(mm)) : - print('There is an Inf value in the mead plane') - print('Mean value of floating numbers in mead plane is = ', np.mean(np.ma.masked_array(mm,np.isinf(mm)))) +print("Total mean value = ", np.mean(np.ma.masked_array(d, np.isinf(d)))) +mm = np.ma.masked_array(growth[:, nmpy], np.isnan(growth[:, nmpy])) +if np.isinf(np.mean(mm)): + print("There is an Inf value in the mead plane") + print( + "Mean value of floating numbers in mead plane is = ", + np.mean(np.ma.masked_array(mm, np.isinf(mm))), + ) else: - print('Mean value in mead plane= ', np.mean(mm)) + print("Mean value in mead plane= ", np.mean(mm)) diff --git a/examples/elm-pb/Python/plotcollapse.py b/examples/elm-pb/Python/plotcollapse.py index ee64ca381f..f68e7b06aa 100755 --- a/examples/elm-pb/Python/plotcollapse.py +++ b/examples/elm-pb/Python/plotcollapse.py @@ -10,49 +10,52 @@ import os from pathlib import Path -#Dynamic matplotlib settings +# Dynamic matplotlib settings from matplotlib import rcParams -rcParams['font.size'] = 20. -rcParams['legend.fontsize'] = 'small' -rcParams['lines.linewidth'] = 2 -if not os.path.exists('image'): - os.makedirs('image') +rcParams["font.size"] = 20.0 +rcParams["legend.fontsize"] = "small" +rcParams["lines.linewidth"] = 2 + +if not os.path.exists("image"): + os.makedirs("image") filename = Path(__file__).with_name("cbm18_dens8.grid_nx68ny64.nc") with DataFile(str(filename)) as f: g = {v: f.read(v) for v in f.keys()} -psi = old_div((g['psixy'][:, 32] - g['psi_axis']), (g['psi_bndry'] - g['psi_axis'])) +psi = old_div((g["psixy"][:, 32] - g["psi_axis"]), (g["psi_bndry"] - g["psi_axis"])) -path = './data' +path = "./data" plt.figure() -p0=collect('P0', path=path) +p0 = collect("P0", path=path) -p=collect('P', path=path) -res = moment_xyzt(p,'RMS','DC') +p = collect("P", path=path) +res = moment_xyzt(p, "RMS", "DC") rmsp = res.rms dcp = res.dc nt = dcp.shape[0] -plt.plot(psi, p0[:, 32], 'k--', label='t=0') -plt.plot(psi, p0[:, 32] + dcp[nt//4, :, 32], 'r-', label='t='+np.str(nt//4)) -plt.plot(psi, p0[:, 32] + dcp[nt//2, :, 32], 'g-', label='t='+np.str(nt//2)) -plt.plot(psi, p0[:, 32] + dcp[3*nt//4, :, 32], 'b-', label='t='+np.str(3*nt//4)) -plt.plot(psi, p0[:, 32] + dcp[-1, :, 32], 'c-', label='t='+np.str(nt)) +plt.plot(psi, p0[:, 32], "k--", label="t=0") +plt.plot(psi, p0[:, 32] + dcp[nt // 4, :, 32], "r-", label="t=" + np.str(nt // 4)) +plt.plot(psi, p0[:, 32] + dcp[nt // 2, :, 32], "g-", label="t=" + np.str(nt // 2)) +plt.plot( + psi, p0[:, 32] + dcp[3 * nt // 4, :, 32], "b-", label="t=" + np.str(3 * nt // 4) +) +plt.plot(psi, p0[:, 32] + dcp[-1, :, 32], "c-", label="t=" + np.str(nt)) plt.legend() -#plt.xlim(0.6, 1.0) -plt.xlabel(r'Normalized poloidal flux ($\psi$)') -plt.ylabel(r'$\langle p\rangle_\xi$') -plt.title(r'Pressure') +# plt.xlim(0.6, 1.0) +plt.xlabel(r"Normalized poloidal flux ($\psi$)") +plt.ylabel(r"$\langle p\rangle_\xi$") +plt.title(r"Pressure") xmin, xmax = plt.xlim() ymin, ymax = plt.ylim() -#plt.savefig('image/plotcollapse.png', bbox_inches='tight') -#plt.savefig('image/plotcollapse.eps', bbox_inches='tight') +# plt.savefig('image/plotcollapse.png', bbox_inches='tight') +# plt.savefig('image/plotcollapse.eps', bbox_inches='tight') plt.tight_layout() diff --git a/examples/elm-pb/Python/plotmode.py b/examples/elm-pb/Python/plotmode.py index d89fdaf940..9325ee8764 100644 --- a/examples/elm-pb/Python/plotmode.py +++ b/examples/elm-pb/Python/plotmode.py @@ -4,60 +4,57 @@ from builtins import range from past.utils import old_div -from numpy import *; -#from scipy.io import readsav; -import matplotlib.pyplot as plt; +from numpy import * + +# from scipy.io import readsav; +import matplotlib.pyplot as plt # Dynamic matplotlib settings -from matplotlib import rcParams; -rcParams['font.size'] = 20; -rcParams['legend.fontsize'] = 'small'; -rcParams['legend.labelspacing'] = 0.1; -rcParams['lines.linewidth'] = 2; -rcParams['savefig.bbox'] = 'tight'; +from matplotlib import rcParams +rcParams["font.size"] = 20 +rcParams["legend.fontsize"] = "small" +rcParams["legend.labelspacing"] = 0.1 +rcParams["lines.linewidth"] = 2 +rcParams["savefig.bbox"] = "tight" # Create image directory if not exists -import os; -if not os.path.exists('image'): - os.makedirs('image'); - -#fphi = transpose(readsav('fphi.idl.dat')['fphi'])[:,:,:,]; -fphi = load('fp.npy') - -plt.figure(); -for i in range(1, 9): - print("Growth rate for mode number", i) - print(gradient(log(abs(fphi[34, 32, i, :])))) - plt.semilogy(((abs(fphi[34, 32, i, :]))), label = 'n=' + str(i * 5)); +import os -plt.legend(loc=2); -plt.xlabel('Time'); -plt.savefig('image/plotmode.png'); -plt.savefig('image/plotmode.eps'); +if not os.path.exists("image"): + os.makedirs("image") +# fphi = transpose(readsav('fphi.idl.dat')['fphi'])[:,:,:,]; +fphi = load("fp.npy") -plt.show(block=False); -plt.figure(); +plt.figure() for i in range(1, 9): - plt.plot(abs(fphi[:, 32, i, -1]), label = 'n=' + str(i * 5)); - -plt.legend(); -plt.xlabel('X index'); - -plt.savefig('image/plotmodeamp.png'); -plt.savefig('image/plotmodeamp.eps'); - -plt.show(block=False); - -plt.figure(); + print("Growth rate for mode number", i) + print(gradient(log(abs(fphi[34, 32, i, :])))) + plt.semilogy((abs(fphi[34, 32, i, :])), label="n=" + str(i * 5)) + +plt.legend(loc=2) +plt.xlabel("Time") +plt.savefig("image/plotmode.png") +plt.savefig("image/plotmode.eps") +plt.show(block=False) +plt.figure() for i in range(1, 9): - plt.plot(old_div(abs(fphi[:, 32, i, -1]),abs(fphi[:, 32, i, -1]).max()), label = 'n=' + str(i * 5)); - -plt.legend(); -plt.xlabel('X index'); - -plt.savefig('image/plotmodenorm.png'); -plt.savefig('image/plotmodenorm.eps'); - -plt.show(); - + plt.plot(abs(fphi[:, 32, i, -1]), label="n=" + str(i * 5)) + +plt.legend() +plt.xlabel("X index") +plt.savefig("image/plotmodeamp.png") +plt.savefig("image/plotmodeamp.eps") +plt.show(block=False) +plt.figure() +for i in range(1, 9): + plt.plot( + old_div(abs(fphi[:, 32, i, -1]), abs(fphi[:, 32, i, -1]).max()), + label="n=" + str(i * 5), + ) + +plt.legend() +plt.xlabel("X index") +plt.savefig("image/plotmodenorm.png") +plt.savefig("image/plotmodenorm.eps") +plt.show() diff --git a/examples/elm-pb/Python/plotmode2.py b/examples/elm-pb/Python/plotmode2.py index d0c63f32d9..c298a3a5ef 100644 --- a/examples/elm-pb/Python/plotmode2.py +++ b/examples/elm-pb/Python/plotmode2.py @@ -4,63 +4,61 @@ from builtins import range from past.utils import old_div -from numpy import *; -#from scipy.io import readsav; -import matplotlib.pyplot as plt; +from numpy import * + +# from scipy.io import readsav; +import matplotlib.pyplot as plt from boutdata.collect import collect # Dynamic matplotlib settings -from matplotlib import rcParams; -rcParams['font.size'] = 20; -rcParams['legend.fontsize'] = 'small'; -rcParams['legend.labelspacing'] = 0.1; -rcParams['lines.linewidth'] = 2; -rcParams['savefig.bbox'] = 'tight'; +from matplotlib import rcParams +rcParams["font.size"] = 20 +rcParams["legend.fontsize"] = "small" +rcParams["legend.labelspacing"] = 0.1 +rcParams["lines.linewidth"] = 2 +rcParams["savefig.bbox"] = "tight" # Create image directory if not exists -import os; -if not os.path.exists('image'): - os.makedirs('image'); +import os + +if not os.path.exists("image"): + os.makedirs("image") -path='./data/' -data=collect('P',path=path) +path = "./data/" +data = collect("P", path=path) -#fphi = transpose(readsav('fphi.idl.dat')['fphi'])[:,:,:,]; +# fphi = transpose(readsav('fphi.idl.dat')['fphi'])[:,:,:,]; fphi = fft.fft(data, axis=3) -plt.figure(); +plt.figure() for i in range(1, 9): - print("Growth rate for mode number", i) - print(gradient(log(abs(fphi[:,34, 32, i])))) - plt.semilogy(((abs(fphi[:,34, 32, i]))), label = 'n=' + str(i * 5)); - -plt.legend(loc=2); -plt.xlabel('Time'); -plt.savefig('image/plotmode.png'); -plt.savefig('image/plotmode.eps'); - - -plt.show(block=False); -plt.figure(); + print("Growth rate for mode number", i) + print(gradient(log(abs(fphi[:, 34, 32, i])))) + plt.semilogy((abs(fphi[:, 34, 32, i])), label="n=" + str(i * 5)) + +plt.legend(loc=2) +plt.xlabel("Time") +plt.savefig("image/plotmode.png") +plt.savefig("image/plotmode.eps") +plt.show(block=False) +plt.figure() for i in range(1, 9): - plt.plot(abs(fphi[-1, :, 32, i]), label = 'n=' + str(i * 5)); - -plt.legend(); -plt.xlabel('X index'); - -plt.savefig('image/plotmodeamp.png'); -plt.savefig('image/plotmodeamp.eps'); - -plt.show(block=False); - -plt.figure(); + plt.plot(abs(fphi[-1, :, 32, i]), label="n=" + str(i * 5)) + +plt.legend() +plt.xlabel("X index") +plt.savefig("image/plotmodeamp.png") +plt.savefig("image/plotmodeamp.eps") +plt.show(block=False) +plt.figure() for i in range(1, 9): - plt.plot(old_div(abs(fphi[-1, :, 32, i]),abs(fphi[-1, :, 32, i]).max()), label = 'n=' + str(i * 5)); - -plt.legend(); -plt.xlabel('X index'); - -plt.savefig('image/plotmodenorm.png'); -plt.savefig('image/plotmodenorm.eps'); - -plt.show(); + plt.plot( + old_div(abs(fphi[-1, :, 32, i]), abs(fphi[-1, :, 32, i]).max()), + label="n=" + str(i * 5), + ) + +plt.legend() +plt.xlabel("X index") +plt.savefig("image/plotmodenorm.png") +plt.savefig("image/plotmodenorm.eps") +plt.show() diff --git a/examples/elm-pb/Python/plotphase.py b/examples/elm-pb/Python/plotphase.py index 9225e498ae..10f4279cf4 100755 --- a/examples/elm-pb/Python/plotphase.py +++ b/examples/elm-pb/Python/plotphase.py @@ -4,34 +4,33 @@ from numpy import save, load, angle import matplotlib.pyplot as plt -fphi = load('fphi.npy') +fphi = load("fphi.npy") -fte = load('fte.npy') -phase_te = angle(old_div(fphi,fte)) -save('phase_te', phase_te) +fte = load("fte.npy") +phase_te = angle(old_div(fphi, fte)) +save("phase_te", phase_te) plt.figure() -plt.plot(mean(mean(phase_te[:,:,3,:],axis=1),axis=1)) -plt.title('Te') -plt.savefig('image/phase_te.png') -plt.savefig('image/phase_te.eps') +plt.plot(mean(mean(phase_te[:, :, 3, :], axis=1), axis=1)) +plt.title("Te") +plt.savefig("image/phase_te.png") +plt.savefig("image/phase_te.eps") -fti = load('fti.npy') -phase_ti = angle(old_div(fphi,fti)) -save('phase_ti', phase_ti) +fti = load("fti.npy") +phase_ti = angle(old_div(fphi, fti)) +save("phase_ti", phase_ti) plt.figure() -plt.plot(mean(mean(phase_ti[:,:,3,:],axis=1),axis=1)) -plt.title('ti') -plt.savefig('image/phase_ti.png') -plt.savefig('image/phase_ti.eps') +plt.plot(mean(mean(phase_ti[:, :, 3, :], axis=1), axis=1)) +plt.title("ti") +plt.savefig("image/phase_ti.png") +plt.savefig("image/phase_ti.eps") -fni = load('fni.npy') -phase_ni = angle(old_div(fphi,fni)) -save('phase_ni', phase_ni) +fni = load("fni.npy") +phase_ni = angle(old_div(fphi, fni)) +save("phase_ni", phase_ni) plt.figure() -plt.plot(mean(mean(phase_ni[:,:,3,:],axis=1),axis=1)) -plt.title('ni') -plt.savefig('image/phase_ni.png') -plt.savefig('image/phase_ni.eps') +plt.plot(mean(mean(phase_ni[:, :, 3, :], axis=1), axis=1)) +plt.title("ni") +plt.savefig("image/phase_ni.png") +plt.savefig("image/phase_ni.eps") plt.show() - diff --git a/examples/elm-pb/Python/polslice.py b/examples/elm-pb/Python/polslice.py index fe78495b5a..6a1179b2ae 100644 --- a/examples/elm-pb/Python/polslice.py +++ b/examples/elm-pb/Python/polslice.py @@ -11,31 +11,33 @@ # Specify parameters -path='./data/' +path = "./data/" -variable="P" +variable = "P" p = collect(variable, path=path) -period=15 +period = 15 -grid='../cbm18_dens8.grid_nx68ny64.nc' +grid = "../cbm18_dens8.grid_nx68ny64.nc" ######################################################## # Call plotpolslice once to get extended poloidal grid -r,z,fun=plotpolslice(p[0,:,:,:],grid,period=period,rz=1) +r, z, fun = plotpolslice(p[0, :, :, :], grid, period=period, rz=1) -nx=r.shape[0] # number of points in r -ny=r.shape[1] # number of points in z -nt=p.shape[0] # time intervals +nx = r.shape[0] # number of points in r +ny = r.shape[1] # number of points in z +nt = p.shape[0] # time intervals -fm=np.zeros((nt,nx,ny)) # array to store the time sequence of the poloidal cross section +fm = np.zeros( + (nt, nx, ny) +) # array to store the time sequence of the poloidal cross section -#Compute all time frames +# Compute all time frames for k in range(nt): - fm[k,:,:]=plotpolslice(p[k,:,:,:],grid,period=period,rz=0) + fm[k, :, :] = plotpolslice(p[k, :, :, :], grid, period=period, rz=0) -np.savez('pslice',fm=fm, z=z, r=r) +np.savez("pslice", fm=fm, z=z, r=r) diff --git a/examples/elm-pb/Python/post.py b/examples/elm-pb/Python/post.py index 9bb67347f0..cdaefa98cf 100644 --- a/examples/elm-pb/Python/post.py +++ b/examples/elm-pb/Python/post.py @@ -15,12 +15,12 @@ from mayavi import mlab -path0="./data0/" -path1="./data/" +path0 = "./data0/" +path1 = "./data/" -period=15 +period = 15 -gfile='./cbm18_dens8.grid_nx68ny64.nc' +gfile = "./cbm18_dens8.grid_nx68ny64.nc" with DataFile(gfile) as f: @@ -28,71 +28,115 @@ Dphi0 = collect("Dphi0", path=path0) -phi0 = collect("phi0", path=path1) # needs diamagnetic effects +phi0 = collect("phi0", path=path1) # needs diamagnetic effects # -psixy=g.get('psixy') -PSI_AXIS=g.get('psi_axis') -PSI_BNDRY=g.get('psi_bndry') +psixy = g.get("psixy") +PSI_AXIS = g.get("psi_axis") +PSI_BNDRY = g.get("psi_bndry") # -psix=old_div((psixy[:,32]-PSI_AXIS),(PSI_BNDRY-PSI_AXIS)) -Epsi=-deriv(phi0[:,32],psix) +psix = old_div((psixy[:, 32] - PSI_AXIS), (PSI_BNDRY - PSI_AXIS)) +Epsi = -deriv(phi0[:, 32], psix) # # -fig=figure() -plot(psix,-Dphi0[:,32], 'r', linewidth=5) -plot(psix,Epsi,'k',linewidth=5) -annotate('w/o flow', xy=(.3, .7), xycoords='axes fraction',horizontalalignment='center', verticalalignment='center', size=30) -annotate('w/ flow', xy=(.7, .4), xycoords='axes fraction',horizontalalignment='center', verticalalignment='center', color='r', size=30) -xlabel('Radial $\psi$',fontsize=25) -ylabel('$\Omega(\psi)/\omega_A$',fontsize=25) -ylim([-.05,0]) -xlim([0.4,1.2]) +fig = figure() +plot(psix, -Dphi0[:, 32], "r", linewidth=5) +plot(psix, Epsi, "k", linewidth=5) +annotate( + "w/o flow", + xy=(0.3, 0.7), + xycoords="axes fraction", + horizontalalignment="center", + verticalalignment="center", + size=30, +) +annotate( + "w/ flow", + xy=(0.7, 0.4), + xycoords="axes fraction", + horizontalalignment="center", + verticalalignment="center", + color="r", + size=30, +) +xlabel("Radial $\psi$", fontsize=25) +ylabel("$\Omega(\psi)/\omega_A$", fontsize=25) +ylim([-0.05, 0]) +xlim([0.4, 1.2]) fig.set_tight_layout(True) show(block=False) p_f0 = collect("P", path=path0) p_f = collect("P", path=path1) # -rmsp_f0=moment_xyzt(p_f0, 'RMS').rms -rmsp_f=moment_xyzt(p_f, 'RMS').rms +rmsp_f0 = moment_xyzt(p_f0, "RMS").rms +rmsp_f = moment_xyzt(p_f, "RMS").rms # -fig=figure(figsize=(10, 8)) -plot(np.gradient(np.log(rmsp_f0[:,34,32])), color='k',linewidth=3) -plot(np.gradient(np.log(rmsp_f[:,34,32])),color='red',linewidth=3) - -ylabel('$\gamma / \omega_A$',fontsize=25) -xlabel('Time$(\\tau_A)$',fontsize=25) -annotate('w/o flow', xy=(.5, .7), xycoords='axes fraction',horizontalalignment='center', verticalalignment='center', size=30) -annotate('w/ flow', xy=(.5, .4), xycoords='axes fraction',horizontalalignment='center', verticalalignment='center', color='r', size=30) -ylim([0,0.5]) -xlim([0,100]) +fig = figure(figsize=(10, 8)) +plot(np.gradient(np.log(rmsp_f0[:, 34, 32])), color="k", linewidth=3) +plot(np.gradient(np.log(rmsp_f[:, 34, 32])), color="red", linewidth=3) + +ylabel("$\gamma / \omega_A$", fontsize=25) +xlabel("Time$(\\tau_A)$", fontsize=25) +annotate( + "w/o flow", + xy=(0.5, 0.7), + xycoords="axes fraction", + horizontalalignment="center", + verticalalignment="center", + size=30, +) +annotate( + "w/ flow", + xy=(0.5, 0.4), + xycoords="axes fraction", + horizontalalignment="center", + verticalalignment="center", + color="r", + size=30, +) +ylim([0, 0.5]) +xlim([0, 100]) fig.set_tight_layout(True) show(block=False) - -plotpolslice(p_f0[50,:,:,:],gfile,period=period, fig=1) -mlab.text(.01,.99,"w/o flow") - -plotpolslice(p_f[50,:,:,:],gfile,period=period, fig=1) -mlab.text(.01,.99,"w/ flow") - -fig=figure() -mode_structure(p_f0[50,:,:,:], g, period=period) -plot([40,40],[0,.014],'k--',linewidth=5) -annotate('w/o flow', xy=(.3, .7), xycoords='axes fraction',horizontalalignment='center', verticalalignment='center', size=30) -ylim([0,0.014]) -xlim([0,80]) +plotpolslice(p_f0[50, :, :, :], gfile, period=period, fig=1) +mlab.text(0.01, 0.99, "w/o flow") + +plotpolslice(p_f[50, :, :, :], gfile, period=period, fig=1) +mlab.text(0.01, 0.99, "w/ flow") + +fig = figure() +mode_structure(p_f0[50, :, :, :], g, period=period) +plot([40, 40], [0, 0.014], "k--", linewidth=5) +annotate( + "w/o flow", + xy=(0.3, 0.7), + xycoords="axes fraction", + horizontalalignment="center", + verticalalignment="center", + size=30, +) +ylim([0, 0.014]) +xlim([0, 80]) fig.set_tight_layout(True) show(block=False) figure() -mode_structure(p_f[50,:,:,:], g, period=period) -plot([40,40],[0,.014],'k--',linewidth=5) -annotate('w/ flow', xy=(.3, .7), xycoords='axes fraction',horizontalalignment='center', verticalalignment='center', color='k', size=30) -ylim([0,0.0001]) -xlim([0,80]) +mode_structure(p_f[50, :, :, :], g, period=period) +plot([40, 40], [0, 0.014], "k--", linewidth=5) +annotate( + "w/ flow", + xy=(0.3, 0.7), + xycoords="axes fraction", + horizontalalignment="center", + verticalalignment="center", + color="k", + size=30, +) +ylim([0, 0.0001]) +xlim([0, 80]) show(block=False) show() diff --git a/examples/elm-pb/Python/read_elmsize.py b/examples/elm-pb/Python/read_elmsize.py index fcd3d24e53..340f57ce7b 100644 --- a/examples/elm-pb/Python/read_elmsize.py +++ b/examples/elm-pb/Python/read_elmsize.py @@ -4,12 +4,12 @@ from pylab import save, figure, plot, title, xlabel, ylabel, show, tight_layout from elm_size import elm_size -path='./data' +path = "./data" -t_array=collect('t_array', path=path) -save('t_array.dat', t_array) -p0=collect('P0', path=path) -save('p0.dat', p0) +t_array = collect("t_array", path=path) +save("t_array.dat", t_array) +p0 = collect("P0", path=path) +save("p0.dat", p0) # n0=collect('n0', path=path) @@ -22,27 +22,27 @@ with DataFile("./cbm18_dens8.grid_nx68ny64.nc") as f: gfile = {v: f.read(v) for v in f.keys()} -p=collect('P', path=path) -save('p.dat', p) -res=moment_xyzt(p,'RMS','DC') -rmsp=res.rms -dcp=res.dc -save('rmsp.dat', rmsp) -save('dcp.dat', dcp) -elmsp=elm_size(dcp,p0,gfile,yind=32,Bbar=gfile['bmag']) -save('elmsp.dat', elmsp) +p = collect("P", path=path) +save("p.dat", p) +res = moment_xyzt(p, "RMS", "DC") +rmsp = res.rms +dcp = res.dc +save("rmsp.dat", rmsp) +save("dcp.dat", dcp) +elmsp = elm_size(dcp, p0, gfile, yind=32, Bbar=gfile["bmag"]) +save("elmsp.dat", elmsp) figure(0) -plot(t_array,elmsp.s2, 'k-') -xlabel('t/Ta') -ylabel('Elm size') -title('Elm size, P') +plot(t_array, elmsp.s2, "k-") +xlabel("t/Ta") +ylabel("Elm size") +title("Elm size, P") tight_layout() show() -phi=collect('phi', path=path ) -save('phi.dat', phi) -res=moment_xyzt( phi, 'DC', 'RMS') -save('dcphi.dat',res.dc) -save('rmsphi.dat', res.rms) +phi = collect("phi", path=path) +save("phi.dat", phi) +res = moment_xyzt(phi, "DC", "RMS") +save("dcphi.dat", res.dc) +save("rmsphi.dat", res.rms) diff --git a/examples/elm-pb/Python/showpolslice.py b/examples/elm-pb/Python/showpolslice.py index d292575e35..072c53dcd6 100644 --- a/examples/elm-pb/Python/showpolslice.py +++ b/examples/elm-pb/Python/showpolslice.py @@ -6,10 +6,12 @@ import numpy as np from tvtk.tools import visual + try: from enthought.mayavi import mlab except ImportError: - try: from mayavi import mlab + try: + from mayavi import mlab except ImportError: print("No mlab available") @@ -17,40 +19,42 @@ ########################### # Read polslice array -npzfile=np.load('pslice.npz') -r=npzfile['r'] -z=npzfile['z'] -fm=npzfile['fm'] +npzfile = np.load("pslice.npz") +r = npzfile["r"] +z = npzfile["z"] +fm = npzfile["fm"] ######################################################## # Set up the window -f = mlab.figure(size=(800,600)) +f = mlab.figure(size=(800, 600)) # Tell visual to use this as the viewer. visual.set_viewer(f) ######################################################## # Do the appropriate graph -#s = mlab.contour_surf(r,z,fun, contours=30, line_width=.5, transparent=True) -#s=mlab.surf(r,z,fun, colormap='Spectral') -s = mlab.mesh(r,z,fm[0,:,:], scalars=fm[0,:,:], colormap='PuOr')#, wrap_scale='true')#, representation='wireframe') -s.enable_contours=True -s.contour.filled_contours=True +# s = mlab.contour_surf(r,z,fun, contours=30, line_width=.5, transparent=True) +# s=mlab.surf(r,z,fun, colormap='Spectral') +s = mlab.mesh( + r, z, fm[0, :, :], scalars=fm[0, :, :], colormap="PuOr" +) # , wrap_scale='true')#, representation='wireframe') +s.enable_contours = True +s.contour.filled_contours = True # Define perspective and optional attributes. You can also implement from the window afterwards -mlab.view(0,0) -#mlab.view(-94.159958841373324, +mlab.view(0, 0) +# mlab.view(-94.159958841373324, # 53.777002382688906, # 8.2311808018087582) mlab.draw(f) mlab.colorbar(orientation="vertical") -#mlab.axes() -#mlab.outline() +# mlab.axes() +# mlab.outline() ######################################################## # mlab animation -anim(s,fm, save=True) +anim(s, fm, save=True) diff --git a/examples/elm-pb/Python/sprofiles.py b/examples/elm-pb/Python/sprofiles.py index 9d8eea48de..244599f8af 100644 --- a/examples/elm-pb/Python/sprofiles.py +++ b/examples/elm-pb/Python/sprofiles.py @@ -16,45 +16,46 @@ with DataFile(gfile) as f: g = {v: f.read(v) for v in f.keys()} -var=collect("P", path=path) +var = collect("P", path=path) -sol=surface_average(var, g) -#sol=np.mean(var,axis=3) +sol = surface_average(var, g) +# sol=np.mean(var,axis=3) -p0av=collect("P0", path=path) +p0av = collect("P0", path=path) -q=np.zeros(sol.shape) +q = np.zeros(sol.shape) for i in range(sol.shape[1]): - q[:,i]=sol[:,i]+p0av[:,0] + q[:, i] = sol[:, i] + p0av[:, 0] -psixy=g.get('psixy') -psi0=g.get('psi_axis') -psix=g.get('psi_bndry') +psixy = g.get("psixy") +psi0 = g.get("psi_axis") +psix = g.get("psi_bndry") -xarr = psixy[:,0] -xarr = old_div((xarr - psi0), (-psi0 + psix)) #for this grid +xarr = psixy[:, 0] +xarr = old_div((xarr - psi0), (-psi0 + psix)) # for this grid -fig=figure() +fig = figure() -nt=q.shape[1] +nt = q.shape[1] -plot(xarr, p0av,'k',label='t=0') -plot(xarr,q[:,nt/4],'r',label='t='+np.str(nt/4)) -plot(xarr,q[:,nt/2],'b',label='t='+np.str(nt/2)) -plot(xarr,q[:,3*nt/4],'g',label='t='+np.str(3*nt/4)) -plot(xarr, q[:,-1],'k',label='t='+np.str(nt)) +plot(xarr, p0av, "k", label="t=0") +plot(xarr, q[:, nt / 4], "r", label="t=" + np.str(nt / 4)) +plot(xarr, q[:, nt / 2], "b", label="t=" + np.str(nt / 2)) +plot(xarr, q[:, 3 * nt / 4], "g", label="t=" + np.str(3 * nt / 4)) +plot(xarr, q[:, -1], "k", label="t=" + np.str(nt)) from collections import OrderedDict + handles, labels = gca().get_legend_handles_labels() by_label = OrderedDict(list(zip(labels, handles))) legend(list(by_label.values()), list(by_label.keys())) -xlabel(r"$\psi$",fontsize=25) -ylabel(r"$2 \mu_0 / B^2$",fontsize=25) +xlabel(r"$\psi$", fontsize=25) +ylabel(r"$2 \mu_0 / B^2$", fontsize=25) fig.set_tight_layout(True) diff --git a/examples/elm-pb/elm_pb.cxx b/examples/elm-pb/elm_pb.cxx index fe18a756ae..dc9a6bb394 100644 --- a/examples/elm-pb/elm_pb.cxx +++ b/examples/elm-pb/elm_pb.cxx @@ -13,7 +13,6 @@ #include #include #include -#include #include #include @@ -1357,7 +1356,8 @@ class ELMpb : public PhysicsModel { // Only update if simulation time has advanced // Uses an exponential decay of the weighting of the value in the boundary // so that the solution is well behaved for arbitrary steps - BoutReal const weight = exp(-(t - phi_boundary_last_update) / phi_boundary_timescale); + const BoutReal weight = + exp(-(t - phi_boundary_last_update) / phi_boundary_timescale); phi_boundary_last_update = t; if (mesh->firstX()) { @@ -1366,7 +1366,7 @@ class ELMpb : public PhysicsModel { // Calculate a single phi boundary value for all Y slices BoutReal philocal = 0.0; for (int j = mesh->ystart; j <= mesh->yend; j++) { - for (int k = 0; k < mesh->LocalNz; k++) { + for (int k = mesh->zstart; k <= mesh->zend; k++) { philocal += phi(mesh->xstart, j, k); } } @@ -1379,21 +1379,21 @@ class ELMpb : public PhysicsModel { for (int j = mesh->ystart; j <= mesh->yend; j++) { if (!phi_core_averagey) { phivalue = 0.0; // Calculate phi boundary for each Y index separately - for (int k = 0; k < mesh->LocalNz; k++) { + for (int k = mesh->zstart; k <= mesh->zend; k++) { phivalue += phi(mesh->xstart, j, k); } phivalue /= mesh->LocalNz; // Average in Z of point next to boundary } // Old value of phi at boundary. Note: this is constant in Z - BoutReal const oldvalue = + const BoutReal oldvalue = 0.5 * (phi(mesh->xstart - 1, j, 0) + phi(mesh->xstart, j, 0)); // New value of phi at boundary, relaxing towards phivalue - BoutReal const newvalue = weight * oldvalue + (1. - weight) * phivalue; + const BoutReal newvalue = weight * oldvalue + (1. - weight) * phivalue; // Set phi at the boundary to this value - for (int k = 0; k < mesh->LocalNz; k++) { + for (int k = mesh->zstart; k <= mesh->zend; k++) { phi(mesh->xstart - 1, j, k) = 2. * newvalue - phi(mesh->xstart, j, k); phi(mesh->xstart - 2, j, k) = phi(mesh->xstart - 1, j, k); } @@ -1403,7 +1403,7 @@ class ELMpb : public PhysicsModel { if (mesh->lastX()) { for (int j = mesh->ystart; j <= mesh->yend; j++) { BoutReal phivalue = 0.0; - for (int k = 0; k < mesh->LocalNz; k++) { + for (int k = mesh->zstart; k <= mesh->zend; k++) { phivalue += phi(mesh->xend, j, k); } phivalue /= mesh->LocalNz; // Average in Z of point next to boundary @@ -1413,10 +1413,10 @@ class ELMpb : public PhysicsModel { 0.5 * (phi(mesh->xend + 1, j, 0) + phi(mesh->xend, j, 0)); // New value of phi at boundary, relaxing towards phivalue - BoutReal const newvalue = weight * oldvalue + (1. - weight) * phivalue; + const BoutReal newvalue = weight * oldvalue + (1. - weight) * phivalue; // Set phi at the boundary to this value - for (int k = 0; k < mesh->LocalNz; k++) { + for (int k = mesh->zstart; k <= mesh->zend; k++) { phi(mesh->xend + 1, j, k) = 2. * newvalue - phi(mesh->xend, j, k); phi(mesh->xend + 2, j, k) = phi(mesh->xend + 1, j, k); } @@ -1441,7 +1441,7 @@ class ELMpb : public PhysicsModel { if (mesh->firstX()) { for (int j = mesh->ystart; j <= mesh->yend; j++) { - for (int k = 0; k < mesh->LocalNz; k++) { + for (int k = mesh->zstart; k <= mesh->zend; k++) { // Average phi + Pi at the boundary, and set the boundary cell // to this value. The phi solver will then put the value back // onto the cell mid-point @@ -1453,7 +1453,7 @@ class ELMpb : public PhysicsModel { if (mesh->lastX()) { for (int j = mesh->ystart; j <= mesh->yend; j++) { - for (int k = 0; k < mesh->LocalNz; k++) { + for (int k = mesh->zstart; k <= mesh->zend; k++) { phi_shift(mesh->xend + 1, j, k) = 0.5 * (phi_shift(mesh->xend + 1, j, k) + phi_shift(mesh->xend, j, k)); } @@ -1513,7 +1513,7 @@ class ELMpb : public PhysicsModel { if (mesh->firstX()) { for (int i = mesh->xstart - 2; i >= 0; --i) { for (int j = mesh->ystart; j <= mesh->yend; ++j) { - for (int k = 0; k < mesh->LocalNz; ++k) { + for (int k = mesh->zstart; k <= mesh->zend; ++k) { phi(i, j, k) = phi(i + 1, j, k); } } @@ -1523,7 +1523,7 @@ class ELMpb : public PhysicsModel { if (mesh->lastX()) { for (int i = mesh->xend + 2; i < mesh->LocalNx; ++i) { for (int j = mesh->ystart; j <= mesh->yend; ++j) { - for (int k = 0; k < mesh->LocalNz; ++k) { + for (int k = mesh->zstart; k <= mesh->zend; ++k) { phi(i, j, k) = phi(i - 1, j, k); } } @@ -1626,7 +1626,7 @@ class ELMpb : public PhysicsModel { for (int jz = 0; jz < mesh->LocalNz; jz++) { // Zero-gradient potential - BoutReal const phisheath = phi_fa(r.ind, mesh->ystart, jz); + const BoutReal phisheath = phi_fa(r.ind, mesh->ystart, jz); BoutReal jsheath = -(sqrt(mi_me) / (2. * sqrt(PI))) * phisheath; @@ -1647,7 +1647,7 @@ class ELMpb : public PhysicsModel { for (int jz = 0; jz < mesh->LocalNz; jz++) { // Zero-gradient potential - BoutReal const phisheath = phi_fa(r.ind, mesh->yend, jz); + const BoutReal phisheath = phi_fa(r.ind, mesh->yend, jz); BoutReal jsheath = (sqrt(mi_me) / (2. * sqrt(PI))) * phisheath; @@ -2069,7 +2069,8 @@ class ELMpb : public PhysicsModel { ddt(P).applyBoundary("neumann"); Field3D U1 = ddt(U); - U1 += (gamma * B0 * B0) * Grad_par(Jrhs, CELL_CENTRE) + (gamma * b0xcv) * Grad(ddt(P)); + U1 += + (gamma * B0 * B0) * Grad_par(Jrhs, CELL_CENTRE) + (gamma * b0xcv) * Grad(ddt(P)); // Second matrix, solving Alfven wave dynamics static std::unique_ptr invU{nullptr}; diff --git a/examples/elm-pb/runexample.py b/examples/elm-pb/runexample.py index f7ebc01028..b49902d29c 100755 --- a/examples/elm-pb/runexample.py +++ b/examples/elm-pb/runexample.py @@ -28,18 +28,22 @@ # Calculate RMS in toroidal direction prms = np.sqrt(np.mean(p**2, axis=3)) -growth = np.gradient(np.log(prms[:,42,32])) +growth = np.gradient(np.log(prms[:, 42, 32])) # Final growth-rate gamma = growth[-2] import matplotlib.pyplot as plt -plt.plot(tarr, prms[:,42,32], label='Outboard midplane') -plt.plot( [tarr[0], tarr[-1]], - [prms[-1,42,32]*np.exp(gamma*(tarr[0] - tarr[-1])), prms[-1,42,32]], '--', label=r'$\gamma =$'+str(gamma)) +plt.plot(tarr, prms[:, 42, 32], label="Outboard midplane") +plt.plot( + [tarr[0], tarr[-1]], + [prms[-1, 42, 32] * np.exp(gamma * (tarr[0] - tarr[-1])), prms[-1, 42, 32]], + "--", + label=r"$\gamma =$" + str(gamma), +) -plt.yscale('log') +plt.yscale("log") plt.grid() plt.xlabel(r"Time [$1/\tau_A$]") @@ -57,19 +61,21 @@ # Take a poloidal slice at fixed toroidal angle from boutdata.pol_slice import pol_slice -p2d = pol_slice(p[-1,:,:,:], 'cbm18_dens8.grid_nx68ny64.nc', n=15, zangle=0.0) + +p2d = pol_slice(p[-1, :, :, :], "cbm18_dens8.grid_nx68ny64.nc", n=15, zangle=0.0) # Read grid file to get coordinates from boututils.datafile import DataFile -g = DataFile('cbm18_dens8.grid_nx68ny64.nc') -Rxy = g.read("Rxy") # Major radius [m] -Zxy = g.read("Zxy") # Height [m] +g = DataFile("cbm18_dens8.grid_nx68ny64.nc") + +Rxy = g.read("Rxy") # Major radius [m] +Zxy = g.read("Zxy") # Height [m] plt.contourf(Rxy, Zxy, p2d, 30) -plt.axis('equal') # Maintain aspect ratio +plt.axis("equal") # Maintain aspect ratio -plt.colorbar() # Plot a bar down the side with a color scale +plt.colorbar() # Plot a bar down the side with a color scale plt.savefig("poloidal_slice.pdf") diff --git a/examples/fci-wave/CMakeLists.txt b/examples/fci-wave/CMakeLists.txt index 2680b1310e..ea299edb5a 100644 --- a/examples/fci-wave/CMakeLists.txt +++ b/examples/fci-wave/CMakeLists.txt @@ -2,11 +2,13 @@ cmake_minimum_required(VERSION 3.13) project(fci-wave LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(fci-wave +bout_add_example( + fci-wave SOURCES fci-wave.cxx DATA_DIRS div div-integrate logn - EXTRA_FILES compare-density.py) + EXTRA_FILES compare-density.py +) diff --git a/examples/fci-wave/compare-density.py b/examples/fci-wave/compare-density.py index c039c250b6..16e620a08f 100644 --- a/examples/fci-wave/compare-density.py +++ b/examples/fci-wave/compare-density.py @@ -1,4 +1,3 @@ - import matplotlib.pyplot as plt from boutdata import collect import numpy as np @@ -9,60 +8,61 @@ # Note: Data from fci-wave-logn examples commented out. data_noboundary = [ - ("div", "Model 1 (density, point interpolation)") - ,("div-integrate", "Model 2 (density, area integration)") - ,("logn", "Model 3 (log density, area integration)") - #,("../fci-wave-logn/div-integrate", "Model 5 (velocity, log density, area integration)") + ("div", "Model 1 (density, point interpolation)"), + ("div-integrate", "Model 2 (density, area integration)"), + ("logn", "Model 3 (log density, area integration)"), + # ,("../fci-wave-logn/div-integrate", "Model 5 (velocity, log density, area integration)") ] data_boundary = [ - ("boundary", "Model 2 (density, momentum)") - ,("boundary-logn", "Model 3 (log density, momentum)") - #,("../fci-wave-logn/boundary", "Model 5 (log density, velocity)") - ] + ("boundary", "Model 2 (density, momentum)"), + ("boundary-logn", "Model 3 (log density, momentum)"), + # ,("../fci-wave-logn/boundary", "Model 5 (log density, velocity)") +] # Change this to select no boundary or boundary cases data = data_noboundary if run: from boututils.run_wrapper import shell_safe, launch_safe + shell_safe("make > make.log") - for path,label in data: - launch_safe("./fci-wave -d "+path, nproc=nproc, pipe=False) + for path, label in data: + launch_safe("./fci-wave -d " + path, nproc=nproc, pipe=False) -# Collect the results into a dictionary +# Collect the results into a dictionary sum_n_B = {} -for path,label in data: +for path, label in data: n = collect("n", path=path) Bxyz = collect("Bxyz", path=path) time = collect("t_array", path=path) - + nt, nx, ny, nz = n.shape - + n_B = np.ndarray(nt) for t in range(nt): - n_B[t] = np.sum(n[t,:,:,:] / Bxyz) + n_B[t] = np.sum(n[t, :, :, :] / Bxyz) sum_n_B[path] = (time, n_B) # Plot the density at the final time - + plt.figure() - plt.contourf(n[-1,:,0,:].T, 100) + plt.contourf(n[-1, :, 0, :].T, 100) plt.colorbar() plt.xlabel("Major radius") plt.ylabel("Height") - plt.title("Density n, "+label) - plt.savefig(path+".pdf") + plt.title("Density n, " + label) + plt.savefig(path + ".pdf") plt.show() # Make a plot comparing total sum density / B - + plt.figure() -for path,label in data: +for path, label in data: time, n_B = sum_n_B[path] plt.plot(time, n_B, label=label) plt.legend() @@ -71,4 +71,3 @@ plt.savefig("compare-density.pdf") plt.show() - diff --git a/examples/finite-volume/diffusion/CMakeLists.txt b/examples/finite-volume/diffusion/CMakeLists.txt index 0dd7d220f6..ccc3d0f7e0 100644 --- a/examples/finite-volume/diffusion/CMakeLists.txt +++ b/examples/finite-volume/diffusion/CMakeLists.txt @@ -2,10 +2,12 @@ cmake_minimum_required(VERSION 3.13) project(finite-volume-diffusion LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(finite-volume-diffusion +bout_add_example( + finite-volume-diffusion SOURCES diffusion.cxx - EXTRA_FILES mms.py) + EXTRA_FILES mms.py +) diff --git a/examples/finite-volume/diffusion/mms.py b/examples/finite-volume/diffusion/mms.py index 2c609ca82e..31b7714727 100644 --- a/examples/finite-volume/diffusion/mms.py +++ b/examples/finite-volume/diffusion/mms.py @@ -5,26 +5,26 @@ from math import pi # Length of the y domain -Ly = 10. +Ly = 10.0 # metric tensor metric = Metric() # Identity # Define solution in terms of input x,y,z -f = 1 + 0.1*sin(2*y - t) -k = 1 + 0.1*sin(y) +f = 1 + 0.1 * sin(2 * y - t) +k = 1 + 0.1 * sin(y) # Turn solution into real x and z coordinates -replace = [ (y, metric.y*2*pi/Ly) ] +replace = [(y, metric.y * 2 * pi / Ly)] f = f.subs(replace) -k = k.subs(replace) +k = k.subs(replace) ############################## # Calculate time derivatives -dfdt = Div_par( k * Grad_par(f) ) +dfdt = Div_par(k * Grad_par(f)) ############################# # Calculate sources @@ -32,7 +32,7 @@ Sf = diff(f, t) - dfdt # Substitute back to get input y coordinates -replace = [ (metric.y, y*Ly/(2*pi) ) ] +replace = [(metric.y, y * Ly / (2 * pi))] k = k.subs(replace) f = f.subs(replace) diff --git a/examples/finite-volume/fluid/CMakeLists.txt b/examples/finite-volume/fluid/CMakeLists.txt index e9028459ec..e798da77fe 100644 --- a/examples/finite-volume/fluid/CMakeLists.txt +++ b/examples/finite-volume/fluid/CMakeLists.txt @@ -2,11 +2,13 @@ cmake_minimum_required(VERSION 3.13) project(finite-volume-fluid LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(finite-volume-fluid +bout_add_example( + finite-volume-fluid SOURCES fluid.cxx DATA_DIRS data mms - EXTRA_FILES mms.py) + EXTRA_FILES mms.py +) diff --git a/examples/finite-volume/fluid/mms.py b/examples/finite-volume/fluid/mms.py index 8ed8fba517..a31782e2c7 100644 --- a/examples/finite-volume/fluid/mms.py +++ b/examples/finite-volume/fluid/mms.py @@ -5,19 +5,19 @@ from math import pi # Length of the y domain -Ly = 10. +Ly = 10.0 # metric tensor metric = Metric() # Identity # Define solution in terms of input x,y,z -n = 1 + 0.1*sin(2*y - t) -p = 1 + 0.1*cos(3*y + t) -nv = 0.1*sin(y + 2*t) +n = 1 + 0.1 * sin(2 * y - t) +p = 1 + 0.1 * cos(3 * y + t) +nv = 0.1 * sin(y + 2 * t) # Turn solution into real x and z coordinates -replace = [ (y, metric.y*2*pi/Ly) ] +replace = [(y, metric.y * 2 * pi / Ly)] n = n.subs(replace) p = p.subs(replace) @@ -27,16 +27,16 @@ # Calculate time derivatives v = nv / n -gamma = 5./3 +gamma = 5.0 / 3 # Density equation -dndt = - Div_par(nv) +dndt = -Div_par(nv) # Pressure equation -dpdt = - Div_par(p*v) - (gamma-1.0)*p*Div_par(v) +dpdt = -Div_par(p * v) - (gamma - 1.0) * p * Div_par(v) # Momentum equation -dnvdt = - Div_par(nv*v) - Grad_par(p) +dnvdt = -Div_par(nv * v) - Grad_par(p) ############################# # Calculate sources @@ -46,7 +46,7 @@ Snv = diff(nv, t) - dnvdt # Substitute back to get input y coordinates -replace = [ (metric.y, y*Ly/(2*pi) ) ] +replace = [(metric.y, y * Ly / (2 * pi))] n = n.subs(replace) p = p.subs(replace) diff --git a/examples/finite-volume/test/CMakeLists.txt b/examples/finite-volume/test/CMakeLists.txt index 73fe99f960..d09567e193 100644 --- a/examples/finite-volume/test/CMakeLists.txt +++ b/examples/finite-volume/test/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.13) project(finite-volume-test LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() diff --git a/examples/gas-compress/CMakeLists.txt b/examples/gas-compress/CMakeLists.txt index 1b4416d32b..75978ba584 100644 --- a/examples/gas-compress/CMakeLists.txt +++ b/examples/gas-compress/CMakeLists.txt @@ -2,11 +2,13 @@ cmake_minimum_required(VERSION 3.13) project(gas-compress LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(gas-compress +bout_add_example( + gas-compress SOURCES gas_compress.cxx gas_compress.hxx DATA_DIRS rayleigh-taylor sod-shock - EXTRA_FILES rt.grd.nc sod.grd.nc) + EXTRA_FILES rt.grd.nc sod.grd.nc +) diff --git a/examples/gyro-gem/CMakeLists.txt b/examples/gyro-gem/CMakeLists.txt index 7189bb06b8..5d83848688 100644 --- a/examples/gyro-gem/CMakeLists.txt +++ b/examples/gyro-gem/CMakeLists.txt @@ -2,10 +2,12 @@ cmake_minimum_required(VERSION 3.13) project(gyro-gem LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(gyro-gem +bout_add_example( + gyro-gem SOURCES gem.cxx - EXTRA_FILES cyclone_68x32.nc) + EXTRA_FILES cyclone_68x32.nc +) diff --git a/examples/hasegawa-wakatani-3d/CMakeLists.txt b/examples/hasegawa-wakatani-3d/CMakeLists.txt index 0cdb5207f8..c555d6d7f9 100644 --- a/examples/hasegawa-wakatani-3d/CMakeLists.txt +++ b/examples/hasegawa-wakatani-3d/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.13) project(hw3d LANGUAGES CXX C) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() diff --git a/examples/hasegawa-wakatani/CMakeLists.txt b/examples/hasegawa-wakatani/CMakeLists.txt index c9b9401b3a..53f4e5ed4f 100644 --- a/examples/hasegawa-wakatani/CMakeLists.txt +++ b/examples/hasegawa-wakatani/CMakeLists.txt @@ -2,9 +2,8 @@ cmake_minimum_required(VERSION 3.13) project(hasegawa-wakatani LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() bout_add_example(hasegawa-wakatani SOURCES hw.cxx) - diff --git a/examples/invertable_operator/CMakeLists.txt b/examples/invertable_operator/CMakeLists.txt index f054466f23..f18a085ec1 100644 --- a/examples/invertable_operator/CMakeLists.txt +++ b/examples/invertable_operator/CMakeLists.txt @@ -2,10 +2,12 @@ cmake_minimum_required(VERSION 3.13) project(invertable_operator LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(invertable_operator +bout_add_example( + invertable_operator SOURCES invertable_operator.cxx - REQUIRES BOUT_HAS_PETSC) + REQUIRES BOUT_HAS_PETSC +) diff --git a/examples/invertable_operator/invertable_operator.cxx b/examples/invertable_operator/invertable_operator.cxx index 57f9e24b08..99db7f7a1e 100644 --- a/examples/invertable_operator/invertable_operator.cxx +++ b/examples/invertable_operator/invertable_operator.cxx @@ -1,5 +1,4 @@ #include -#include #include #include @@ -26,7 +25,7 @@ class HW : public PhysicsModel { // Drop C term for now Field3D operator()(const Field3D& input) { - TRACE("myLaplacian::operator()"); + Timer timer("invertable_operator_operate"); Field3D result = A * input + D * Delp2(input); @@ -43,7 +42,7 @@ class HW : public PhysicsModel { // Drop C term for now Field3D operator()(const Field3D& input) { - TRACE("myLaplacian::operator()"); + Timer timer("invertable_operator_operate"); Field3D result = A * input + B * Laplace_perp(input); if (withDiv) { diff --git a/examples/laplace-petsc3d/plotcheck.py b/examples/laplace-petsc3d/plotcheck.py index 707e5db1ee..7f2758dd81 100755 --- a/examples/laplace-petsc3d/plotcheck.py +++ b/examples/laplace-petsc3d/plotcheck.py @@ -10,27 +10,27 @@ xg = 2 -f = collect('f', path=datadir)[xg:-xg, :, :] -rhs = collect('rhs', path=datadir)[xg:-xg, :, :] -rhs_check = collect('rhs_check', path=datadir)[xg:-xg, :, :] -error = collect('error', path=datadir)[xg:-xg, :, :] +f = collect("f", path=datadir)[xg:-xg, :, :] +rhs = collect("rhs", path=datadir)[xg:-xg, :, :] +rhs_check = collect("rhs_check", path=datadir)[xg:-xg, :, :] +error = collect("error", path=datadir)[xg:-xg, :, :] pyplot.subplot(221) pyplot.pcolormesh(f[:, yind, :]) pyplot.colorbar() -pyplot.title('f') +pyplot.title("f") pyplot.subplot(222) pyplot.pcolormesh(rhs[:, yind, :]) pyplot.colorbar() -pyplot.title('rhs') +pyplot.title("rhs") pyplot.subplot(223) pyplot.pcolormesh(rhs_check[:, yind, :]) pyplot.colorbar() -pyplot.title('rhs_check') +pyplot.title("rhs_check") pyplot.subplot(224) pyplot.pcolormesh(error[:, yind, :]) pyplot.colorbar() -pyplot.title('error') +pyplot.title("error") pyplot.show() diff --git a/examples/laplacexy/alfven-wave/CMakeLists.txt b/examples/laplacexy/alfven-wave/CMakeLists.txt index 2423400519..ebda7703e6 100644 --- a/examples/laplacexy/alfven-wave/CMakeLists.txt +++ b/examples/laplacexy/alfven-wave/CMakeLists.txt @@ -2,11 +2,13 @@ cmake_minimum_required(VERSION 3.13) project(laplacexy-alfven-wave LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(laplacexy-alfven-wave +bout_add_example( + laplacexy-alfven-wave SOURCES alfven.cxx DATA_DIRS cbm18 data - EXTRA_FILES cbm18_dens8.grid_nx68ny64.nc d3d_119919.nc) + EXTRA_FILES cbm18_dens8.grid_nx68ny64.nc d3d_119919.nc +) diff --git a/examples/laplacexy/laplace_perp/CMakeLists.txt b/examples/laplacexy/laplace_perp/CMakeLists.txt index 388513b044..d12725c5de 100644 --- a/examples/laplacexy/laplace_perp/CMakeLists.txt +++ b/examples/laplacexy/laplace_perp/CMakeLists.txt @@ -2,11 +2,13 @@ cmake_minimum_required(VERSION 3.13) project(laplacexy-laplace_perp LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(laplacexy-laplace_perp +bout_add_example( + laplacexy-laplace_perp SOURCES test.cxx - EXTRA_FILES cbm18_dens8.grid_nx68ny64.nc - DATA_DIRS square torus) + EXTRA_FILES cbm18_dens8.grid_nx68ny64.nc + DATA_DIRS square torus +) diff --git a/examples/laplacexy/simple/CMakeLists.txt b/examples/laplacexy/simple/CMakeLists.txt index 7859a08259..e999b9cf58 100644 --- a/examples/laplacexy/simple/CMakeLists.txt +++ b/examples/laplacexy/simple/CMakeLists.txt @@ -2,11 +2,12 @@ cmake_minimum_required(VERSION 3.13) project(laplacexy-simple LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(laplacexy-simple +bout_add_example( + laplacexy-simple SOURCES test-laplacexy.cxx DATA_DIRS data hypre ) diff --git a/examples/monitor-newapi/CMakeLists.txt b/examples/monitor-newapi/CMakeLists.txt index 0ee3ee7f85..5c2022b792 100644 --- a/examples/monitor-newapi/CMakeLists.txt +++ b/examples/monitor-newapi/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.13) project(monitor-newapi LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() diff --git a/examples/orszag-tang/CMakeLists.txt b/examples/orszag-tang/CMakeLists.txt index 9ac8fd8d1c..46eae5d0f2 100644 --- a/examples/orszag-tang/CMakeLists.txt +++ b/examples/orszag-tang/CMakeLists.txt @@ -2,10 +2,12 @@ cmake_minimum_required(VERSION 3.13) project(orszag-tang LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(orszag-tang +bout_add_example( + orszag-tang SOURCES mhd.cxx - EXTRA_FILES data/otv.grd.nc) + EXTRA_FILES data/otv.grd.nc +) diff --git a/examples/orszag-tang/generate.py b/examples/orszag-tang/generate.py index 3ba6dbbea9..acce7497c3 100644 --- a/examples/orszag-tang/generate.py +++ b/examples/orszag-tang/generate.py @@ -2,14 +2,16 @@ from __future__ import division from builtins import range from past.utils import old_div + # the Scientific Python netCDF 3 interface # http://dirac.cnrs-orleans.fr/ScientificPython/ -#from Scientific.IO.NetCDF import NetCDFFile as Dataset +# from Scientific.IO.NetCDF import NetCDFFile as Dataset # the 'classic' version of the netCDF4 python interface # http://code.google.com/p/netcdf4-python/ import numpy as np from netCDF4 import Dataset -from numpy import dtype # array module from http://numpy.scipy.org +from numpy import dtype # array module from http://numpy.scipy.org + """ This example writes some surface pressure and temperatures The companion program sfc_pres_temp_rd.py shows how to read the netCDF @@ -30,44 +32,45 @@ # # the output array to write will be nx x ny -ny = 100; nx = ny + 4 +ny = 100 +nx = ny + 4 # dy of grid dy = old_div(1.0, np.float(ny)) dx = dy # create grid -dxarr=np.zeros((nx,ny),dtype='float32')+dx -dyarr=np.zeros((nx,ny),dtype='float32')+dy +dxarr = np.zeros((nx, ny), dtype="float32") + dx +dyarr = np.zeros((nx, ny), dtype="float32") + dy -xarr=np.arange(0.,np.float(nx),1.,dtype='float32')*dx -yarr=np.arange(0.,np.float(ny),1.,dtype='float32')*dy +xarr = np.arange(0.0, np.float(nx), 1.0, dtype="float32") * dx +yarr = np.arange(0.0, np.float(ny), 1.0, dtype="float32") * dy # compute initial variables -rho=np.zeros((nx,ny),dtype='float32')+old_div(25.,(36.*np.pi)) -p=np.zeros((nx,ny),dtype='float32')+old_div(5.,(12.*np.pi)) +rho = np.zeros((nx, ny), dtype="float32") + old_div(25.0, (36.0 * np.pi)) +p = np.zeros((nx, ny), dtype="float32") + old_div(5.0, (12.0 * np.pi)) -rho=1. -p=old_div(rho,3.) +rho = 1.0 +p = old_div(rho, 3.0) -v_x=np.zeros((nx,ny),dtype='float32') -Bx=np.zeros((nx,ny),dtype='float32') +v_x = np.zeros((nx, ny), dtype="float32") +Bx = np.zeros((nx, ny), dtype="float32") for y in range(ny): - v_x[:,y]=-np.sin(2.*np.pi*yarr[y]) - Bx[:,y]=-np.sin(2.*np.pi*yarr[y]) - -#Bx=Bx/np.sqrt(4.*np.pi) + v_x[:, y] = -np.sin(2.0 * np.pi * yarr[y]) + Bx[:, y] = -np.sin(2.0 * np.pi * yarr[y]) +# Bx=Bx/np.sqrt(4.*np.pi) -v_y=np.zeros((nx,ny),dtype='float32') -By=np.zeros((nx,ny),dtype='float32') + +v_y = np.zeros((nx, ny), dtype="float32") +By = np.zeros((nx, ny), dtype="float32") for x in range(nx): - v_y[x,:]=np.sin(2.*np.pi*xarr[x]) - By[x,:]=np.sin(4.*np.pi*xarr[x]) - -#By=By/np.sqrt(4.*np.pi) + v_y[x, :] = np.sin(2.0 * np.pi * xarr[x]) + By[x, :] = np.sin(4.0 * np.pi * xarr[x]) + +# By=By/np.sqrt(4.*np.pi) # Domain inside core (periodic) @@ -76,55 +79,62 @@ ixseps2 = nx # open a new netCDF file for writing. -ncfile = Dataset('otv.grd.128.nc','w', format='NETCDF3_CLASSIC') +ncfile = Dataset("otv.grd.128.nc", "w", format="NETCDF3_CLASSIC") # output data. # create the nx and ny dimensions. -ncfile.createDimension('x',nx) -ncfile.createDimension('y',ny) -ncfile.createDimension('single',1) +ncfile.createDimension("x", nx) +ncfile.createDimension("y", ny) +ncfile.createDimension("single", 1) # create and write nx,ny variables -nxx=ncfile.createVariable('nx','i4',('single')) -nyy=ncfile.createVariable('ny','i4',('single')) +nxx = ncfile.createVariable("nx", "i4", ("single")) +nyy = ncfile.createVariable("ny", "i4", ("single")) -nxx[:]=nx -nyy[:]=ny +nxx[:] = nx +nyy[:] = ny # Define the coordinate variables. They will hold the coordinate # information, that is, xarr,yarr -dx = ncfile.createVariable('dx',dtype('float32').char,('x','y')) -dy = ncfile.createVariable('dy',dtype('float32').char,('x','y',)) +dx = ncfile.createVariable("dx", dtype("float32").char, ("x", "y")) +dy = ncfile.createVariable( + "dy", + dtype("float32").char, + ( + "x", + "y", + ), +) # write data to coordinate vars. -dx[:,:] = dxarr -dy[:,:] = dyarr +dx[:, :] = dxarr +dy[:, :] = dyarr # create and write ixseps* dimensions. -ix1=ncfile.createVariable('ixseps1','i4',('single')) -ix2=ncfile.createVariable('ixseps2','i4',('single')) +ix1 = ncfile.createVariable("ixseps1", "i4", ("single")) +ix2 = ncfile.createVariable("ixseps2", "i4", ("single")) -ix1[:]=ixseps1 -ix2[:]=ixseps2 +ix1[:] = ixseps1 +ix2[:] = ixseps2 -# create the corresponding variables -rho0 = ncfile.createVariable('rho0',dtype('float32').char,('x','y')) -p0 = ncfile.createVariable('p0',dtype('float32').char,('x','y')) -v0_x = ncfile.createVariable('v0_x',dtype('float32').char,('x','y')) -v0_y = ncfile.createVariable('v0_y',dtype('float32').char,('x','y')) -B0x = ncfile.createVariable('B0x',dtype('float32').char,('x','y')) -B0y = ncfile.createVariable('B0y',dtype('float32').char,('x','y')) +# create the corresponding variables +rho0 = ncfile.createVariable("rho0", dtype("float32").char, ("x", "y")) +p0 = ncfile.createVariable("p0", dtype("float32").char, ("x", "y")) +v0_x = ncfile.createVariable("v0_x", dtype("float32").char, ("x", "y")) +v0_y = ncfile.createVariable("v0_y", dtype("float32").char, ("x", "y")) +B0x = ncfile.createVariable("B0x", dtype("float32").char, ("x", "y")) +B0y = ncfile.createVariable("B0y", dtype("float32").char, ("x", "y")) # write data to variables. -rho0[:,:]=rho -p0[:,:]=p -v0_x[:,:]=v_x -v0_y[:,:]=v_y -B0x[:,:]=Bx -B0y[:,:]=By +rho0[:, :] = rho +p0[:, :] = p +v0_x[:, :] = v_x +v0_y[:, :] = v_y +B0x[:, :] = Bx +B0y[:, :] = By ncfile.close() -print('*** SUCCESS writing file otv.grd.py.nc!') +print("*** SUCCESS writing file otv.grd.py.nc!") diff --git a/examples/performance/bracket/bracket.cxx b/examples/performance/bracket/bracket.cxx index 50012510ff..0ef9e241fb 100644 --- a/examples/performance/bracket/bracket.cxx +++ b/examples/performance/bracket/bracket.cxx @@ -68,9 +68,6 @@ int main(int argc, char** argv) { ITERATOR_TEST_BLOCK("Bracket [2D,3D] ARAKAWA", result = bracket(a, c, BRACKET_ARAKAWA);); - ITERATOR_TEST_BLOCK("Bracket [2D,3D] ARAKAWA_OLD", - result = bracket(a, c, BRACKET_ARAKAWA_OLD);); - ITERATOR_TEST_BLOCK("Bracket [2D,3D] SIMPLE", result = bracket(a, c, BRACKET_SIMPLE);); @@ -81,21 +78,12 @@ int main(int argc, char** argv) { ITERATOR_TEST_BLOCK("Bracket [3D,3D] ARAKAWA", result = bracket(a, b, BRACKET_ARAKAWA);); - ITERATOR_TEST_BLOCK("Bracket [3D,3D] ARAKAWA_OLD", - result = bracket(a, b, BRACKET_ARAKAWA_OLD);); - ITERATOR_TEST_BLOCK("Bracket [3D,3D] SIMPLE", result = bracket(a, b, BRACKET_SIMPLE);); ITERATOR_TEST_BLOCK("Bracket [3D,3D] DEFAULT", result = bracket(a, b, BRACKET_STD);); } - // Uncomment below for a "correctness" check - // Field3D resNew = bracket(a, b, BRACKET_ARAKAWA); mesh->communicate(resNew); - // Field3D resOld = bracket(a, b, BRACKET_ARAKAWA_OLD); mesh->communicate(resOld); - // time_output << "Max abs diff is - // "<LocalNx; ++i) { for (int j = mesh->ystart; j < mesh->yend; ++j) { - for (int k = 0; k < mesh->LocalNz; ++k) { + for (int k = mesh->zstart; k <= mesh->zend; ++k) { result(i, j, k) = (a(i, j + 1, k) - a(i, j - 1, k)); } } @@ -76,7 +76,7 @@ int main(int argc, char** argv) { BOUT_OMP_PERF(parallel for) for(int i=0;iLocalNx;++i) { for (int j = mesh->ystart; j < mesh->yend; ++j) { - for (int k = 0; k < mesh->LocalNz; ++k) { + for (int k = mesh->zstart; k <= mesh->zend; ++k) { result(i, j, k) = (a(i, j + 1, k) - a(i, j - 1, k)); } } diff --git a/examples/performance/iterator/iterator.cxx b/examples/performance/iterator/iterator.cxx index 7a29b00298..e1fc59d067 100644 --- a/examples/performance/iterator/iterator.cxx +++ b/examples/performance/iterator/iterator.cxx @@ -77,7 +77,7 @@ int main(int argc, char** argv) { ITERATOR_TEST_BLOCK( "Nested loop", for (int i = 0; i < mesh->LocalNx; ++i) { for (int j = 0; j < mesh->LocalNy; ++j) { - for (int k = 0; k < mesh->LocalNz; ++k) { + for (int k = mesh->zstart; k <= mesh->zend; ++k) { result(i, j, k) = a(i, j, k) + b(i, j, k); } } @@ -88,7 +88,7 @@ int main(int argc, char** argv) { BOUT_OMP_PERF(parallel for) for(int i=0;iLocalNx;++i) { for (int j = 0; j < mesh->LocalNy; ++j) { - for (int k = 0; k < mesh->LocalNz; ++k) { + for (int k = mesh->zstart; k <= mesh->zend; ++k) { result(i, j, k) = a(i, j, k) + b(i, j, k); } } diff --git a/examples/performance/iterator/scaling_parser.py b/examples/performance/iterator/scaling_parser.py index d7a23842d4..15f0ea8780 100644 --- a/examples/performance/iterator/scaling_parser.py +++ b/examples/performance/iterator/scaling_parser.py @@ -2,8 +2,7 @@ def read_file(filename): - reader = csv.reader(open(filename, 'r'), delimiter='\t', - skipinitialspace=True) + reader = csv.reader(open(filename, "r"), delimiter="\t", skipinitialspace=True) # Skip header for _, _ in zip(range(4), reader): @@ -13,7 +12,7 @@ def read_file(filename): for line in reader: if line == []: break - case_lines[line[0].rstrip('.')] = line[1] + case_lines[line[0].rstrip(".")] = line[1] titles = next(reader) cases_weak = {col.strip(): [] for col in titles[:-1]} diff --git a/examples/preconditioning/wave/CMakeLists.txt b/examples/preconditioning/wave/CMakeLists.txt index 437f39fe3a..05d7548832 100644 --- a/examples/preconditioning/wave/CMakeLists.txt +++ b/examples/preconditioning/wave/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.13) project(preconditioning-wave LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() diff --git a/examples/staggered_grid/CMakeLists.txt b/examples/staggered_grid/CMakeLists.txt index dd2b3b463e..1950d16c0b 100644 --- a/examples/staggered_grid/CMakeLists.txt +++ b/examples/staggered_grid/CMakeLists.txt @@ -2,11 +2,13 @@ cmake_minimum_required(VERSION 3.13) project(staggered_grid LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(staggered_grid +bout_add_example( + staggered_grid SOURCES test_staggered.cxx EXTRA_FILES generate.py run test-staggered.nc - DATA_DIRS data test) + DATA_DIRS data test +) diff --git a/examples/staggered_grid/generate.py b/examples/staggered_grid/generate.py index da84e9c2e6..a5a0120612 100755 --- a/examples/staggered_grid/generate.py +++ b/examples/staggered_grid/generate.py @@ -4,11 +4,11 @@ # Generate an input mesh # -from boututils.datafile import DataFile # Wrapper around NetCDF4 libraries +from boututils.datafile import DataFile # Wrapper around NetCDF4 libraries -nx = 5 # Minimum is 5: 2 boundary, one evolved +nx = 5 # Minimum is 5: 2 boundary, one evolved ny = 32 # Minimum 5. Should be divisible by number of processors (so powers of 2 nice) -dy = 1. # distance between points in y, in m/g22/lengthunit +dy = 1.0 # distance between points in y, in m/g22/lengthunit ixseps1 = -1 ixseps2 = -1 diff --git a/examples/subsampling/CMakeLists.txt b/examples/subsampling/CMakeLists.txt index 86f71d98f5..c6c487e687 100644 --- a/examples/subsampling/CMakeLists.txt +++ b/examples/subsampling/CMakeLists.txt @@ -2,11 +2,12 @@ cmake_minimum_required(VERSION 3.13) project(subsampling LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(subsampling +bout_add_example( + subsampling SOURCES monitor.cxx - EXTRA_FILES show.py) - + EXTRA_FILES show.py +) diff --git a/examples/subsampling/show.py b/examples/subsampling/show.py index 34c6ee3084..b79c5e2964 100755 --- a/examples/subsampling/show.py +++ b/examples/subsampling/show.py @@ -11,14 +11,14 @@ for pack in monitors: filename, data_name = pack - t = DataFile(path+'/'+filename+'.dmp.0.nc').read('t_array') - data = DataFile(path+'/'+filename+'.dmp.0.nc').read(data_name).flatten() + t = DataFile(path + "/" + filename + ".dmp.0.nc").read("t_array") + data = DataFile(path + "/" + filename + ".dmp.0.nc").read(data_name).flatten() plt.plot(t, data, label="{} {}".format(filename, data_name)) -time = DataFile(path+'/BOUT.dmp.0.nc').read('t_array') -data = DataFile(path+'/BOUT.dmp.0.nc').read("T")[:, 2, 2, 0] +time = DataFile(path + "/BOUT.dmp.0.nc").read("t_array") +data = DataFile(path + "/BOUT.dmp.0.nc").read("T")[:, 2, 2, 0] -plt.plot(time, data, marker='+', label="BOUT++ T") +plt.plot(time, data, marker="+", label="BOUT++ T") plt.xlabel("Time") plt.legend() diff --git a/examples/wave-slab/CMakeLists.txt b/examples/wave-slab/CMakeLists.txt index b1943c4e1c..8de548b71d 100644 --- a/examples/wave-slab/CMakeLists.txt +++ b/examples/wave-slab/CMakeLists.txt @@ -2,10 +2,12 @@ cmake_minimum_required(VERSION 3.13) project(wave-slab LANGUAGES CXX) -if (NOT TARGET bout++::bout++) +if(NOT TARGET bout++::bout++) find_package(bout++ REQUIRED) endif() -bout_add_example(wave-slab +bout_add_example( + wave-slab SOURCES wave_slab.cxx - EXTRA_FILES generate.py) + EXTRA_FILES generate.py +) diff --git a/examples/wave-slab/generate.py b/examples/wave-slab/generate.py index 0a2e8f3049..12698d1e24 100755 --- a/examples/wave-slab/generate.py +++ b/examples/wave-slab/generate.py @@ -51,7 +51,7 @@ for x in range(nx): Bpxy[x, :] = Bpx[x] -Bxy = sqrt(Bpxy ** 2 + Bt ** 2) +Bxy = sqrt(Bpxy**2 + Bt**2) # Calculate change in poloidal flux dr = Lx / nx # Constant mesh spacing in radius diff --git a/externalpackages/PVODE/CMakeLists.txt b/externalpackages/PVODE/CMakeLists.txt index 5e3f8f2f63..51afe8047c 100644 --- a/externalpackages/PVODE/CMakeLists.txt +++ b/externalpackages/PVODE/CMakeLists.txt @@ -6,14 +6,17 @@ else() cmake_policy(VERSION 3.12) endif() -project(PVODE +project( + PVODE DESCRIPTION "ODE Solver" VERSION 0.1 - LANGUAGES CXX) + LANGUAGES CXX +) find_package(MPI REQUIRED) -add_library(pvode +add_library( + pvode source/cvode.cpp source/nvector.cpp source/llnlmath.cpp @@ -33,45 +36,43 @@ add_library(pvode include/pvode/smalldense.h include/pvode/spgmr.h include/pvode/vector.h - ) - -target_include_directories(pvode PUBLIC - $ - $ - $ - ) +) + +target_include_directories( + pvode + PUBLIC $ + $ + $ +) target_link_libraries(pvode PUBLIC MPI::MPI_CXX) -add_library(pvpre - include/pvode/pvbbdpre.h - precon/pvbbdpre.cpp - precon/band.cpp - precon/band.h - ) - +add_library( + pvpre include/pvode/pvbbdpre.h precon/pvbbdpre.cpp precon/band.cpp + precon/band.h +) -set_target_properties(pvode PROPERTIES - SOVERSION 1.0.0) +set_target_properties(pvode PROPERTIES SOVERSION 1.0.0) -target_include_directories(pvpre PUBLIC - $ - $ - $ - ) +target_include_directories( + pvpre + PUBLIC $ + $ + $ +) target_link_libraries(pvpre PUBLIC pvode MPI::MPI_CXX) - -set_target_properties(pvpre PROPERTIES - SOVERSION 1.0.0) +set_target_properties(pvpre PROPERTIES SOVERSION 1.0.0) include(GNUInstallDirs) -install(TARGETS pvode pvpre +install( + TARGETS pvode pvpre EXPORT PVODETargets LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" - INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" - ) + INCLUDES + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" +) install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) include(CMakePackageConfigHelpers) @@ -79,17 +80,19 @@ write_basic_package_version_file( PVODEConfigVersion.cmake VERSION ${PACKAGE_VERSION} COMPATIBILITY SameMajorVersion - ) +) -install(EXPORT PVODETargets +install( + EXPORT PVODETargets FILE PVODEConfig.cmake NAMESPACE PVODE:: DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PVODE" - ) +) -export(EXPORT PVODETargets +export( + EXPORT PVODETargets FILE "${CMAKE_CURRENT_BINARY_DIR}/PVODEConfig.cmake" NAMESPACE PVODE:: - ) +) export(PACKAGE PVODE) diff --git a/externalpackages/boutdata b/externalpackages/boutdata index 7164a89c16..11f5d0acb3 160000 --- a/externalpackages/boutdata +++ b/externalpackages/boutdata @@ -1 +1 @@ -Subproject commit 7164a89c16dba1049d42e1715506b988c0af5926 +Subproject commit 11f5d0acb3d92a30eb0f97866c206a238c487076 diff --git a/externalpackages/cpptrace b/externalpackages/cpptrace new file mode 160000 index 0000000000..027f9aee2d --- /dev/null +++ b/externalpackages/cpptrace @@ -0,0 +1 @@ +Subproject commit 027f9aee2d34dbe1c98f26224e1fbe1654cb4aae diff --git a/externalpackages/googletest b/externalpackages/googletest index 244cec869d..73a63ea05d 160000 --- a/externalpackages/googletest +++ b/externalpackages/googletest @@ -1 +1 @@ -Subproject commit 244cec869d12e53378fa0efb610cd4c32a454ec8 +Subproject commit 73a63ea05dc8ca29ec1d2c1d66481dd0de1950f1 diff --git a/include/bout/array.hxx b/include/bout/array.hxx index 2305fe3b22..82677fd38a 100644 --- a/include/bout/array.hxx +++ b/include/bout/array.hxx @@ -227,6 +227,12 @@ public: ptr = get(new_size); } + /*! + * Change shape of the container. + * Invalidates contents. + */ + void reshape(std::tuple new_shape) { reallocate(std::get<0>(new_shape)); } + /*! * Holds a static variable which controls whether * memory blocks (dataBlock) are put into a store diff --git a/include/bout/bout_enum_class.hxx b/include/bout/bout_enum_class.hxx index 585e5b020e..6f632747ed 100644 --- a/include/bout/bout_enum_class.hxx +++ b/include/bout/bout_enum_class.hxx @@ -22,13 +22,12 @@ #ifndef BOUT_ENUM_CLASS_H #define BOUT_ENUM_CLASS_H -#include "bout/boutexception.hxx" +#include "bout/boutexception.hxx" // IWYU pragma: keep #include "bout/macro_for_each.hxx" -#include "bout/msg_stack.hxx" -#include "bout/options.hxx" +#include "bout/options.hxx" // IWYU pragma: keep -#include -#include +#include // IWYU pragma: keep +#include // IWYU pragma: keep /// Create some macro magic similar to bout/macro_for_each.hxx, but allowing for the enum /// class name to be passed through to each _call @@ -70,7 +69,7 @@ enum class enumname { __VA_ARGS__ }; \ \ inline std::string toString(enumname e) { \ - AUTO_TRACE(); \ + \ const static std::map toString_map = { \ BOUT_ENUM_CLASS_MAP_ARGS(BOUT_ENUM_CLASS_STR, enumname, __VA_ARGS__)}; \ auto found = toString_map.find(e); \ @@ -81,16 +80,17 @@ } \ \ inline enumname BOUT_MAKE_FROMSTRING_NAME(enumname)(const std::string& s) { \ - AUTO_TRACE(); \ + \ const static std::map fromString_map = { \ BOUT_ENUM_CLASS_MAP_ARGS(BOUT_STR_ENUM_CLASS, enumname, __VA_ARGS__)}; \ auto found = fromString_map.find(s); \ if (found == fromString_map.end()) { \ - std::string valid_values {}; \ + std::string valid_values{}; \ for (auto const& entry : fromString_map) { \ valid_values += std::string(" ") + entry.first; \ } \ - throw BoutException("Did not find enum {:s}. Valid values: {:s}", s, valid_values); \ + throw BoutException("Did not find enum {:s}. Valid values: {:s}", s, \ + valid_values); \ } \ return found->second; \ } \ diff --git a/include/bout/boutcomm.hxx b/include/bout/boutcomm.hxx index 68c7559fee..29bff53087 100644 --- a/include/bout/boutcomm.hxx +++ b/include/bout/boutcomm.hxx @@ -4,9 +4,9 @@ * * ************************************************************************** -* Copyright 2010 B.D.Dudson, S.Farley, M.V.Umansky, X.Q.Xu +* Copyright 2010 - 2026 BOUT++ contributors * -* Contact: Ben Dudson, bd512@york.ac.uk +* Contact: Ben Dudson, dudson2@llnl.gov * * This file is part of BOUT++. * @@ -49,6 +49,8 @@ public: static int rank(); ///< Rank: my processor number static int size(); ///< Size: number of processors + static void abort(int errorcode); ///< MPI abort + // Setting options void setComm(MPI_Comm c); diff --git a/include/bout/boutexception.hxx b/include/bout/boutexception.hxx index 66e4fddd4a..735f98c7d6 100644 --- a/include/bout/boutexception.hxx +++ b/include/bout/boutexception.hxx @@ -1,13 +1,11 @@ #ifndef BOUT_EXCEPTION_H #define BOUT_EXCEPTION_H -#include "bout/build_defines.hxx" - -#include #include #include #include +#include "fmt/base.h" #include "fmt/core.h" /// Throw BoutRhsFail with \p message if any one process has non-zero @@ -16,11 +14,16 @@ void BoutParallelThrowRhsFail(int status, const char* message); class BoutException : public std::exception { public: + BoutException(const BoutException&) = default; + BoutException(BoutException&&) = delete; + BoutException& operator=(const BoutException&) = default; + BoutException& operator=(BoutException&&) = delete; BoutException(std::string msg); - template - BoutException(const S& format, Args&&... args) - : BoutException(fmt::format(format, std::forward(args)...)) {} + template + // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) + BoutException(fmt::format_string format, Args&&... args) + : BoutException(fmt::vformat(format, fmt::make_format_args(args...))) {} ~BoutException() override; @@ -30,32 +33,31 @@ public: /// backtrace (if available) std::string getBacktrace() const; + static void enableBacktrace() { show_backtrace = true; } + static void disableBacktrace() { show_backtrace = false; } + private: std::string message; -#if BOUT_USE_BACKTRACE - static constexpr unsigned int TRACE_MAX = 128; - std::array trace{}; - int trace_size; - char** messages; -#endif - void makeBacktrace(); + static bool show_backtrace; }; class BoutRhsFail : public BoutException { public: BoutRhsFail(std::string message) : BoutException(std::move(message)) {} - template - BoutRhsFail(const S& format, const Args&... args) - : BoutRhsFail(fmt::format(format, args...)) {} + template + // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) + BoutRhsFail(fmt::format_string format, Args&&... args) + : BoutRhsFail(fmt::vformat(format, fmt::make_format_args(args...))) {} }; class BoutIterationFail : public BoutException { public: BoutIterationFail(std::string message) : BoutException(std::move(message)) {} - template - BoutIterationFail(const S& format, const Args&... args) - : BoutIterationFail(fmt::format(format, args...)) {} + template + // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) + BoutIterationFail(fmt::format_string format, Args&&... args) + : BoutIterationFail(fmt::vformat(format, fmt::make_format_args(args...))) {} }; #endif diff --git a/include/bout/coordinates.hxx b/include/bout/coordinates.hxx index d7c80ed8bc..e7ead42ee5 100644 --- a/include/bout/coordinates.hxx +++ b/include/bout/coordinates.hxx @@ -33,10 +33,11 @@ #ifndef BOUT_COORDINATES_H #define BOUT_COORDINATES_H -#include "bout/bout_types.hxx" -#include "bout/field2d.hxx" -#include "bout/field3d.hxx" -#include "bout/paralleltransform.hxx" +#include +#include +#include +#include +#include class Mesh; diff --git a/include/bout/cyclic_reduction.hxx b/include/bout/cyclic_reduction.hxx index d4c0920910..cf54b63059 100644 --- a/include/bout/cyclic_reduction.hxx +++ b/include/bout/cyclic_reduction.hxx @@ -47,7 +47,6 @@ //#define DIAGNOSE 1 #include "mpi.h" -#include "bout/msg_stack.hxx" #include "bout/utils.hxx" #include @@ -118,7 +117,6 @@ public: /// @param[in] b Diagonal values. Should have size [nsys][N] /// @param[in] c Right diagonal. Should have size [nsys][N] void setCoefs(const Matrix& a, const Matrix& b, const Matrix& c) { - TRACE("CyclicReduce::setCoefs"); int nsys = std::get<0>(a.shape()); @@ -169,7 +167,7 @@ public: /// @param[in] rhs Matrix storing Values of the rhs for each system /// @param[out] x Matrix storing the result for each system void solve(const Matrix& rhs, Matrix& x) { - TRACE("CyclicReduce::solve"); + ASSERT2(static_cast(std::get<0>(rhs.shape())) == Nsys); ASSERT2(static_cast(std::get<0>(x.shape())) == Nsys); ASSERT2(static_cast(std::get<1>(rhs.shape())) == N); diff --git a/include/bout/deriv_store.hxx b/include/bout/deriv_store.hxx index 6dc44c76ad..b2b7928c2e 100644 --- a/include/bout/deriv_store.hxx +++ b/include/bout/deriv_store.hxx @@ -29,6 +29,7 @@ #ifndef __DERIV_STORE_HXX__ #define __DERIV_STORE_HXX__ +#include #include #include #include @@ -38,7 +39,6 @@ #include #include -#include #include /// Here we have a templated singleton that is used to store DerivativeFunctions @@ -75,21 +75,14 @@ struct DerivativeStore { } /// Report if store has any registered methods - bool isEmpty() const { - AUTO_TRACE(); - return registeredMethods.empty(); - }; + bool isEmpty() const { return registeredMethods.empty(); }; /// Report if store has any registered methods for specific type determined by key - bool isEmpty(std::size_t key) const { - AUTO_TRACE(); - return registeredMethods.count(key) == 0; - } + bool isEmpty(std::size_t key) const { return registeredMethods.count(key) == 0; } /// Report if store has any registered methods for specific type bool isEmpty(DERIV derivType, DIRECTION direction, STAGGER stagger = STAGGER::None) const { - AUTO_TRACE(); // Get the key auto key = getKey(direction, stagger, toString(derivType)); @@ -100,7 +93,6 @@ struct DerivativeStore { /// specified derivative type, direction and stagger. std::set getAvailableMethods(DERIV derivType, DIRECTION direction, STAGGER stagger = STAGGER::None) const { - AUTO_TRACE(); // Get the key auto key = getKey(direction, stagger, toString(derivType)); @@ -115,7 +107,6 @@ struct DerivativeStore { /// specified derivative type, direction and stagger. void listAvailableMethods(DERIV derivType, DIRECTION direction, STAGGER stagger = STAGGER::None) const { - AUTO_TRACE(); // Introductory information output_info << "Available methods for derivative type '"; @@ -134,7 +125,7 @@ struct DerivativeStore { /// depends on the derivType input. void registerDerivative(standardFunc func, DERIV derivType, DIRECTION direction, STAGGER stagger, std::string methodName) { - AUTO_TRACE(); + const auto key = getKey(direction, stagger, methodName); switch (derivType) { @@ -176,7 +167,7 @@ struct DerivativeStore { /// depends on the derivType input. void registerDerivative(upwindFunc func, DERIV derivType, DIRECTION direction, STAGGER stagger, std::string methodName) { - AUTO_TRACE(); + const auto key = getKey(direction, stagger, methodName); switch (derivType) { @@ -210,14 +201,14 @@ struct DerivativeStore { template void registerDerivative(standardFunc func, Direction direction, Stagger stagger, Method method) { - AUTO_TRACE(); + registerDerivative(func, method.meta.derivType, direction.lookup(), stagger.lookup(), method.meta.key); } template void registerDerivative(upwindFunc func, Direction direction, Stagger stagger, Method method) { - AUTO_TRACE(); + registerDerivative(func, method.meta.derivType, direction.lookup(), stagger.lookup(), method.meta.key); } @@ -231,7 +222,6 @@ struct DerivativeStore { STAGGER stagger = STAGGER::None, DERIV derivType = DERIV::Standard) const { - AUTO_TRACE(); const auto realName = nameLookup( name, defaultMethods.at(getKey(direction, stagger, toString(derivType)))); const auto key = getKey(direction, stagger, realName); @@ -262,20 +252,20 @@ struct DerivativeStore { standardFunc getStandard2ndDerivative(std::string name, DIRECTION direction, STAGGER stagger = STAGGER::None) const { - AUTO_TRACE(); + return getStandardDerivative(name, direction, stagger, DERIV::StandardSecond); }; standardFunc getStandard4thDerivative(std::string name, DIRECTION direction, STAGGER stagger = STAGGER::None) const { - AUTO_TRACE(); + return getStandardDerivative(name, direction, stagger, DERIV::StandardFourth); }; flowFunc getFlowDerivative(std::string name, DIRECTION direction, STAGGER stagger = STAGGER::None, DERIV derivType = DERIV::Upwind) const { - AUTO_TRACE(); + const auto realName = nameLookup( name, defaultMethods.at(getKey(direction, stagger, toString(derivType)))); const auto key = getKey(direction, stagger, realName); @@ -305,18 +295,17 @@ struct DerivativeStore { upwindFunc getUpwindDerivative(std::string name, DIRECTION direction, STAGGER stagger = STAGGER::None) const { - AUTO_TRACE(); + return getFlowDerivative(name, direction, stagger, DERIV::Upwind); }; fluxFunc getFluxDerivative(std::string name, DIRECTION direction, STAGGER stagger = STAGGER::None) const { - AUTO_TRACE(); + return getFlowDerivative(name, direction, stagger, DERIV::Flux); }; void initialise(Options* options) { - AUTO_TRACE(); // To replicate the existing behaviour we first search for a section called //"dd?" and if the option isn't in there we search a section called "diff" @@ -490,7 +479,7 @@ private: std::string getMethodName(std::string name, DIRECTION direction, STAGGER stagger = STAGGER::None) const { - AUTO_TRACE(); + return name + " (" + toString(direction) + ", " + toString(stagger) + ")"; }; @@ -506,7 +495,7 @@ private: /// methods with the same function interface in the same map, which /// might be nice. std::size_t getKey(DIRECTION direction, STAGGER stagger, std::string key) const { - AUTO_TRACE(); + // Note this key is indepedent of the field type (and hence the // key is the same for 3D/2D fields) as we have to use different // maps to store the different field types as the signature is @@ -524,7 +513,7 @@ private: /// that can be used to account for run-time choices template std::size_t getKey() const { - AUTO_TRACE(); + // Note this key is indepedent of the field type (and hence the // key is the same for 3D/2D fields) as we have to use different // maps to store the different field types as the signature is diff --git a/include/bout/difops.hxx b/include/bout/difops.hxx index 71053d454a..18220b63ad 100644 --- a/include/bout/difops.hxx +++ b/include/bout/difops.hxx @@ -40,7 +40,9 @@ #include "bout/field3d.hxx" #include "bout/bout_types.hxx" -#include "bout/solver.hxx" +#include "bout/coordinates.hxx" + +class Solver; /*! * Parallel derivative (central differencing) in Y @@ -271,18 +273,16 @@ Field3D b0xGrad_dot_Grad(const Field3D& phi, const Field3D& A, * Poisson bracket methods */ enum class BRACKET_METHOD { - standard, ///< Use b0xGrad_dot_Grad - simple, ///< Keep only terms in X-Z - arakawa, ///< Arakawa method in X-Z (optimised) - ctu, ///< Corner Transport Upwind (CTU) method. Explicit method only, needs the - /// timestep from the solver - arakawa_old ///< Older version, for regression testing of optimised version. + standard, ///< Use b0xGrad_dot_Grad + simple, ///< Keep only terms in X-Z + arakawa, ///< Arakawa method in X-Z + ctu, ///< Corner Transport Upwind (CTU) method. Explicit method only, needs the + /// timestep from the solver }; constexpr BRACKET_METHOD BRACKET_STD = BRACKET_METHOD::standard; constexpr BRACKET_METHOD BRACKET_SIMPLE = BRACKET_METHOD::simple; constexpr BRACKET_METHOD BRACKET_ARAKAWA = BRACKET_METHOD::arakawa; constexpr BRACKET_METHOD BRACKET_CTU = BRACKET_METHOD::ctu; -constexpr BRACKET_METHOD BRACKET_ARAKAWA_OLD = BRACKET_METHOD::arakawa_old; /*! * Compute advection operator terms, which can be cast as diff --git a/include/bout/fft.hxx b/include/bout/fft.hxx index fdec8b7bec..b7d36d2166 100644 --- a/include/bout/fft.hxx +++ b/include/bout/fft.hxx @@ -28,10 +28,15 @@ #ifndef BOUT_FFT_H #define BOUT_FFT_H -#include "bout/dcomplex.hxx" +#include "bout/build_defines.hxx" + #include #include +#include + +#include +class Mesh; class Options; BOUT_ENUM_CLASS(FFT_MEASUREMENT_FLAG, estimate, measure, exhaustive); @@ -111,6 +116,16 @@ Array rfft(const Array& in); /// Expects that `in.size() == (length / 2) + 1` Array irfft(const Array& in, int length); +/// Check simulation is using 1 processor in Z, throw exception if not +/// +/// Generally, FFTs must be done over the full Z domain. Currently, most +/// methods using FFTs don't handle parallelising in Z +#if BOUT_CHECK_LEVEL > 0 +void assertZSerial(const Mesh& mesh, std::string_view name); +#else +inline void assertZSerial([[maybe_unused]] const Mesh& mesh, + [[maybe_unused]] std::string_view name) {} +#endif } // namespace fft } // namespace bout diff --git a/include/bout/field.hxx b/include/bout/field.hxx index 61edff6723..273cc08342 100644 --- a/include/bout/field.hxx +++ b/include/bout/field.hxx @@ -6,7 +6,7 @@ * Copyright 2010 B.D.Dudson, S.Farley, M.V.Umansky, X.Q.Xu * * Contact: Ben Dudson, bd512@york.ac.uk - * + * * This file is part of BOUT++. * * BOUT++ is free software: you can redistribute it and/or modify @@ -31,13 +31,13 @@ class Field; #include #include +#include #include #include "bout/bout_types.hxx" #include "bout/boutcomm.hxx" #include "bout/boutexception.hxx" #include "bout/field_data.hxx" -#include "bout/msg_stack.hxx" #include "bout/region.hxx" #include "bout/traits.hxx" #include "bout/utils.hxx" @@ -127,6 +127,17 @@ public: swap(first.directions, second.directions); } + /// Dummy functions to increase portability + virtual void setRegion([[maybe_unused]] size_t regionID) {} + virtual void setRegion([[maybe_unused]] std::optional regionID) {} + virtual void setRegion([[maybe_unused]] const std::string& region_name) {} + virtual void resetRegion() {} + virtual std::optional getRegionID() const { return {}; } + virtual bool hasParallelSlices() const { return true; } + virtual void calcParallelSlices() {} + virtual void splitParallelSlices() {} + virtual void clearParallelSlices() {} + private: /// Labels for the type of coordinate system this field is defined over DirectionTypes directions{YDirectionType::Standard, ZDirectionType::Standard}; @@ -178,7 +189,8 @@ inline bool areFieldsCompatible(const Field& field1, const Field& field2) { template inline T emptyFrom(const T& f) { static_assert(bout::utils::is_Field_v, "emptyFrom only works on Fields"); - return T(f.getMesh(), f.getLocation(), {f.getDirectionY(), f.getDirectionZ()}) + return T(f.getMesh(), f.getLocation(), {f.getDirectionY(), f.getDirectionZ()}, + f.getRegionID()) .allocate(); } @@ -239,7 +251,6 @@ namespace bout { template inline void checkFinite(const T& f, const std::string& name = "field", const std::string& rgn = "RGN_ALL") { - AUTO_TRACE(); if (!f.isAllocated()) { throw BoutException("{:s} is not allocated", name); @@ -263,7 +274,6 @@ inline void checkFinite(const T& f, const std::string& name = "field", template inline void checkPositive(const T& f, const std::string& name = "field", const std::string& rgn = "RGN_ALL") { - AUTO_TRACE(); if (!f.isAllocated()) { throw BoutException("{:s} is not allocated", name); @@ -285,6 +295,7 @@ inline void checkPositive(const T& f, const std::string& name = "field", template inline T toFieldAligned(const T& f, const std::string& region = "RGN_ALL") { static_assert(bout::utils::is_Field_v, "toFieldAligned only works on Fields"); + ASSERT3(f.getCoordinates() != nullptr); return f.getCoordinates()->getParallelTransform().toFieldAligned(f, region); } @@ -292,6 +303,7 @@ inline T toFieldAligned(const T& f, const std::string& region = "RGN_ALL") { template inline T fromFieldAligned(const T& f, const std::string& region = "RGN_ALL") { static_assert(bout::utils::is_Field_v, "fromFieldAligned only works on Fields"); + ASSERT3(f.getCoordinates() != nullptr); return f.getCoordinates()->getParallelTransform().fromFieldAligned(f, region); } @@ -307,7 +319,6 @@ inline T fromFieldAligned(const T& f, const std::string& region = "RGN_ALL") { template > inline BoutReal min(const T& f, bool allpe = false, const std::string& rgn = "RGN_NOBNDRY") { - AUTO_TRACE(); checkData(f); @@ -392,7 +403,6 @@ inline BoutReal getUniform(const T& f, [[maybe_unused]] bool allpe = false, template > inline BoutReal max(const T& f, bool allpe = false, const std::string& rgn = "RGN_NOBNDRY") { - AUTO_TRACE(); checkData(f); @@ -426,7 +436,6 @@ inline BoutReal max(const T& f, bool allpe = false, template > inline BoutReal mean(const T& f, bool allpe = false, const std::string& rgn = "RGN_NOBNDRY") { - AUTO_TRACE(); checkData(f); @@ -457,7 +466,6 @@ inline BoutReal mean(const T& f, bool allpe = false, /// If CHECK >= 3 then the result will be checked for non-finite numbers template > T pow(const T& lhs, const T& rhs, const std::string& rgn = "RGN_ALL") { - AUTO_TRACE(); ASSERT1(areFieldsCompatible(lhs, rhs)); @@ -471,7 +479,6 @@ T pow(const T& lhs, const T& rhs, const std::string& rgn = "RGN_ALL") { template > T pow(const T& lhs, BoutReal rhs, const std::string& rgn = "RGN_ALL") { - AUTO_TRACE(); // Check if the inputs are allocated checkData(lhs); @@ -487,7 +494,6 @@ T pow(const T& lhs, BoutReal rhs, const std::string& rgn = "RGN_ALL") { template > T pow(BoutReal lhs, const T& rhs, const std::string& rgn = "RGN_ALL") { - AUTO_TRACE(); // Check if the inputs are allocated checkData(lhs); @@ -524,7 +530,7 @@ T pow(BoutReal lhs, const T& rhs, const std::string& rgn = "RGN_ALL") { #define FIELD_FUNC(name, func) \ template > \ inline T name(const T& f, const std::string& rgn = "RGN_ALL") { \ - AUTO_TRACE(); \ + \ /* Check if the input is allocated */ \ checkData(f); \ /* Define and allocate the output result */ \ @@ -632,7 +638,6 @@ FIELD_FUNC(tanh, ::tanh) /// default (can be changed using the \p rgn argument template > inline bool finite(const T& f, const std::string& rgn = "RGN_ALL") { - AUTO_TRACE(); if (!f.isAllocated()) { return false; diff --git a/include/bout/field2d.hxx b/include/bout/field2d.hxx index 92658f1bbf..418a1dbbde 100644 --- a/include/bout/field2d.hxx +++ b/include/bout/field2d.hxx @@ -4,9 +4,9 @@ * \brief Definition of 2D scalar field class * ************************************************************************** - * Copyright 2010 B.D.Dudson, S.Farley, M.V.Umansky, X.Q.Xu + * Copyright 2010 - 2026 BOUT++ developers * - * Contact: Ben Dudson, bd512@york.ac.uk + * Contact: Ben Dudson, dudson2@llnl.gov * * This file is part of BOUT++. * @@ -24,6 +24,8 @@ * along with BOUT++. If not, see . * */ +#include "bout/utils.hxx" +#include class Field2D; #pragma once @@ -31,12 +33,16 @@ class Field2D; #define BOUT_FIELD2D_H #include "bout/array.hxx" -#include "bout/build_config.hxx" +#include "bout/bout_types.hxx" +#include "bout/build_defines.hxx" #include "bout/field.hxx" #include "bout/field_data.hxx" #include "bout/fieldperp.hxx" #include "bout/region.hxx" -#include "bout/unused.hxx" + +#include +#include +#include #if BOUT_HAS_RAJA #include "RAJA/RAJA.hpp" // using RAJA lib @@ -66,7 +72,8 @@ public: */ Field2D(Mesh* localmesh = nullptr, CELL_LOC location_in = CELL_CENTRE, DirectionTypes directions_in = {YDirectionType::Standard, - ZDirectionType::Average}); + ZDirectionType::Average}, + std::optional region = {}); /*! * Copy constructor. After this both fields @@ -133,21 +140,14 @@ public: return *this; } - /// Check if this field has yup and ydown fields - bool hasParallelSlices() const { return true; } - - Field2D& yup(std::vector::size_type UNUSED(index) = 0) { return *this; } - const Field2D& yup(std::vector::size_type UNUSED(index) = 0) const { - return *this; - } + Field2D& yup([[maybe_unused]] size_t index = 0) { return *this; } + const Field2D& yup([[maybe_unused]] size_t index = 0) const { return *this; } - Field2D& ydown(std::vector::size_type UNUSED(index) = 0) { return *this; } - const Field2D& ydown(std::vector::size_type UNUSED(index) = 0) const { - return *this; - } + Field2D& ydown([[maybe_unused]] size_t index = 0) { return *this; } + const Field2D& ydown([[maybe_unused]] size_t index = 0) const { return *this; } - Field2D& ynext(int UNUSED(dir)) { return *this; } - const Field2D& ynext(int UNUSED(dir)) const { return *this; } + Field2D& ynext([[maybe_unused]] int dir) { return *this; } + const Field2D& ynext([[maybe_unused]] int dir) const { return *this; } // Operators @@ -209,7 +209,7 @@ public: } #endif - return data[jx * ny + jy]; + return data[(jx * ny) + jy]; } inline const BoutReal& operator()(int jx, int jy) const { #if CHECK > 2 && !BOUT_HAS_CUDA @@ -223,15 +223,17 @@ public: } #endif - return data[jx * ny + jy]; + return data[(jx * ny) + jy]; } /*! * DIrect access to underlying array. This version is for compatibility * with Field3D objects */ - BoutReal& operator()(int jx, int jy, int UNUSED(jz)) { return operator()(jx, jy); } - const BoutReal& operator()(int jx, int jy, int UNUSED(jz)) const { + BoutReal& operator()(int jx, int jy, [[maybe_unused]] int jz) { + return operator()(jx, jy); + } + const BoutReal& operator()(int jx, int jy, [[maybe_unused]] int jz) const { return operator()(jx, jy); } @@ -253,8 +255,7 @@ public: Field2D& operator/=(BoutReal rhs); // FieldData virtual functions - - bool is3D() const override { return false; } + FieldType field_type() const override { return FieldType::field2d; } #if CHECK > 0 void doneComms() override { bndry_xin = bndry_xout = bndry_yup = bndry_ydown = true; } @@ -274,7 +275,7 @@ public: friend void swap(Field2D& first, Field2D& second) noexcept; - int size() const override { return nx * ny; }; + int size() const override { return nx * ny; } private: /// Internal data array. Handles allocation/freeing of memory @@ -318,12 +319,12 @@ Field2D operator-(const Field2D& f); // Non-member functions inline Field2D toFieldAligned(const Field2D& f, - const std::string& UNUSED(region) = "RGN_ALL") { + [[maybe_unused]] const std::string& region = "RGN_ALL") { return f; } inline Field2D fromFieldAligned(const Field2D& f, - const std::string& UNUSED(region) = "RGN_ALL") { + [[maybe_unused]] const std::string& region = "RGN_ALL") { return f; } @@ -334,15 +335,15 @@ inline Field2D fromFieldAligned(const Field2D& f, /// default (can be changed using the \p rgn argument void checkData(const Field2D& f, const std::string& region = "RGN_NOBNDRY"); #else -inline void checkData(const Field2D& UNUSED(f), - std::string UNUSED(region) = "RGN_NOBNDRY") {} +inline void checkData([[maybe_unused]] const Field2D& f, + [[maybe_unused]] std::string region = "RGN_NOBNDRY") {} #endif /// Force guard cells of passed field \p var to NaN #if CHECK > 2 void invalidateGuards(Field2D& var); #else -inline void invalidateGuards(Field2D& UNUSED(var)) {} +inline void invalidateGuards([[maybe_unused]] Field2D& var) {} #endif /// Average in the Z direction @@ -358,7 +359,7 @@ inline Field2D& ddt(Field2D& f) { return *(f.timeDeriv()); } /// toString template specialisation /// Defined in utils.hxx template <> -inline std::string toString<>(const Field2D& UNUSED(val)) { +inline std::string toString<>([[maybe_unused]] const Field2D& val) { return ""; } diff --git a/include/bout/field3d.hxx b/include/bout/field3d.hxx index a75e38df36..ad3249270a 100644 --- a/include/bout/field3d.hxx +++ b/include/bout/field3d.hxx @@ -1,8 +1,8 @@ /************************************************************************** - * Copyright 2010 B.D.Dudson, S.Farley, M.V.Umansky, X.Q.Xu + * Copyright 2010 - 2026 BOUT++ contributors + * + * Contact: Ben Dudson, dudson2@llnl.gov * - * Contact: Ben Dudson, bd512@york.ac.uk - * * This file is part of BOUT++. * * BOUT++ is free software: you can redistribute it and/or modify @@ -20,6 +20,8 @@ * **************************************************************************/ +#include "bout/utils.hxx" +#include class Field3D; #pragma once @@ -31,13 +33,18 @@ class Field3D; #include "bout/bout_types.hxx" #include "bout/field.hxx" #include "bout/field2d.hxx" +#include "bout/field_data.hxx" #include "bout/fieldperp.hxx" #include "bout/region.hxx" +#include "bout/traits.hxx" +#include #include +#include #include class Mesh; +class Options; /// Class for 3D X-Y-Z scalar fields /*! @@ -166,7 +173,8 @@ public: */ Field3D(Mesh* localmesh = nullptr, CELL_LOC location_in = CELL_CENTRE, DirectionTypes directions_in = {YDirectionType::Standard, - ZDirectionType::Standard}); + ZDirectionType::Standard}, + std::optional regionID = {}); /*! * Copy constructor @@ -234,15 +242,15 @@ public: * Ensure that this field has separate fields * for yup and ydown. */ - void splitParallelSlices(); + void splitParallelSlices() override; /*! * Clear the parallel slices, yup and ydown */ - void clearParallelSlices(); + void clearParallelSlices() override; /// Check if this field has yup and ydown fields - bool hasParallelSlices() const { + bool hasParallelSlices() const override { #if CHECK > 2 if (yup_fields.size() != ydown_fields.size()) { throw BoutException( @@ -259,24 +267,24 @@ public: /// Check if this field has yup and ydown fields /// Return reference to yup field - Field3D& yup(std::vector::size_type index = 0) { + Field3D& yup(size_t index = 0) { ASSERT2(index < yup_fields.size()); return yup_fields[index]; } /// Return const reference to yup field - const Field3D& yup(std::vector::size_type index = 0) const { + const Field3D& yup(size_t index = 0) const { ASSERT2(index < yup_fields.size()); return yup_fields[index]; } /// Return reference to ydown field - Field3D& ydown(std::vector::size_type index = 0) { + Field3D& ydown(size_t index = 0) { ASSERT2(index < ydown_fields.size()); return ydown_fields[index]; } /// Return const reference to ydown field - const Field3D& ydown(std::vector::size_type index = 0) const { + const Field3D& ydown(size_t index = 0) const { ASSERT2(index < ydown_fields.size()); return ydown_fields[index]; } @@ -291,6 +299,17 @@ public: /// cuts on closed field lines? bool requiresTwistShift(bool twist_shift_enabled); + /// Enable a special tracking mode for debugging + /// Save all changes that, are done to the field, to tracking + Field3D& enableTracking(const std::string& name, std::weak_ptr tracking); + + /// Disable tracking + Field3D& disableTracking() { + tracking.reset(); + tracking_state = 0; + return *this; + } + ///////////////////////////////////////////////////////// // Data access @@ -312,11 +331,12 @@ public: const Region& getRegion(const std::string& region_name) const; /// Use region provided by the default, and if none is set, use the provided one const Region& getValidRegionWithDefault(const std::string& region_name) const; - void setRegion(const std::string& region_name); - void resetRegion() { regionID.reset(); }; - void setRegion(size_t id) { regionID = id; }; - void setRegion(std::optional id) { regionID = id; }; - std::optional getRegionID() const { return regionID; }; + void setRegion(const std::string& region_name) override; + void resetRegion() override { regionID.reset(); }; + void resetRegionParallel(); + void setRegion(size_t id) override { regionID = id; }; + void setRegion(std::optional id) override { regionID = id; }; + std::optional getRegionID() const override { return regionID; }; /// Return a Region reference to use to iterate over the x- and /// y-indices of this field @@ -343,7 +363,7 @@ public: * Direct access to the underlying data array * * If CHECK > 2 then bounds checking is performed - * + * * If CHECK <= 2 then no checks are performed, to * allow inlining and optimisation of inner loops */ @@ -455,7 +475,7 @@ public: ///@} // FieldData virtual functions - bool is3D() const override { return true; } + FieldType field_type() const override { return FieldType::field3d; } #if CHECK > 0 void doneComms() override { bndry_xin = bndry_xout = bndry_yup = bndry_ydown = true; } @@ -466,7 +486,7 @@ public: friend class Vector3D; friend class Vector2D; - Field3D& calcParallelSlices(); + void calcParallelSlices() override; void applyBoundary(bool init = false) override; void applyBoundary(BoutReal t); @@ -481,6 +501,7 @@ public: /// Note: does not just copy values in boundary region. void setBoundaryTo(const Field3D& f3d); + using FieldData::applyParallelBoundary; void applyParallelBoundary() override; void applyParallelBoundary(BoutReal t) override; void applyParallelBoundary(const std::string& condition) override; @@ -493,6 +514,11 @@ public: int size() const override { return nx * ny * nz; }; + std::weak_ptr getTracking() { return tracking; }; + + bool areCalcParallelSlicesAllowed() const { return _allowCalcParallelSlices; }; + void disallowCalcParallelSlices() { _allowCalcParallelSlices = false; }; + private: /// Array sizes (from fieldmesh). These are valid only if fieldmesh is not null int nx{-1}, ny{-1}, nz{-1}; @@ -508,6 +534,24 @@ private: /// RegionID over which the field is valid std::optional regionID; + + /// counter for tracking, to assign unique names to the variable names + int tracking_state{0}; + std::weak_ptr tracking; + + bool _allowCalcParallelSlices{true}; + // name is changed if we assign to the variable, while selfname is a + // non-changing copy that is used for the variable names in the dump files + std::string selfname; + template + void track(const T& change, const std::string& operation) { + if (tracking_state != 0) { + _track(change, operation); + } + } + template > + void _track(const T& change, std::string operation); + void _track(const BoutReal& change, std::string operation); }; // Non-member overloaded operators @@ -566,8 +610,8 @@ void checkData(const Field3D& f, const std::string& region = "RGN_NOBNDRY"); #else /// Ignored with disabled CHECK; Throw an exception if \p f is not /// allocated or if any elements are non-finite (for CHECK > 2) -inline void checkData(const Field3D& UNUSED(f), - const std::string& UNUSED(region) = "RGN_NOBNDRY"){}; +inline void checkData([[maybe_unused]] const Field3D& f, + [[maybe_unused]] const std::string& region = "RGN_NOBNDRY") {}; #endif /// Fourier filtering, removes all except one mode @@ -628,7 +672,7 @@ Field2D DC(const Field3D& f, const std::string& rgn = "RGN_ALL"); #if CHECK > 2 void invalidateGuards(Field3D& var); #else -inline void invalidateGuards(Field3D& UNUSED(var)) {} +inline void invalidateGuards([[maybe_unused]] Field3D& var) {} #endif /// Returns a reference to the time-derivative of a field \p f @@ -639,7 +683,7 @@ inline Field3D& ddt(Field3D& f) { return *(f.timeDeriv()); } /// toString template specialisation /// Defined in utils.hxx template <> -inline std::string toString<>(const Field3D& UNUSED(val)) { +inline std::string toString<>([[maybe_unused]] const Field3D& val) { return ""; } diff --git a/include/bout/field_data.hxx b/include/bout/field_data.hxx index 185dcabf2d..2f0d17a36d 100644 --- a/include/bout/field_data.hxx +++ b/include/bout/field_data.hxx @@ -6,7 +6,7 @@ * Copyright 2010 B.D.Dudson, S.Farley, M.V.Umansky, X.Q.Xu * * Contact: Ben Dudson, bd512@york.ac.uk - * + * * This file is part of BOUT++. * * BOUT++ is free software: you can redistribute it and/or modify @@ -33,6 +33,7 @@ class FieldData; #include "bout/bout_types.hxx" #include "bout/unused.hxx" +#include #include #include #include @@ -71,13 +72,22 @@ public: /// Get variable location virtual CELL_LOC getLocation() const; + /// Enum to distinguish the different kinds of Fields + enum class FieldType : std::uint8_t { field3d, field2d, fieldperp }; + /// Is this an instance of `Field3D`, `Field2D`, or `FieldPerp`? + virtual FieldType field_type() const = 0; + // Defines interface which must be implemented /// True if variable is 3D - virtual bool is3D() const = 0; + [[deprecated("Use `field_type()` instead")]] + bool is3D() const { + return field_type() == FieldType::field3d; + } + /// Number of BoutReals in one element virtual int elementSize() const { return 1; } - virtual void doneComms(){}; // Notifies that communications done + virtual void doneComms() {}; // Notifies that communications done // Boundary conditions void setBoundary(const std::string& name); ///< Set the boundary conditions @@ -86,13 +96,13 @@ public: copyBoundary(const FieldData& f); ///< Copy the boundary conditions from another field virtual void applyBoundary(bool UNUSED(init) = false) {} - virtual void applyTDerivBoundary(){}; + virtual void applyTDerivBoundary() {}; - virtual void applyParallelBoundary(){}; - virtual void applyParallelBoundary(BoutReal UNUSED(t)){}; - virtual void applyParallelBoundary(const std::string& UNUSED(condition)){}; + virtual void applyParallelBoundary() {}; + virtual void applyParallelBoundary(BoutReal UNUSED(t)) {}; + virtual void applyParallelBoundary(const std::string& UNUSED(condition)) {}; virtual void applyParallelBoundary(const std::string& UNUSED(region), - const std::string& UNUSED(condition)){}; + const std::string& UNUSED(condition)) {}; // JMAD void addBndryFunction(FuncPtr userfunc, BndryLoc location); void addBndryGenerator(FieldGeneratorPtr gen, BndryLoc location); diff --git a/include/bout/fieldgroup.hxx b/include/bout/fieldgroup.hxx index 184766c6b8..440655e7ce 100644 --- a/include/bout/fieldgroup.hxx +++ b/include/bout/fieldgroup.hxx @@ -1,22 +1,23 @@ #ifndef BOUT_FIELDGROUP_H #define BOUT_FIELDGROUP_H -#include "bout/field_data.hxx" -#include - +#include #include #include #include -#include +class Field2D; +class Field3D; +class FieldPerp; +class Field; /// Group together fields for easier communication /// -/// Note: The FieldData class is used as a base class, -/// which is inherited by Field2D, Field3D, Vector2D and Vector3D -/// however Vector2D and Vector3D are stored by reference to their -/// components (x,y,z) as Field2D or Field3D objects. +/// Note: The `Field` class is used as a base class, +/// which is inherited by `Field2D`, `Field3D`, `FieldPerp`; +/// however `Vector2D` and `Vector3D` are stored by reference to their +/// components ``(x, y, z)`` as `Field2D` or `Field3D` objects. class FieldGroup { public: FieldGroup() = default; @@ -24,11 +25,9 @@ public: FieldGroup(FieldGroup&& other) = default; FieldGroup& operator=(const FieldGroup& other) = default; FieldGroup& operator=(FieldGroup&& other) = default; + ~FieldGroup() = default; - /// Constructor with a single FieldData \p f - FieldGroup(FieldData& f) { fvec.push_back(&f); } - - /// Constructor with a single Field3D \p f + FieldGroup(Field& f) { fvec.push_back(&f); } FieldGroup(Field3D& f) { fvec.push_back(&f); f3vec.push_back(&f); @@ -56,7 +55,7 @@ public: } /// Variadic constructor. Allows an arbitrary number of - /// FieldData arguments + /// Field arguments /// /// The explicit keyword prevents FieldGroup being constructed with arbitrary /// types. In particular arguments to add() cannot be implicitly converted @@ -78,12 +77,12 @@ public: return *this; } - /// Add a FieldData \p f to the group. + /// Add a Field \p f to the group. /// /// A pointer to this field will be stored internally, /// so the lifetime of this variable should be longer /// than the lifetime of this group. - void add(FieldData& f) { fvec.push_back(&f); } + void add(Field& f) { fvec.push_back(&f); } // Add a 3D field \p f, which goes into both vectors. // @@ -121,12 +120,8 @@ public: } /// Add multiple fields to this group - /// - /// This is a variadic template which allows Field3D objects to be - /// treated as a special case. An arbitrary number of fields can be - /// added. template - void add(FieldData& t, Ts&... ts) { + void add(Field& t, Ts&... ts) { add(t); // Add the first using functions above add(ts...); // Add the rest } @@ -165,16 +160,14 @@ public: } /// Iteration over all fields - using iterator = std::vector::iterator; - iterator begin() { return fvec.begin(); } - iterator end() { return fvec.end(); } + auto begin() { return fvec.begin(); } + auto end() { return fvec.end(); } /// Const iteration over all fields - using const_iterator = std::vector::const_iterator; - const_iterator begin() const { return fvec.begin(); } - const_iterator end() const { return fvec.end(); } + auto begin() const { return fvec.cbegin(); } + auto end() const { return fvec.cend(); } - const std::vector& get() const { return fvec; } + const std::vector& get() const { return fvec; } /// Iteration over 3D fields const std::vector& field3d() const { return f3vec; } @@ -183,8 +176,8 @@ public: void makeUnique(); private: - std::vector fvec; // Vector of fields - std::vector f3vec; // Vector of 3D fields + std::vector fvec; // Vector of fields + std::vector f3vec; // Vector of 3D fields }; /// Combine two FieldGroups diff --git a/include/bout/fieldperp.hxx b/include/bout/fieldperp.hxx index 6995308dbe..bc28cf9ce9 100644 --- a/include/bout/fieldperp.hxx +++ b/include/bout/fieldperp.hxx @@ -5,7 +5,7 @@ * Copyright 2010 B.D.Dudson, S.Farley, M.V.Umansky, X.Q.Xu * * Contact: Ben Dudson, bd512@york.ac.uk - * + * * This file is part of BOUT++. * * BOUT++ is free software: you can redistribute it and/or modify @@ -23,6 +23,9 @@ * **************************************************************************/ +#include +#include +#include class FieldPerp; #ifndef BOUT_FIELDPERP_H @@ -44,8 +47,8 @@ class Field3D; // #include "bout/field3d.hxx" /*! * Represents a 2D field perpendicular to the magnetic field - * at a particular index in Y, which only varies in X-Z. - * + * at a particular index in Y, which only varies in X-Z. + * * Primarily used inside field solvers */ class FieldPerp : public Field { @@ -58,7 +61,8 @@ public: FieldPerp(Mesh* fieldmesh = nullptr, CELL_LOC location_in = CELL_CENTRE, int yindex_in = -1, DirectionTypes directions_in = {YDirectionType::Standard, - ZDirectionType::Standard}); + ZDirectionType::Standard}, + std::optional regionID = {}); /*! * Copy constructor. After this the data @@ -157,6 +161,19 @@ public: return *this; } + FieldPerp& yup(std::vector::size_type UNUSED(index) = 0) { return *this; } + const FieldPerp& yup(std::vector::size_type UNUSED(index) = 0) const { + return *this; + } + + FieldPerp& ydown(std::vector::size_type UNUSED(index) = 0) { return *this; } + const FieldPerp& ydown(std::vector::size_type UNUSED(index) = 0) const { + return *this; + } + + FieldPerp& ynext(int UNUSED(dir)) { return *this; } + const FieldPerp& ynext(int UNUSED(dir)) const { return *this; } + /*! * Ensure that data array is allocated and unique */ @@ -191,7 +208,7 @@ public: /*! * Access to the underlying data array at a given x,z index - * + * * If CHECK > 2 then bounds checking is performed, otherwise * no checks are performed */ @@ -227,9 +244,9 @@ public: } /*! - * Access to the underlying data array. (X,Y,Z) indices for consistency with + * Access to the underlying data array. (X,Y,Z) indices for consistency with * other field types - * + * */ BoutReal& operator()(int jx, int UNUSED(jy), int jz) { return (*this)(jx, jz); } @@ -238,7 +255,7 @@ public: } /*! - * Addition, modifying in-place. + * Addition, modifying in-place. * This loops over the entire domain, including guard/boundary cells */ FieldPerp& operator+=(const FieldPerp& rhs); @@ -247,7 +264,7 @@ public: FieldPerp& operator+=(BoutReal rhs); /*! - * Subtraction, modifying in place. + * Subtraction, modifying in place. * This loops over the entire domain, including guard/boundary cells */ FieldPerp& operator-=(const FieldPerp& rhs); @@ -256,7 +273,7 @@ public: FieldPerp& operator-=(BoutReal rhs); /*! - * Multiplication, modifying in place. + * Multiplication, modifying in place. * This loops over the entire domain, including guard/boundary cells */ FieldPerp& operator*=(const FieldPerp& rhs); @@ -265,7 +282,7 @@ public: FieldPerp& operator*=(BoutReal rhs); /*! - * Division, modifying in place. + * Division, modifying in place. * This loops over the entire domain, including guard/boundary cells */ FieldPerp& operator/=(const FieldPerp& rhs); @@ -286,7 +303,7 @@ public: */ int getNz() const override { return nz; }; - bool is3D() const override { return false; } + FieldType field_type() const override { return FieldType::fieldperp; } friend void swap(FieldPerp& first, FieldPerp& second) noexcept; diff --git a/include/bout/fv_ops.hxx b/include/bout/fv_ops.hxx index 94007a57a2..0ec1fbe3ad 100644 --- a/include/bout/fv_ops.hxx +++ b/include/bout/fv_ops.hxx @@ -5,11 +5,11 @@ #ifndef BOUT_FV_OPS_H #define BOUT_FV_OPS_H +#include "bout/build_defines.hxx" #include "bout/field3d.hxx" #include "bout/globals.hxx" -#include "bout/vector2d.hxx" - #include "bout/utils.hxx" +#include "bout/vector2d.hxx" #include namespace FV { @@ -258,7 +258,7 @@ const Field3D Div_par(const Field3D& f_in, const Field3D& v_in, BoutReal flux_factor_lm = common_factor / (coord->dy(i, j - 1) * coord->J(i, j - 1)); #endif - for (int k = 0; k < mesh->LocalNz; k++) { + for (int k = mesh->zstart; k <= mesh->zend; k++) { #if BOUT_USE_METRIC_3D // For right cell boundaries BoutReal common_factor = diff --git a/include/bout/globalindexer.hxx b/include/bout/globalindexer.hxx index e756ead3b2..bd4203a092 100644 --- a/include/bout/globalindexer.hxx +++ b/include/bout/globalindexer.hxx @@ -86,7 +86,7 @@ public: int localSize = size(); MPI_Comm comm = - std::is_same_v ? fieldmesh->getXcomm() : BoutComm::get(); + std::is_same_v ? fieldmesh->getXZcomm() : BoutComm::get(); fieldmesh->getMpi().MPI_Scan(&localSize, &globalEnd, 1, MPI_INT, MPI_SUM, comm); globalEnd--; int counter = globalStart = globalEnd - size() + 1; diff --git a/include/bout/hypre_interface.hxx b/include/bout/hypre_interface.hxx index ae392de4f3..189e91d159 100644 --- a/include/bout/hypre_interface.hxx +++ b/include/bout/hypre_interface.hxx @@ -159,7 +159,7 @@ public: explicit HypreVector(IndexerPtr indConverter) : indexConverter(indConverter) { Mesh& mesh = *indConverter->getMesh(); const MPI_Comm comm = - std::is_same_v ? mesh.getXcomm() : BoutComm::get(); + std::is_same_v ? mesh.getXZcomm() : BoutComm::get(); HYPRE_BigInt jlower = indConverter->getGlobalStart(); HYPRE_BigInt jupper = jlower + indConverter->size() - 1; // inclusive end @@ -380,7 +380,7 @@ public: : hypre_matrix(new HYPRE_IJMatrix, MatrixDeleter{}), index_converter(indConverter) { Mesh* mesh = indConverter->getMesh(); const MPI_Comm comm = - std::is_same_v ? mesh->getXcomm() : BoutComm::get(); + std::is_same_v ? mesh->getXZcomm() : BoutComm::get(); parallel_transform = &mesh->getCoordinates()->getParallelTransform(); ilower = indConverter->getGlobalStart(); @@ -444,19 +444,19 @@ public: value = matrix->getVal(row, column); } Element& operator=(const Element& other) { - AUTO_TRACE(); + ASSERT3(finite(static_cast(other))); return *this = static_cast(other); } Element& operator=(BoutReal value_) { - AUTO_TRACE(); + ASSERT3(finite(value_)); value = value_; setValues(value); return *this; } Element& operator+=(BoutReal value_) { - AUTO_TRACE(); + ASSERT3(finite(value_)); auto column_position = std::find(cbegin(positions), cend(positions), column); if (column_position != cend(positions)) { @@ -812,7 +812,7 @@ public: "values are: gmres, bicgstab, pcg") .withDefault(HYPRE_SOLVER_TYPE::bicgstab); - comm = std::is_same_v ? mesh.getXcomm() : BoutComm::get(); + comm = std::is_same_v ? mesh.getXZcomm() : BoutComm::get(); auto print_level = options["hypre_print_level"] diff --git a/include/bout/index_derivs.hxx b/include/bout/index_derivs.hxx index 456f98f8c2..ccce9a7f5e 100644 --- a/include/bout/index_derivs.hxx +++ b/include/bout/index_derivs.hxx @@ -43,7 +43,6 @@ #include #include #include -#include #include #include @@ -84,7 +83,7 @@ class DerivativeType { public: template void standard(const T& var, T& result, const std::string& region) const { - AUTO_TRACE(); + ASSERT2(meta.derivType == DERIV::Standard || meta.derivType == DERIV::StandardSecond || meta.derivType == DERIV::StandardFourth) ASSERT2(var.getMesh()->getNguard(direction) >= nGuards); @@ -98,7 +97,7 @@ public: template void upwindOrFlux(const T& vel, const T& var, T& result, const std::string& region) const { - AUTO_TRACE(); + ASSERT2(meta.derivType == DERIV::Upwind || meta.derivType == DERIV::Flux) ASSERT2(var.getMesh()->getNguard(direction) >= nGuards); @@ -133,7 +132,7 @@ struct registerMethod { template void operator()(Direction, Stagger, FieldTypeContainer, Method) { - AUTO_TRACE(); + using namespace std::placeholders; // Now we want to get the actual field type out of the TypeContainer diff --git a/include/bout/index_derivs_interface.hxx b/include/bout/index_derivs_interface.hxx index 8f7e41a68e..242492c3f8 100644 --- a/include/bout/index_derivs_interface.hxx +++ b/include/bout/index_derivs_interface.hxx @@ -45,7 +45,6 @@ namespace index { template T flowDerivative(const T& vel, const T& f, CELL_LOC outloc, const std::string& method, const std::string& region) { - AUTO_TRACE(); // Checks static_assert(bout::utils::is_Field2D_v || bout::utils::is_Field3D_v, @@ -110,7 +109,6 @@ T flowDerivative(const T& vel, const T& f, CELL_LOC outloc, const std::string& m template T standardDerivative(const T& f, CELL_LOC outloc, const std::string& method, const std::string& region) { - AUTO_TRACE(); // Checks static_assert(bout::utils::is_Field2D_v || bout::utils::is_Field3D_v, @@ -174,14 +172,14 @@ T standardDerivative(const T& f, CELL_LOC outloc, const std::string& method, template T DDX(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "DEFAULT", const std::string& region = "RGN_NOBNDRY") { - AUTO_TRACE(); + return standardDerivative(f, outloc, method, region); } template T D2DX2(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "DEFAULT", const std::string& region = "RGN_NOBNDRY") { - AUTO_TRACE(); + return standardDerivative(f, outloc, method, region); } @@ -189,7 +187,7 @@ T D2DX2(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = template T D4DX4(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "DEFAULT", const std::string& region = "RGN_NOBNDRY") { - AUTO_TRACE(); + return standardDerivative(f, outloc, method, region); } @@ -199,7 +197,7 @@ T D4DX4(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = template T DDY(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "DEFAULT", const std::string& region = "RGN_NOBNDRY") { - AUTO_TRACE(); + if (f.hasParallelSlices()) { ASSERT1(f.getDirectionY() == YDirectionType::Standard); return standardDerivative(f, outloc, @@ -216,7 +214,7 @@ T DDY(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "D template T D2DY2(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "DEFAULT", const std::string& region = "RGN_NOBNDRY") { - AUTO_TRACE(); + if (f.hasParallelSlices()) { ASSERT1(f.getDirectionY() == YDirectionType::Standard); return standardDerivative( @@ -233,7 +231,7 @@ T D2DY2(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = template T D4DY4(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "DEFAULT", const std::string& region = "RGN_NOBNDRY") { - AUTO_TRACE(); + if (f.hasParallelSlices()) { ASSERT1(f.getDirectionY() == YDirectionType::Standard); return standardDerivative( @@ -251,14 +249,14 @@ T D4DY4(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = template T DDZ(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "DEFAULT", const std::string& region = "RGN_NOBNDRY") { - AUTO_TRACE(); + return standardDerivative(f, outloc, method, region); } template T D2DZ2(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "DEFAULT", const std::string& region = "RGN_NOBNDRY") { - AUTO_TRACE(); + return standardDerivative(f, outloc, method, region); } @@ -266,7 +264,7 @@ T D2DZ2(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = template T D4DZ4(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "DEFAULT", const std::string& region = "RGN_NOBNDRY") { - AUTO_TRACE(); + return standardDerivative(f, outloc, method, region); } @@ -291,14 +289,14 @@ T D4DZ4(const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = template T VDDX(const T& vel, const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "DEFAULT", const std::string& region = "RGN_NOBNDRY") { - AUTO_TRACE(); + return flowDerivative(vel, f, outloc, method, region); } template T FDDX(const T& vel, const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "DEFAULT", const std::string& region = "RGN_NOBNDRY") { - AUTO_TRACE(); + return flowDerivative(vel, f, outloc, method, region); } @@ -307,7 +305,6 @@ T FDDX(const T& vel, const T& f, CELL_LOC outloc = CELL_DEFAULT, template T VDDY(const T& vel, const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "DEFAULT", const std::string& region = "RGN_NOBNDRY") { - AUTO_TRACE(); // Note the following chunk is copy+pasted from flowDerivative // above. Not pulled out as a separate function due the number of @@ -363,7 +360,7 @@ T VDDY(const T& vel, const T& f, CELL_LOC outloc = CELL_DEFAULT, template T FDDY(const T& vel, const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "DEFAULT", const std::string& region = "RGN_NOBNDRY") { - AUTO_TRACE(); + const bool fHasParallelSlices = (f.hasParallelSlices()); const bool velHasParallelSlices = (vel.hasParallelSlices()); if (fHasParallelSlices && velHasParallelSlices) { @@ -389,14 +386,14 @@ T FDDY(const T& vel, const T& f, CELL_LOC outloc = CELL_DEFAULT, template T VDDZ(const T& vel, const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "DEFAULT", const std::string& region = "RGN_NOBNDRY") { - AUTO_TRACE(); + return flowDerivative(vel, f, outloc, method, region); } template T FDDZ(const T& vel, const T& f, CELL_LOC outloc = CELL_DEFAULT, const std::string& method = "DEFAULT", const std::string& region = "RGN_NOBNDRY") { - AUTO_TRACE(); + return flowDerivative(vel, f, outloc, method, region); } diff --git a/include/bout/interpolation.hxx b/include/bout/interpolation.hxx index 2c7df4472d..1753dfe0c0 100644 --- a/include/bout/interpolation.hxx +++ b/include/bout/interpolation.hxx @@ -56,7 +56,7 @@ inline BoutReal interp(const stencil& s) { */ template const T interp_to(const T& var, CELL_LOC loc, const std::string region = "RGN_ALL") { - AUTO_TRACE(); + static_assert(bout::utils::is_Field2D_v || bout::utils::is_Field3D_v, "interp_to must be templated with one of Field2D or Field3D."); ASSERT1(loc != CELL_DEFAULT); // doesn't make sense to interplote to CELL_DEFAULT diff --git a/include/bout/interpolation_xz.hxx b/include/bout/interpolation_xz.hxx index 6c7419f7e4..1a8be9ef6b 100644 --- a/include/bout/interpolation_xz.hxx +++ b/include/bout/interpolation_xz.hxx @@ -1,8 +1,7 @@ /************************************************************************** - * Copyright 2010-2020 B.D.Dudson, S.Farley, P. Hill, J.T. Omotani, J.T. Parker, - * M.V.Umansky, X.Q.Xu + * Copyright 2010 - 2026 BOUT++ contributors * - * Contact: Ben Dudson, bd512@york.ac.uk + * Contact: Ben Dudson, dudson2@llnl.gov * * This file is part of BOUT++. * @@ -24,18 +23,23 @@ #ifndef BOUT_INTERP_XZ_H #define BOUT_INTERP_XZ_H -#include "bout/mask.hxx" +#include +#include +#include +#include -#define USE_NEW_WEIGHTS 1 #if BOUT_HAS_PETSC -#define HS_USE_PETSC 1 -#endif - -#ifdef HS_USE_PETSC #include "bout/petsclib.hxx" #endif +namespace bout { +namespace details { +enum class implementation_type { new_weights, petsc, legacy }; +} +} // namespace bout + class Options; +class GlobalField3DAccess; /// Interpolate a field onto a perturbed set of points const Field3D interpolate(const Field3D& f, const Field3D& delta_x, @@ -133,7 +137,16 @@ public: } }; -class XZHermiteSpline : public XZInterpolation { +/// Monotonic Hermite spline interpolator +/// +/// Similar to XZHermiteSpline, so uses most of the same code. +/// Forces the interpolated result to be in the range of the +/// neighbouring cell values. This prevents unphysical overshoots, +/// but also degrades accuracy near maxima and minima. +/// Perhaps should only impose near boundaries, since that is where +/// problems most obviously occur. +template +class XZHermiteSplineBase : public XZInterpolation { protected: /// This is protected rather than private so that it can be /// extended and used by HermiteSplineMonotonic @@ -141,6 +154,9 @@ protected: Tensor> i_corner; // index of bottom-left grid point Tensor k_corner; // z-index of bottom-left grid point + std::unique_ptr gf3daccess; + Tensor> g3dinds; + // Basis functions for cubic Hermite spline interpolation // see http://en.wikipedia.org/wiki/Cubic_Hermite_spline // The h00 and h01 basis functions are applied to the function itself @@ -158,22 +174,28 @@ protected: std::vector newWeights; -#if HS_USE_PETSC +#if BOUT_HAS_PETSC PetscLib* petsclib; bool isInit{false}; - Mat petscWeights; - Vec rhs, result; + Mat petscWeights{}; + Vec rhs{}, result{}; #endif + /// Factors to allow for some wiggleroom + BoutReal abs_fac_monotonic{1e-2}; + BoutReal rel_fac_monotonic{1e-1}; + public: - XZHermiteSpline(Mesh* mesh = nullptr) : XZHermiteSpline(0, mesh) {} - XZHermiteSpline(int y_offset = 0, Mesh* mesh = nullptr); - XZHermiteSpline(const BoutMask& mask, int y_offset = 0, Mesh* mesh = nullptr) - : XZHermiteSpline(y_offset, mesh) { + XZHermiteSplineBase(Mesh* mesh = nullptr, [[maybe_unused]] Options* options = nullptr) + : XZHermiteSplineBase(0, mesh, options) {} + XZHermiteSplineBase(int y_offset = 0, Mesh* mesh = nullptr, Options* options = nullptr); + XZHermiteSplineBase(const BoutMask& mask, int y_offset = 0, Mesh* mesh = nullptr, + Options* options = nullptr) + : XZHermiteSplineBase(y_offset, mesh, options) { setRegion(regionFromMask(mask, localmesh)); } - ~XZHermiteSpline() { -#if HS_USE_PETSC + ~XZHermiteSplineBase() override { +#if BOUT_HAS_PETSC if (isInit) { MatDestroy(&petscWeights); VecDestroy(&rhs); @@ -202,41 +224,25 @@ public: getWeightsForYApproximation(int i, int j, int k, int yoffset) override; }; -/// Monotonic Hermite spline interpolator -/// -/// Similar to XZHermiteSpline, so uses most of the same code. -/// Forces the interpolated result to be in the range of the -/// neighbouring cell values. This prevents unphysical overshoots, -/// but also degrades accuracy near maxima and minima. -/// Perhaps should only impose near boundaries, since that is where -/// problems most obviously occur. -class XZMonotonicHermiteSpline : public XZHermiteSpline { -public: - XZMonotonicHermiteSpline(Mesh* mesh = nullptr) : XZHermiteSpline(0, mesh) { - if (localmesh->getNXPE() > 1) { - throw BoutException("Do not support MPI splitting in X"); - } - } - XZMonotonicHermiteSpline(int y_offset = 0, Mesh* mesh = nullptr) - : XZHermiteSpline(y_offset, mesh) { - if (localmesh->getNXPE() > 1) { - throw BoutException("Do not support MPI splitting in X"); - } - } - XZMonotonicHermiteSpline(const BoutMask& mask, int y_offset = 0, Mesh* mesh = nullptr) - : XZHermiteSpline(mask, y_offset, mesh) { - if (localmesh->getNXPE() > 1) { - throw BoutException("Do not support MPI splitting in X"); - } - } - - using XZHermiteSpline::interpolate; - /// Interpolate using precalculated weights. - /// This function is called by the other interpolate functions - /// in the base class XZHermiteSpline. - Field3D interpolate(const Field3D& f, - const std::string& region = "RGN_NOBNDRY") const override; -}; +using XZMonotonicHermiteSplineSerial = + XZHermiteSplineBase; +using XZHermiteSplineSerial = + XZHermiteSplineBase; +using XZMonotonicHermiteSplineLegacy = + XZHermiteSplineBase; +using XZHermiteSplineLegacy = + XZHermiteSplineBase; +#if BOUT_HAS_PETSC +using XZMonotonicHermiteSpline = + XZHermiteSplineBase; +using XZHermiteSpline = + XZHermiteSplineBase; +#else +using XZMonotonicHermiteSpline = + XZHermiteSplineBase; +using XZHermiteSpline = + XZHermiteSplineBase; +#endif /// XZLagrange4pt interpolation class /// @@ -248,7 +254,8 @@ class XZLagrange4pt : public XZInterpolation { Field3D t_x, t_z; public: - XZLagrange4pt(Mesh* mesh = nullptr) : XZLagrange4pt(0, mesh) {} + XZLagrange4pt(Mesh* mesh = nullptr, [[maybe_unused]] Options* options = nullptr) + : XZLagrange4pt(0, mesh) {} XZLagrange4pt(int y_offset = 0, Mesh* mesh = nullptr); XZLagrange4pt(const BoutMask& mask, int y_offset = 0, Mesh* mesh = nullptr) : XZLagrange4pt(y_offset, mesh) { @@ -284,7 +291,8 @@ class XZBilinear : public XZInterpolation { Field3D w0, w1, w2, w3; public: - XZBilinear(Mesh* mesh = nullptr) : XZBilinear(0, mesh) {} + XZBilinear(Mesh* mesh = nullptr, [[maybe_unused]] Options* options = nullptr) + : XZBilinear(0, mesh) {} XZBilinear(int y_offset = 0, Mesh* mesh = nullptr); XZBilinear(const BoutMask& mask, int y_offset = 0, Mesh* mesh = nullptr) : XZBilinear(y_offset, mesh) { @@ -308,7 +316,7 @@ public: }; class XZInterpolationFactory - : public Factory { + : public Factory { public: static constexpr auto type_name = "XZInterpolation"; static constexpr auto section_name = "xzinterpolation"; @@ -316,10 +324,10 @@ public: static constexpr auto default_type = "hermitespline"; ReturnType create(Options* options = nullptr, Mesh* mesh = nullptr) const { - return Factory::create(getType(options), mesh); + return Factory::create(getType(options), mesh, options); } - ReturnType create(const std::string& type, [[maybe_unused]] Options* options) const { - return Factory::create(type, nullptr); + ReturnType create(const std::string& type, Options* options) const { + return Factory::create(type, nullptr, options); } static void ensureRegistered(); diff --git a/include/bout/invert_laplace.hxx b/include/bout/invert_laplace.hxx index 187056d115..73e58e8483 100644 --- a/include/bout/invert_laplace.hxx +++ b/include/bout/invert_laplace.hxx @@ -40,6 +40,7 @@ class Laplacian; #define PVEC_REAL_MPI_TYPE MPI_DOUBLE #endif +#include "bout/bout_types.hxx" #include "bout/field2d.hxx" #include "bout/field3d.hxx" #include "bout/fieldperp.hxx" @@ -50,6 +51,8 @@ class Laplacian; #include "bout/dcomplex.hxx" +class DSTTransform; +class FFTTransform; class Solver; constexpr auto LAPLACE_SPT = "spt"; @@ -138,7 +141,11 @@ public: static constexpr auto type_name = "Laplacian"; static constexpr auto section_name = "laplace"; static constexpr auto option_name = "type"; +#if BOUT_USE_METRIC_3D + static constexpr auto default_type = LAPLACE_PETSC; +#else static constexpr auto default_type = LAPLACE_CYCLIC; +#endif ReturnType create(Options* options = nullptr, CELL_LOC loc = CELL_CENTRE, Mesh* mesh = nullptr, Solver* solver = nullptr) { @@ -258,7 +265,7 @@ public: /// Coefficients in tridiagonal inversion void tridagCoefs(int jx, int jy, int jz, dcomplex& a, dcomplex& b, dcomplex& c, const Field2D* ccoef = nullptr, const Field2D* d = nullptr, - CELL_LOC loc = CELL_DEFAULT); + CELL_LOC loc = CELL_DEFAULT) const; /*! * Create a new Laplacian solver @@ -301,6 +308,10 @@ public: void savePerformance(Solver& solver, const std::string& name); protected: + // Give access for tridagMatrix + friend class DSTTransform; + friend class FFTTransform; + bool async_send; ///< If true, use asyncronous send in parallel algorithms int maxmode; ///< The maximum Z mode to solve for @@ -332,23 +343,24 @@ protected: void tridagCoefs(int jx, int jy, BoutReal kwave, dcomplex& a, dcomplex& b, dcomplex& c, const Field2D* ccoef = nullptr, const Field2D* d = nullptr, - CELL_LOC loc = CELL_DEFAULT) { + CELL_LOC loc = CELL_DEFAULT) const { tridagCoefs(jx, jy, kwave, a, b, c, ccoef, ccoef, d, loc); } void tridagCoefs(int jx, int jy, BoutReal kwave, dcomplex& a, dcomplex& b, dcomplex& c, const Field2D* c1coef, const Field2D* c2coef, const Field2D* d, - CELL_LOC loc = CELL_DEFAULT); + CELL_LOC loc = CELL_DEFAULT) const; void tridagMatrix(dcomplex* avec, dcomplex* bvec, dcomplex* cvec, dcomplex* bk, int jy, int kz, BoutReal kwave, const Field2D* a, const Field2D* ccoef, - const Field2D* d, bool includeguards = true, bool zperiodic = true) { + const Field2D* d, bool includeguards = true, + bool zperiodic = true) const { tridagMatrix(avec, bvec, cvec, bk, jy, kz, kwave, a, ccoef, ccoef, d, includeguards, zperiodic); } void tridagMatrix(dcomplex* avec, dcomplex* bvec, dcomplex* cvec, dcomplex* bk, int jy, int kz, BoutReal kwave, const Field2D* a, const Field2D* c1coef, const Field2D* c2coef, const Field2D* d, bool includeguards = true, - bool zperiodic = true); + bool zperiodic = true) const; CELL_LOC location; ///< staggered grid location of this solver Mesh* localmesh; ///< Mesh object for this solver Coordinates* coords; ///< Coordinates object, so we only have to call diff --git a/include/bout/invertable_operator.hxx b/include/bout/invertable_operator.hxx index 9b9dbba41a..6dcd92897a 100644 --- a/include/bout/invertable_operator.hxx +++ b/include/bout/invertable_operator.hxx @@ -42,7 +42,6 @@ class InvertableOperator; #include #include #include -#include #include #include #include @@ -61,14 +60,14 @@ namespace inversion { /// No-op function to use as a default -- may wish to remove once testing phase complete template T identity(const T& in) { - AUTO_TRACE(); + return in; }; /// Pack a PetscVec from a Field template PetscErrorCode fieldToPetscVec(const T& in, Vec out) { - TRACE("fieldToPetscVec"); + Timer timer("invertable_operator_packing"); PetscScalar* vecData; @@ -93,7 +92,7 @@ PetscErrorCode fieldToPetscVec(const T& in, Vec out) { /// Pack a Field from a PetscVec template PetscErrorCode petscVecToField(Vec in, T& out) { - TRACE("petscVecToField"); + Timer timer("invertable_operator_packing"); const PetscScalar* vecData; @@ -137,12 +136,11 @@ public: opt(optIn == nullptr ? Options::getRoot()->getSection("invertableOperator") : optIn), localmesh(localmeshIn == nullptr ? bout::globals::mesh : localmeshIn), lib(opt) { - AUTO_TRACE(); - }; + + }; /// Destructor just has to cleanup the PETSc owned objects. ~InvertableOperator() { - TRACE("InvertableOperator::destructor"); KSPDestroy(&ksp); MatDestroy(&matOperator); @@ -157,7 +155,7 @@ public: /// do this they can set alsoSetPreconditioner to false. void setOperatorFunction(const function_signature& func, bool alsoSetPreconditioner = true) { - TRACE("InvertableOperator::setOperatorFunction"); + operatorFunction = func; if (alsoSetPreconditioner) { preconditionerFunction = func; @@ -166,28 +164,21 @@ public: /// Allow the user to override the existing preconditioner function void setPreconditionerFunction(const function_signature& func) { - TRACE("InvertableOperator::setPreconditionerFunction"); + preconditionerFunction = func; } /// Provide a way to apply the operator to a Field - T operator()(const T& input) { - TRACE("InvertableOperator::operator()"); - return operatorFunction(input); - } + T operator()(const T& input) { return operatorFunction(input); } /// Provide a synonym for applying the operator to a Field - T apply(const T& input) { - AUTO_TRACE(); - return operator()(input); - } + T apply(const T& input) { return operator()(input); } /// Sets up the PETSc objects required for inverting the operator /// Currently also takes the functor that applies the operator this class /// represents. Not actually required by any of the setup so this should /// probably be moved to a separate place (maybe the constructor). PetscErrorCode setup() { - TRACE("InvertableOperator::setup"); Timer timer("invertable_operator_setup"); if (doneSetup) { @@ -386,7 +377,7 @@ public: // but suspect it's not as there are KSPGuess objects // to deal with. T invert(const T& rhsField, const T& guess) { - AUTO_TRACE(); + auto ierr = fieldToPetscVec(guess, lhs); CHKERRQ(ierr); return invert(rhsField); @@ -397,7 +388,7 @@ public: /// of the operator we represent. Should probably provide an overload or similar as a /// way of setting the initial guess. T invert(const T& rhsField) { - TRACE("InvertableOperator::invert"); + Timer timer("invertable_operator_invert"); if (!doneSetup) { @@ -444,7 +435,6 @@ public: /// applying the registered function on the calculated inverse gives /// back the initial values. bool verify(const T& rhsIn, BoutReal tol = 1.0e-5) { - TRACE("InvertableOperator::verify"); T result = invert(rhsIn); localmesh->communicate(result); @@ -463,7 +453,7 @@ public: /// that as the Timer "labels" are not unique to an instance the time /// reported is summed across all different instances. static void reportTime() { - TRACE("InvertableOperator::reportTime"); + BoutReal time_setup = Timer::resetTime("invertable_operator_setup"); BoutReal time_invert = Timer::resetTime("invertable_operator_invert"); BoutReal time_packing = Timer::resetTime("invertable_operator_packing"); @@ -503,7 +493,7 @@ private: /// Copies data from v1 into a field of type T, calls the function on this and then /// copies the result into the v2 argument. static PetscErrorCode functionWrapper(Mat m, Vec v1, Vec v2) { - TRACE("InvertableOperator::functionWrapper"); + InvertableOperator* ctx; auto ierr = MatShellGetContext(m, &ctx); T tmpField(ctx->localmesh); @@ -534,7 +524,7 @@ private: /// Copies data from v1 into a field of type T, calls the function on this and then /// copies the result into the v2 argument. static PetscErrorCode preconditionerWrapper(Mat m, Vec v1, Vec v2) { - TRACE("InvertableOperator::functionWrapper"); + InvertableOperator* ctx; auto ierr = MatShellGetContext(m, &ctx); T tmpField(ctx->localmesh); diff --git a/include/bout/mask.hxx b/include/bout/mask.hxx index fd90ae7345..9033080d44 100644 --- a/include/bout/mask.hxx +++ b/include/bout/mask.hxx @@ -26,7 +26,7 @@ #include "bout/globals.hxx" #include "bout/mesh.hxx" -#include "bout/msg_stack.hxx" +#include "bout/region.hxx" /** * 3D array of bools to mask Field3Ds @@ -81,4 +81,12 @@ inline std::unique_ptr> regionFromMask(const BoutMask& mask, } return std::make_unique>(indices); } + +inline BoutMask maskFromRegion(const Region& region, const Mesh* mesh) { + BoutMask mask{mesh, false}; + + BOUT_FOR(i, region) { mask[i] = true; } + return mask; +} + #endif //BOUT_MASK_H diff --git a/include/bout/mesh.hxx b/include/bout/mesh.hxx index b5a51015df..ca03548a31 100644 --- a/include/bout/mesh.hxx +++ b/include/bout/mesh.hxx @@ -3,7 +3,7 @@ * * Interface for mesh classes. Contains standard variables and useful * routines. - * + * * Changelog * ========= * @@ -11,7 +11,7 @@ * * Removing coordinate system into separate * Coordinates class * * Adding index derivative functions from derivs.cxx - * + * * 2010-06 Ben Dudson, Sean Farley * * Initial version, adapted from GridData class * * Incorporates code from topology.cpp and Communicator @@ -20,7 +20,7 @@ * Copyright 2010-2025 BOUT++ contributors * * Contact: Ben Dudson, dudson2@llnl.gov - * + * * This file is part of BOUT++. * * BOUT++ is free software: you can redistribute it and/or modify @@ -58,13 +58,12 @@ class Mesh; #include "bout/sys/range.hxx" // RangeIterator #include "bout/unused.hxx" -#include "mpi.h" - #include #include #include #include #include +#include class BoundaryRegion; class BoundaryRegionPar; @@ -301,11 +300,6 @@ public: /// @param g The group of fields to communicate. Guard cells will be modified void communicateYZ(FieldGroup& g); - /*! - * Communicate an X-Z field - */ - virtual void communicate(FieldPerp& f); - /*! * Send a list of FieldData objects * Packs arguments into a FieldGroup and passes @@ -354,10 +348,14 @@ public: // non-local communications - virtual int getNXPE() = 0; ///< The number of processors in the X direction - virtual int getNYPE() = 0; ///< The number of processors in the Y direction - virtual int getXProcIndex() = 0; ///< This processor's index in X direction - virtual int getYProcIndex() = 0; ///< This processor's index in Y direction + virtual int getNXPE() const = 0; ///< The number of processors in the X direction + virtual int getNYPE() const = 0; ///< The number of processors in the Y direction + virtual int getNZPE() const = 0; ///< The number of processors in the Z direction + virtual int getXProcIndex() const = 0; ///< This processor's index in X direction + virtual int getYProcIndex() const = 0; ///< This processor's index in Y direction + virtual int getZProcIndex() const = 0; ///< This processor's index in Z direction + /// The index of a processor with given X, Y, and Z index + virtual int getProcIndex(int X, int Y, int Z) const = 0; // X communications virtual bool firstX() @@ -368,8 +366,6 @@ public: /// Domain is periodic in X? bool periodicX{false}; - int NXPE, PE_XIND; ///< Number of processors in X, and X processor index - /// Send a buffer of data to processor at X index +1 /// /// @param[in] buffer The data to send. Must be at least length \p size @@ -403,6 +399,7 @@ public: } ///< Return communicator containing all processors in X virtual MPI_Comm getXcomm(int jy) const = 0; ///< Return X communicator virtual MPI_Comm getYcomm(int jx) const = 0; ///< Return Y communicator + virtual MPI_Comm getXZcomm() const = 0; ///< Communicator in X-Z /// Return pointer to the mesh's MPI Wrapper object MpiWrapper& getMpi() { return *mpi; } @@ -505,13 +502,12 @@ public: virtual void addBoundaryPar(std::shared_ptr UNUSED(bndry), BoundaryParType UNUSED(type)) {} - /// Branch-cut special handling (experimental) - virtual Field3D smoothSeparatrix(const Field3D& f) { return f; } - virtual BoutReal GlobalX(int jx) const = 0; ///< Continuous X index between 0 and 1 virtual BoutReal GlobalY(int jy) const = 0; ///< Continuous Y index (0 -> 1) + virtual BoutReal GlobalZ(int jz) const = 0; ///< Continuous Z index (0 -> 1) virtual BoutReal GlobalX(BoutReal jx) const = 0; ///< Continuous X index between 0 and 1 virtual BoutReal GlobalY(BoutReal jy) const = 0; ///< Continuous Y index (0 -> 1) + virtual BoutReal GlobalZ(BoutReal jz) const = 0; ///< Continuous Z index (0 -> 1) ////////////////////////////////////////////////////////// @@ -601,7 +597,7 @@ public: virtual int getLocalZIndexNoBoundaries(int zglobal) const = 0; /// Size of the mesh on this processor including guard/boundary cells - int LocalNx, LocalNy, LocalNz; + int LocalNx{0}, LocalNy{0}, LocalNz{0}; /// Local ranges of data (inclusive), excluding guard cells int xstart, xend, ystart, yend, zstart, zend; @@ -652,7 +648,7 @@ public: /// Returns the non-CELL_CENTRE location /// allowed as a staggered location static CELL_LOC getAllowedStaggerLoc(DIRECTION direction) { - AUTO_TRACE(); + switch (direction) { case (DIRECTION::X): return CELL_XLOW; @@ -670,7 +666,7 @@ public: /// Returns the number of grid points in the /// particular direction int getNpoints(DIRECTION direction) const { - AUTO_TRACE(); + switch (direction) { case (DIRECTION::X): return LocalNx; @@ -688,7 +684,7 @@ public: /// Returns the number of guard points in the /// particular direction int getNguard(DIRECTION direction) const { - AUTO_TRACE(); + switch (direction) { case (DIRECTION::X): return xstart; @@ -767,7 +763,7 @@ public: void addRegionPerp(const std::string& region_name, const Region& region); /// Converts an Ind2D to an Ind3D using calculation - Ind3D ind2Dto3D(const Ind2D& ind2D, int jz = 0) { + Ind3D ind2Dto3D(const Ind2D& ind2D, int jz = 0) const { return {ind2D.ind * LocalNz + jz, LocalNy, LocalNz}; } @@ -814,8 +810,8 @@ protected: const std::vector readInts(const std::string& name, int n); /// Calculates the size of a message for a given x and y range - int msg_len(const std::vector& var_list, int xge, int xlt, int yge, - int ylt); + int msg_len(const std::vector& var_list, int xge, int xlt, int yge, + int ylt) const; /// Initialise derivatives void derivs_init(Options* options); diff --git a/include/bout/msg_stack.hxx b/include/bout/msg_stack.hxx index 6d19469e25..ddc0ccad98 100644 --- a/include/bout/msg_stack.hxx +++ b/include/bout/msg_stack.hxx @@ -31,27 +31,14 @@ class MsgStack; #include "bout/build_defines.hxx" -#include "bout/unused.hxx" - +#include "fmt/base.h" #include "fmt/core.h" -#include +#include #include #include #include -/// The __PRETTY_FUNCTION__ variable is defined by GCC (and some other families) but is -/// not a part of the standard. The __func__ variable *is* a part of the c++11 standard so -/// we'd like to fall back to this if possible. However as these are variables/constants -/// and not macros we can't just check if __PRETTY_FUNCITON__ is defined or not. Instead -/// we need to say if we support this or not by defining BOUT_HAS_PRETTY_FUNCTION (to be -/// implemented in configure) -#if BOUT_HAS_PRETTY_FUNCTION -#define __thefunc__ __PRETTY_FUNCTION__ -#else -#define __thefunc__ __func__ -#endif - /*! * Message stack * @@ -74,9 +61,10 @@ public: int push(std::string message); int push() { return push(""); } - template - int push(const S& format, const Args&... args) { - return push(fmt::format(format, args...)); + template + // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) + int push(fmt::format_string format, Args&&... args) { + return push(fmt::vformat(format, fmt::make_format_args(args...))); } void pop(); ///< Remove the last message @@ -88,19 +76,22 @@ public: #else /// Dummy functions which should be optimised out int push(const std::string&) { return 0; } - template - int push(const S&, const Args&...) { + template + int push(fmt::format_string format, const Args&... args) { return 0; } void pop() {} - void pop(int UNUSED(id)) {} + void pop([[maybe_unused]] int id) {} void clear() {} void dump() {} std::string getDump() { return ""; } #endif + /// Current stack size + std::size_t size() const { return position; } + private: std::vector stack; ///< Message stack; std::vector::size_type position{0}; ///< Position in stack @@ -143,10 +134,13 @@ public: MsgStackItem(const std::string& file, int line, const char* msg) : point(msg_stack.push("{:s} on line {:d} of '{:s}'", msg, line, file)) {} - template - MsgStackItem(const std::string& file, int line, const S& msg, const Args&... args) - : point(msg_stack.push("{:s} on line {:d} of '{:s}'", fmt::format(msg, args...), - line, file)) {} + template + // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) + MsgStackItem(const std::string& file, int line, fmt::format_string msg, + Args&&... args) + : point(msg_stack.push("{:s} on line {:d} of '{:s}'", + fmt::vformat(msg, fmt::make_format_args(args...)), line, + file)) {} ~MsgStackItem() { // If an exception has occurred, don't pop the message if (exception_count == std::uncaught_exceptions()) { @@ -193,23 +187,4 @@ private: #define TRACE(...) #endif -/*! - * The AUTO_TRACE macro provides a convenient way to put messages onto the msg_stack - * It pushes a message onto the stack, and pops it when the scope ends - * The message is automatically derived from the function signature - * as identified by the compiler. This will be PRETTY_FUNCTION if available - * else it will be the mangled form. - * - * This is implemented as a use of the TRACE macro with specific arguments. - * - * Example - * ------- - * - * { - * AUTO_TRACE(); - * - * } // Scope ends, message popped - */ -#define AUTO_TRACE() TRACE(__thefunc__) // NOLINT - #endif // BOUT_MSG_STACK_H diff --git a/include/bout/options.hxx b/include/bout/options.hxx index 1c552cd4f9..48840a7b6f 100644 --- a/include/bout/options.hxx +++ b/include/bout/options.hxx @@ -52,6 +52,7 @@ class Options; #include "bout/utils.hxx" #include +#include #include #include @@ -73,19 +74,19 @@ class Options; * which can be used as a map. * * Options options; - * + * * // Set values * options["key"] = 1.0; * * // Get values. Throws BoutException if not found - * int val = options["key"]; // Sets val to 1 + * int val = options["key"]; // Sets val to 1 * * // Return as specified type. Throws BoutException if not found * BoutReal var = options["key"].as(); * * // A default value can be used if key is not found * BoutReal value = options["pi"].withDefault(3.14); - * + * * // Assign value with source label. Throws if already has a value from same source * options["newkey"].assign(1.0, "some source"); * @@ -93,7 +94,7 @@ class Options; * options["newkey"].force(2.0, "some source"); * * A legacy interface is also supported: - * + * * options.set("key", 1.0, "code"); // Sets a key from source "code" * * int val; @@ -118,9 +119,9 @@ class Options; * * Each Options object can also contain any number of sections, which are * themselves Options objects. - * + * * Options §ion = options["section"]; - * + * * which can be nested: * * options["section"]["subsection"]["value"] = 3; @@ -133,13 +134,13 @@ class Options; * * e.g. * options->getSection("section")->getSection("subsection")->set("value", 3); - * + * * Options also know about their parents: * * Options &parent = section.parent(); - * + * * or - * + * * Options *parent = section->getParent(); * * Root options object @@ -149,8 +150,8 @@ class Options; * there is a global singleton Options object which can be accessed with a static function * * Options &root = Options::root(); - * - * or + * + * or * * Options *root = Options::getRoot(); * @@ -192,7 +193,7 @@ public: /// @param[in] parent Parent object /// @param[in] sectionName Name of the section, including path from the root Options(Options* parent_instance, std::string full_name) - : parent_instance(parent_instance), full_name(std::move(full_name)){}; + : parent_instance(parent_instance), full_name(std::move(full_name)) {}; /// Initialise with a value /// These enable Options to be constructed using initializer lists @@ -626,8 +627,8 @@ public: // Option not found. Copy the value from the default. this->_set_no_check(def.value, DEFAULT_SOURCE); - output_info << _("\tOption ") << full_name << " = " << def.full_name << " (" - << DEFAULT_SOURCE << ")\n"; + output_info.write("{}{} = {}({})\n", _("\tOption "), full_name, def.full_name, + DEFAULT_SOURCE); } else { // Check if this was previously set as a default option if (bout::utils::variantEqualTo(attributes.at("source"), DEFAULT_SOURCE)) { @@ -911,8 +912,8 @@ private: << ")\n"; } else { throw BoutException( - _("Options: Setting a value from same source ({:s}) to new value " - "'{:s}' - old value was '{:s}'."), + _f("Options: Setting a value from same source ({:s}) to new value " + "'{:s}' - old value was '{:s}'."), source, toString(val), bout::utils::variantToString(value)); } } @@ -1012,6 +1013,9 @@ auto Options::as(const Tensor& similar_to) const -> Tensor; /// Convert \p value to string std::string toString(const Options& value); +/// Save the parallel fields +void saveParallel(Options& opt, const std::string& name, const Field3D& tosave); + /// Output a stringified \p value to a stream /// /// This is templated to avoid implict casting: anything is @@ -1043,7 +1047,40 @@ namespace details { /// so that we can put the function definitions in the .cxx file, /// avoiding lengthy recompilation if we change it struct OptionsFormatterBase { - auto parse(fmt::format_parse_context& ctx) -> fmt::format_parse_context::iterator; + constexpr auto parse(fmt::format_parse_context& ctx) { + const auto* it = ctx.begin(); + const auto* const end = ctx.end(); + + while (it != end and *it != '}') { + switch (*it) { + case 'd': + docstrings = true; + ++it; + break; + case 'i': + inline_section_names = true; + ++it; + break; + case 'k': + key_only = true; + ++it; + break; + case 's': + source = true; + ++it; + break; + case 'u': + unused = true; + ++it; + break; + default: + throw fmt::format_error("invalid format for 'Options'"); + } + } + + return it; + } + auto format(const Options& options, fmt::format_context& ctx) const -> fmt::format_context::iterator; @@ -1060,8 +1097,6 @@ private: bool key_only{false}; /// Include the 'source' attribute, if present bool source{false}; - /// Format string to passed down to subsections - std::string format_string; }; } // namespace details } // namespace bout diff --git a/include/bout/options_io.hxx b/include/bout/options_io.hxx index 509305aad0..77fdd4712b 100644 --- a/include/bout/options_io.hxx +++ b/include/bout/options_io.hxx @@ -1,3 +1,19 @@ +#pragma once + +#ifndef OPTIONS_IO_H +#define OPTIONS_IO_H + +#include "bout/build_defines.hxx" +#include "bout/generic_factory.hxx" +#include "bout/options.hxx" + +#include +#include + +class Mesh; + +namespace bout { + /// Parent class for IO to binary files and streams /// /// @@ -5,8 +21,8 @@ /// /// 1. Dump files, containing time history: /// -/// auto dump = OptionsIOFactory::getInstance().createOutput(); -/// dump->write(data); +/// auto dump = OptionsIOFactory::getInstance().createOutput(); +/// dump->write(data); /// /// where data is an Options tree. By default dump files are configured /// with the root `output` section, or an Option tree can be passed to @@ -14,36 +30,28 @@ /// /// 2. Restart files: /// -/// auto restart = OptionsIOFactory::getInstance().createOutput(); -/// restart->write(data); +/// auto restart = OptionsIOFactory::getInstance().createRestart(); +/// restart->write(data); /// /// where data is an Options tree. By default restart files are configured /// with the root `restart_files` section, or an Option tree can be passed to /// `createRestart`. /// /// 3. Ad-hoc single files -/// Note: The caller should consider how multiple processors interact with the file. +/// Note: The caller should consider how multiple processors interact with the file. /// -/// auto file = OptionsIOFactory::getInstance().createFile("some_file.nc"); -/// or -/// auto file = OptionsIO::create("some_file.nc"); +/// auto file = OptionsIOFactory::getInstance().createFile("some_file.nc"); +/// or +/// auto file = OptionsIO::create("some_file.nc"); /// +/// 4. Ad-hoc parallel files +/// This adds also metric information, such that the file can be read with +/// boutdata or xBOUT +/// +/// OptionIO::write("some_file", data, mesh); +/// +/// if mesh is omitted, no grid information is added. /// - -#pragma once - -#ifndef OPTIONS_IO_H -#define OPTIONS_IO_H - -#include "bout/build_defines.hxx" -#include "bout/generic_factory.hxx" -#include "bout/options.hxx" - -#include -#include - -namespace bout { - class OptionsIO { public: /// No default constructor, as settings are required @@ -51,7 +59,7 @@ public: /// Constructor specifies the kind of file, and options to control /// the name of file, mode of operation etc. - OptionsIO(Options&) {} + OptionsIO(Options& /*unused*/) {} virtual ~OptionsIO() = default; @@ -73,10 +81,18 @@ public: /// ADIOS: Indicate completion of an output step. virtual void verifyTimesteps() const = 0; + /// NetCDF: Flush file to disk + virtual void flush() {} + /// Create an OptionsIO for I/O to the given file. /// This uses the default file type and default options. static std::unique_ptr create(const std::string& file); + /// Write some data to a file with a given name prefix + /// This will be done in parallel. If Mesh is given, also mesh data will be + /// added, which is needed for xBOUT or boutdata to read the files. + static void write(const std::string& prefix, Options& data, Mesh* mesh = nullptr); + /// Create an OptionsIO for I/O to the given file. /// The file will be configured using the given `config` options: /// - "type" : string The file type e.g. "netcdf" or "adios" diff --git a/include/bout/optionsreader.hxx b/include/bout/optionsreader.hxx index de3d40514d..c5cda6ef1f 100644 --- a/include/bout/optionsreader.hxx +++ b/include/bout/optionsreader.hxx @@ -36,7 +36,8 @@ class OptionsReader; #include "bout/options.hxx" -#include "fmt/core.h" +#include +#include #include #include @@ -69,9 +70,10 @@ public: /// @param[in] file The name of the file. printf style arguments can be used to create the file name. void read(Options* options, const std::string& filename); - template - void read(Options* options, const S& format, const Args&... args) { - return read(options, fmt::format(format, args...)); + template + // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) + void read(Options* options, fmt::format_string format, Args&&... args) { + return read(options, fmt::vformat(format, fmt::make_format_args(args...))); } /// Write options to file @@ -80,9 +82,10 @@ public: /// @param[in] file The name of the file to (over)write void write(Options* options, const std::string& filename); - template - void write(Options* options, const S& format, const Args&... args) { - return write(options, fmt::format(format, args...)); + template + // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) + void write(Options* options, fmt::format_string format, Args&&... args) { + return write(options, fmt::vformat(format, fmt::make_format_args(args...))); } /// Parse options from the command line diff --git a/include/bout/output.hxx b/include/bout/output.hxx index 2862899067..43fc6c59bf 100644 --- a/include/bout/output.hxx +++ b/include/bout/output.hxx @@ -29,18 +29,20 @@ class Output; #ifndef BOUT_OUTPUT_H #define BOUT_OUTPUT_H -#include "bout/multiostream.hxx" #include -#include #include #include #include "bout/assert.hxx" -#include "bout/boutexception.hxx" -#include "bout/sys/gettext.hxx" // for gettext _() macro +#include "bout/build_defines.hxx" +#include "bout/multiostream.hxx" +#include "bout/sys/gettext.hxx" // IWYU pragma: keep for gettext _() macro #include "bout/unused.hxx" -#include "fmt/core.h" +#include +#include + +#include using std::endl; @@ -61,22 +63,29 @@ using std::endl; class Output : private multioutbuf_init>, public std::basic_ostream> { - using _Tr = std::char_traits; - using multioutbuf_init = ::multioutbuf_init; + using Tr = std::char_traits; + using multioutbuf_init = ::multioutbuf_init; public: - Output() : multioutbuf_init(), std::basic_ostream(multioutbuf_init::buf()) { + Output() : multioutbuf_init(), std::basic_ostream(multioutbuf_init::buf()) { Output::enable(); } + Output(const Output&) = delete; + Output(Output&&) = delete; + Output& operator=(const Output&) = delete; + Output& operator=(Output&&) = delete; + /// Specify a log file to open Output(const std::string& filename) { Output::enable(); open(filename); } - template - Output(const S& format, const Args&... args) : Output(fmt::format(format, args...)) {} + template + // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) + Output(fmt::format_string format, Args&&... args) + : Output(fmt::vformat(format, fmt::make_format_args(args...))) {} ~Output() override { close(); } @@ -86,9 +95,10 @@ public: /// Open an output log file int open(const std::string& filename); - template - int open(const S& format, const Args&... args) { - return open(fmt::format(format, args...)); + template + // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) + int open(fmt::format_string format, Args&&... args) { + return open(fmt::vformat(format, fmt::make_format_args(args...))); } /// Close the log file @@ -97,25 +107,25 @@ public: /// Write a string using fmt format virtual void write(const std::string& message); - template - void write(const S& format, const Args&... args) { - write(fmt::format(format, args...)); + template + // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) + void write(fmt::format_string format, Args&&... args) { + write(fmt::vformat(format, fmt::make_format_args(args...))); } /// Same as write, but only to screen virtual void print(const std::string& message); - template - void print(const S& format, const Args&... args) { - print(fmt::format(format, args...)); + template + // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) + void print(fmt::format_string format, Args&&... args) { + print(fmt::vformat(format, fmt::make_format_args(args...))); } /// Add an output stream. All output will be sent to all streams - void add(std::basic_ostream& str) { multioutbuf_init::buf()->add(str); } + void add(std::basic_ostream& str) { multioutbuf_init::buf()->add(str); } /// Remove an output stream - void remove(std::basic_ostream& str) { - multioutbuf_init::buf()->remove(str); - } + void remove(std::basic_ostream& str) { multioutbuf_init::buf()->remove(str); } static Output* getInstance(); ///< Return pointer to instance @@ -126,7 +136,7 @@ protected: private: std::ofstream file; ///< Log file stream - bool enabled; ///< Whether output to stdout is enabled + bool enabled{}; ///< Whether output to stdout is enabled }; /// Class which behaves like Output, but has no effect. @@ -136,14 +146,14 @@ private: class DummyOutput : public Output { public: template - void write([[maybe_unused]] const S& format, [[maybe_unused]] const Args&... args){}; + void write([[maybe_unused]] const S& format, [[maybe_unused]] Args&&... args) {}; template - void print([[maybe_unused]] const S& format, [[maybe_unused]] const Args&... args){}; - void write([[maybe_unused]] const std::string& message) override{}; - void print([[maybe_unused]] const std::string& message) override{}; - void enable() override{}; - void disable() override{}; - void enable([[maybe_unused]] bool enable){}; + void print([[maybe_unused]] const S& format, [[maybe_unused]] Args&&... args) {}; + void write([[maybe_unused]] const std::string& message) override {}; + void print([[maybe_unused]] const std::string& message) override {}; + void enable() override {}; + void disable() override {}; + void enable([[maybe_unused]] bool enable) {}; bool isEnabled() override { return false; } }; @@ -156,24 +166,25 @@ class ConditionalOutput : public Output { public: /// @param[in] base The Output object which will be written to if enabled /// @param[in] enabled Should this be enabled by default? - ConditionalOutput(Output* base, bool enabled = true) : base(base), enabled(enabled){}; + ConditionalOutput(Output* base, bool enabled = true) : base(base), enabled(enabled) {}; /// Constuctor taking ConditionalOutput. This allows several layers of conditions /// /// @param[in] base A ConditionalOutput which will be written to if enabled /// - ConditionalOutput(ConditionalOutput* base) : base(base), enabled(base->enabled){}; + ConditionalOutput(ConditionalOutput* base) : base(base), enabled(base->enabled) {}; /// If enabled, writes a string using fmt formatting /// by calling base->write /// This string is then sent to log file and stdout (on processor 0) void write(const std::string& message) override; - template - void write(const S& format, const Args&... args) { + template + // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) + void write(fmt::format_string format, Args&&... args) { if (enabled) { ASSERT1(base != nullptr); - base->write(fmt::format(format, args...)); + base->write(fmt::vformat(format, fmt::make_format_args(args...))); } } @@ -181,11 +192,12 @@ public: /// note: unlike write, this is not also sent to log files void print(const std::string& message) override; - template - void print(const S& format, const Args&... args) { + template + // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) + void print(fmt::format_string format, Args&&... args) { if (enabled) { ASSERT1(base != nullptr); - base->print(fmt::format(format, args...)); + base->print(fmt::vformat(format, fmt::make_format_args(args...))); } } @@ -216,8 +228,6 @@ private: /// The lower-level Output to send output to Output* base; -protected: - friend class WithQuietOutput; /// Does this instance output anything? bool enabled; }; @@ -227,7 +237,7 @@ protected: /// output_debug << "debug message"; /// compile but have no effect if BOUT_USE_OUTPUT_DEBUG is false template -DummyOutput& operator<<(DummyOutput& out, T const& UNUSED(t)) { +DummyOutput& operator<<(DummyOutput& out, const T& UNUSED(t)) { return out; } @@ -251,7 +261,7 @@ inline ConditionalOutput& operator<<(ConditionalOutput& out, stream_manipulator } template -ConditionalOutput& operator<<(ConditionalOutput& out, T const& t) { +ConditionalOutput& operator<<(ConditionalOutput& out, const T& t) { if (out.isEnabled()) { *out.getBase() << t; } @@ -276,18 +286,23 @@ ConditionalOutput& operator<<(ConditionalOutput& out, const T* t) { /// // output now enabled class WithQuietOutput { public: - explicit WithQuietOutput(ConditionalOutput& output_in) : output(output_in) { - state = output.enabled; - output.disable(); + WithQuietOutput(const WithQuietOutput&) = delete; + WithQuietOutput(WithQuietOutput&&) = delete; + WithQuietOutput& operator=(const WithQuietOutput&) = delete; + WithQuietOutput& operator=(WithQuietOutput&&) = delete; + explicit WithQuietOutput(ConditionalOutput& output_in) + : output(&output_in), state(output->isEnabled()) { + output->disable(); } - ~WithQuietOutput() { output.enable(state); } + ~WithQuietOutput() { output->enable(state); } private: - ConditionalOutput& output; + ConditionalOutput* output; bool state; }; +// NOLINTBEGIN(cppcoreguidelines-avoid-non-const-global-variables) /// To allow statements like "output.write(...)" or "output << ..." /// Output for debugging #ifdef BOUT_USE_OUTPUT_DEBUG @@ -303,5 +318,6 @@ extern ConditionalOutput output_verbose; ///< less interesting messages /// Generic output, given the same level as output_progress extern ConditionalOutput output; +// NOLINTEND(cppcoreguidelines-avoid-non-const-global-variables) #endif // BOUT_OUTPUT_H diff --git a/include/bout/output_bout_types.hxx b/include/bout/output_bout_types.hxx index b67762521b..77ea8ebf2d 100644 --- a/include/bout/output_bout_types.hxx +++ b/include/bout/output_bout_types.hxx @@ -5,10 +5,21 @@ #ifndef OUTPUT_BOUT_TYPES_H #define OUTPUT_BOUT_TYPES_H +#include "bout/bout_types.hxx" +#include "bout/mesh.hxx" +#include "bout/output.hxx" // IWYU pragma: keep +#include "bout/region.hxx" +#include "bout/traits.hxx" + +#include +#include #include -#include "bout/output.hxx" -#include "bout/region.hxx" +#include +#include +#include +#include +#include template struct fmt::formatter> { @@ -17,7 +28,8 @@ struct fmt::formatter> { // Parses format specifications of the form ['c' | 'i']. constexpr auto parse(format_parse_context& ctx) { - auto it = ctx.begin(), end = ctx.end(); + const auto* it = ctx.begin(); + const auto* end = ctx.end(); if (it != end && (*it == 'c' || *it == 'i')) { presentation = *it++; } @@ -50,4 +62,224 @@ struct fmt::formatter> { } }; +namespace bout { +namespace details { +template +/// Transpose a region so that it iterates in Z first, then Y, then X +/// +/// Caution: this is the most inefficient memory order! +auto region_transpose(const Region& region) -> Region; + +auto colour(BoutReal value, BoutReal min, BoutReal max) -> fmt::text_style; + +// Parses the range [begin, end) as an unsigned integer. This function assumes +// that the range is non-empty and the first character is a digit. +// +// Taken from fmt +// Copyright (c) 2012 - present, Victor Zverovich +// SPDX-License-Identifier: MIT +constexpr auto parse_nonnegative_int(const char*& begin, const char* end, + int error_value) noexcept -> int { + const auto* p = begin; + unsigned value = *p - '0'; + unsigned prev = 0; + ++p; + + while (p != end && '0' <= *p && *p <= '9') { + prev = value; + value = (value * 10) + unsigned(*p - '0'); + ++p; + } + auto num_digits = p - begin; + begin = p; + const auto digits10 = static_cast(sizeof(int) * CHAR_BIT * 3 / 10); + if (num_digits <= digits10) { + return static_cast(value); + } + // Check for overflow. + const unsigned max = INT_MAX; + return num_digits == digits10 + 1 and (prev * 10ULL) + unsigned(p[-1] - '0') <= max + ? static_cast(value) + : error_value; +} +} // namespace details +} // namespace bout + +/// Formatter for Fields +/// +/// Format specification: +/// +/// - ``n``: Don't show indices +/// - ``r''``: Use given region (default: ``RGN_ALL``) +/// - ``T``: Transpose field so X is first dimension +/// - ``#``: Plot slices as 2D heatmap +/// - ``e``: Number of elements at each edge to show +/// - ``f``: Show full field +template +struct fmt::formatter, char>> { +private: + fmt::formatter underlying; + + static constexpr auto default_region = "RGN_ALL"; + std::string_view region = default_region; + + bool show_indices = true; + bool transpose = false; + bool plot = false; + int edgeitems = 4; + bool show_full = false; + +public: + constexpr auto parse(format_parse_context& ctx) { + const auto* it = ctx.begin(); + const auto* end = ctx.end(); + + if (it == end) { + return underlying.parse(ctx); + } + + while (it != end and *it != ':' and *it != '}') { + // Other cases handled explicitly below + // NOLINTNEXTLINE(bugprone-switch-missing-default-case) + switch (*it) { + case 'e': + ++it; + edgeitems = bout::details::parse_nonnegative_int(it, end, -1); + if (edgeitems == -1) { + throw fmt::format_error("number is too big"); + } + break; + case 'f': + show_full = true; + ++it; + break; + case 'r': + ++it; + if (*it != '\'') { + throw fmt::format_error("invalid format for Field"); + } + { + const auto* rgn_start = ++it; + std::size_t size = 0; + while (*it != '\'') { + ++size; + ++it; + } + region = std::string_view(rgn_start, size); + } + ++it; + break; + case 'n': + show_indices = false; + ++it; + break; + case 'T': + transpose = true; + ++it; + break; + case '#': + plot = true; + show_indices = false; + ++it; + break; + } + } + + if (it != end and *it != '}') { + if (*it != ':') { + throw fmt::format_error("invalid format specifier"); + } + ++it; + } + + ctx.advance_to(it); + return underlying.parse(ctx); + } + + auto format(const T& f, format_context& ctx) const -> format_context::iterator { + using namespace bout::details; + + const auto* mesh = f.getMesh(); + + const auto rgn_str = std::string{region}; + const auto rgn_ = f.getRegion(rgn_str); + const auto rgn = transpose ? region_transpose(rgn_) : rgn_; + + const auto i = rgn.begin(); + int previous_x = i->x(); + int previous_y = i->y(); + int previous_z = i->z(); + + const auto last_i = rgn.getIndices().rbegin(); + const int last_x = last_i->x(); + const int last_y = last_i->y(); + const int last_z = last_i->z(); + + // Indices of edges + const int start_x = previous_x + edgeitems; + const int start_y = previous_y + edgeitems; + const int start_z = previous_z + edgeitems; + const int end_x = last_x - edgeitems; + const int end_y = last_y - edgeitems; + const int end_z = last_z - edgeitems; + + // Range of the data for plotting + BoutReal plot_min = 0.0; + BoutReal plot_max = 0.0; + if (plot) { + plot_min = min(f, false, rgn_str); + plot_max = max(f, false, rgn_str); + } + + // Separators + const auto block_sep = fmt::runtime("\n\n"); + const auto item_sep = fmt::runtime(plot ? "" : " "); + const auto x_sep = transpose ? item_sep : block_sep; + const auto z_sep = transpose ? block_sep : item_sep; + + // If we've shown the skip sep already this dim + bool shown_skip = false; + constexpr auto skip_sep = "..."; + + BOUT_FOR_SERIAL(i, rgn) { + const auto ix = i.x(); + const auto iy = i.y(); + const auto iz = i.z(); + + const bool should_show = + show_full + or ((ix < start_x or ix > end_x) and (iy < start_y or iy > end_y) + and (iz < start_z or iz > end_z)); + + if ((not shown_skip or should_show) and iz > previous_z) { + format_to(ctx.out(), z_sep); + } + if ((not shown_skip or should_show) and iy > previous_y) { + format_to(ctx.out(), "\n"); + } + if ((not shown_skip or should_show) and ix > previous_x) { + format_to(ctx.out(), x_sep); + } + + if (show_indices and should_show) { + format_to(ctx.out(), "{:c}: ", i); + } + if (plot) { + format_to(ctx.out(), "{}", styled("█", colour(f[i], plot_min, plot_max))); + } else if (should_show) { + underlying.format(f[i], ctx); + format_to(ctx.out(), ";"); + shown_skip = false; + } else if (not shown_skip) { + format_to(ctx.out(), skip_sep); + shown_skip = true; + } + previous_x = ix; + previous_y = iy; + previous_z = iz; + } + return format_to(ctx.out(), ""); + } +}; + #endif // OUTPUT_BOUT_TYPES_H diff --git a/include/bout/petsc_interface.hxx b/include/bout/petsc_interface.hxx index 3ae02f41a6..2ce71d0549 100644 --- a/include/bout/petsc_interface.hxx +++ b/include/bout/petsc_interface.hxx @@ -40,18 +40,27 @@ #include #include +#include +#include #include #include +#include #include #include +#include #include #include #include #include #include +#include +#include #include #include +#include + +#include /*! * A class which wraps PETSc vector objects, allowing them to be @@ -70,7 +79,7 @@ inline MPI_Comm getComm([[maybe_unused]] const T& field) { template <> inline MPI_Comm getComm([[maybe_unused]] const FieldPerp& field) { - return field.getMesh()->getXcomm(); + return field.getMesh()->getXZcomm(); } template @@ -284,7 +293,7 @@ public: PetscMatrix(IndexerPtr indConverter, bool preallocate = true) : matrix(new Mat()), indexConverter(indConverter), pt(&indConverter->getMesh()->getCoordinates()->getParallelTransform()) { - MPI_Comm comm = std::is_same_v ? indConverter->getMesh()->getXcomm() + MPI_Comm comm = std::is_same_v ? indConverter->getMesh()->getXZcomm() : BoutComm::get(); const int size = indexConverter->size(); @@ -363,7 +372,7 @@ public: } } Element& operator=(const Element& other) { - AUTO_TRACE(); + if (this == &other) { return *this; } @@ -372,14 +381,14 @@ public: return *this; } Element& operator=(BoutReal val) { - AUTO_TRACE(); + ASSERT3(finite(val)); value = val; setValues(val, INSERT_VALUES); return *this; } Element& operator+=(BoutReal val) { - AUTO_TRACE(); + ASSERT3(finite(val)); auto columnPosition = std::find(positions.begin(), positions.end(), petscCol); if (columnPosition != positions.end()) { @@ -394,7 +403,6 @@ public: private: void setValues(BoutReal val, InsertMode mode) { - TRACE("PetscMatrix setting values at ({}, {})", petscRow, petscCol); ASSERT3(positions.size() > 0); std::vector values; std::transform(weights.begin(), weights.end(), std::back_inserter(values), @@ -405,7 +413,8 @@ public: status = MatSetValues(*petscMatrix, 1, &petscRow, positions.size(), positions.data(), values.data(), mode); if (status != 0) { - throw BoutException("Error when setting elements of a PETSc matrix."); + throw BoutException("Error when setting elements of a PETSc matrix at ({}, {})", + petscRow, petscCol); } } Mat* petscMatrix; diff --git a/include/bout/physicsmodel.hxx b/include/bout/physicsmodel.hxx index 9fa25d8b0f..8e8e3814cd 100644 --- a/include/bout/physicsmodel.hxx +++ b/include/bout/physicsmodel.hxx @@ -1,20 +1,20 @@ /*!************************************************************************ * \file physicsmodel.hxx - * + * * @brief Base class for Physics Models - * - * + * + * * * Changelog: - * + * * 2013-08 Ben Dudson * * Initial version - * + * ************************************************************************** * Copyright 2013 B.D.Dudson * * Contact: Ben Dudson, bd512@york.ac.uk - * + * * This file is part of BOUT++. * * BOUT++ is free software: you can redistribute it and/or modify @@ -47,6 +47,9 @@ class PhysicsModel; #include "bout/unused.hxx" #include "bout/utils.hxx" +#include +#include +#include #include #include @@ -164,9 +167,9 @@ public: * * Output * ------ - * + * * The time derivatives will be put in the ddt() variables - * + * * Returns a flag: 0 indicates success, non-zero an error flag */ int runRHS(BoutReal time, bool linear = false); @@ -200,7 +203,7 @@ public: bool hasPrecon(); /*! - * Run the preconditioner. The system state should be in the + * Run the preconditioner. The system state should be in the * evolving variables, and the vector to be solved in the ddt() variables. * The result will be put in the ddt() variables. * @@ -216,7 +219,7 @@ public: /*! * Run the Jacobian-vector multiplication function - * + * * Note: this is usually only called by the Solver */ int runJacobian(BoutReal t); @@ -238,10 +241,10 @@ protected: // The init and rhs functions are implemented by user code to specify problem /*! * @brief This function is called once by the solver at the start of a simulation. - * + * * A valid PhysicsModel must implement this function - * - * Variables should be read from the inputs, and the variables to + * + * Variables should be read from the inputs, and the variables to * be evolved should be specified. */ virtual int init(bool restarting) = 0; @@ -255,7 +258,7 @@ protected: /*! * @brief This function is called by the time integration solver * at least once per time step - * + * * Variables being evolved will be set by the solver * before the call, and this function must calculate * and set the time-derivatives. @@ -275,10 +278,10 @@ protected: /// Add additional variables other than the evolving variables to the restart files virtual void restartVars(Options& options); - /* + /* If split operator is set to true, then convective() and diffusive() are called instead of rhs() - + For implicit-explicit schemes, convective() will typically be treated explicitly, whilst diffusive() will be treated implicitly. For unsplit methods, both convective and diffusive will be called @@ -331,7 +334,7 @@ protected: * * @param[in] var The variable to evolve * @param[in] name The name to use for variable initialisation and output - * + * * Note that the variable must not be destroyed (e.g. go out of scope) * after this call, since a pointer to \p var is stored in the solver. * @@ -355,11 +358,11 @@ protected: * Specify a constrained variable \p var, which will be * adjusted to make \p F_var equal to zero. * If the solver does not support constraints then this will throw an exception - * + * * @param[in] var The variable the solver should modify * @param[in] F_var The control variable, which the user will set * @param[in] name The name to use for initialisation and output - * + * */ bool bout_constrain(Field3D& var, Field3D& F_var, const char* name); @@ -376,6 +379,9 @@ protected: PhysicsModel* model; }; + /// Set timestep counter for flushing file + void setFlushCounter(std::size_t iteration) { flush_counter = iteration; } + private: /// State for outputs Options output_options; @@ -399,6 +405,10 @@ private: bool initialised{false}; /// write restarts and pass outputMonitor method inside a Monitor subclass PhysicsModelMonitor modelMonitor{this}; + /// How often to flush to disk + std::size_t flush_frequency{1}; + /// Current timestep counter + std::size_t flush_counter{0}; }; /*! @@ -418,14 +428,14 @@ private: */ #define BOUTMAIN(ModelClass) \ int main(int argc, char** argv) { \ - int init_err = BoutInitialise(argc, argv); \ - if (init_err < 0) { \ - return 0; \ - } \ - if (init_err > 0) { \ - return init_err; \ - } \ try { \ + int init_err = BoutInitialise(argc, argv); \ + if (init_err < 0) { \ + return 0; \ + } \ + if (init_err > 0) { \ + return init_err; \ + } \ auto model = bout::utils::make_unique(); \ auto solver = Solver::create(); \ solver->setModel(model.get()); \ @@ -433,8 +443,8 @@ private: solver->addMonitor(bout_monitor.get(), Solver::BACK); \ solver->solve(); \ } catch (const BoutException& e) { \ - output << "Error encountered: " << e.what(); \ - output << e.getBacktrace() << endl; \ + output.write("Error encountered: {}\n", e.what()); \ + std::this_thread::sleep_for(std::chrono::milliseconds(100)); \ MPI_Abort(BoutComm::get(), 1); \ } \ BoutFinalise(); \ @@ -481,8 +491,7 @@ private: /// Add fields to the solver. /// This should accept up to ten arguments -#define SOLVE_FOR(...) \ - { MACRO_FOR_EACH(SOLVE_FOR1, __VA_ARGS__) } +#define SOLVE_FOR(...) {MACRO_FOR_EACH(SOLVE_FOR1, __VA_ARGS__)} /// Write this variable once to the grid file #define SAVE_ONCE1(var) dump.addOnce(var, #var); @@ -522,8 +531,7 @@ private: dump.addOnce(var6, #var6); \ } -#define SAVE_ONCE(...) \ - { MACRO_FOR_EACH(SAVE_ONCE1, __VA_ARGS__) } +#define SAVE_ONCE(...) {MACRO_FOR_EACH(SAVE_ONCE1, __VA_ARGS__)} /// Write this variable every timestep #define SAVE_REPEAT1(var) dump.addRepeat(var, #var); @@ -563,7 +571,6 @@ private: dump.addRepeat(var6, #var6); \ } -#define SAVE_REPEAT(...) \ - { MACRO_FOR_EACH(SAVE_REPEAT1, __VA_ARGS__) } +#define SAVE_REPEAT(...) {MACRO_FOR_EACH(SAVE_REPEAT1, __VA_ARGS__)} #endif // BOUT_PHYSICS_MODEL_H diff --git a/include/bout/region.hxx b/include/bout/region.hxx index bb1cf82bf1..e00ad6d41d 100644 --- a/include/bout/region.hxx +++ b/include/bout/region.hxx @@ -139,7 +139,7 @@ class BoutMask; BOUT_FOR_OMP(index, (region), for schedule(BOUT_OPENMP_SCHEDULE) nowait) // NOLINTEND(cppcoreguidelines-macro-usage,bugprone-macro-parentheses) -enum class IND_TYPE { IND_3D = 0, IND_2D = 1, IND_PERP = 2 }; +enum class IND_TYPE { IND_3D = 0, IND_2D = 1, IND_PERP = 2, IND_GLOBAL_3D = 3 }; /// Indices base class for Fields -- Regions are dereferenced into these /// @@ -386,6 +386,7 @@ inline SpecificInd operator-(SpecificInd lhs, const SpecificInd& rhs) { using Ind3D = SpecificInd; using Ind2D = SpecificInd; using IndPerp = SpecificInd; +using IndG3D = SpecificInd; /// Get string representation of Ind3D inline std::string toString(const Ind3D& i) { @@ -972,6 +973,12 @@ Region operator+(const Region& lhs, const Region& rhs) { return Region(indices); } +template +Region operator+(Region&& lhs, const Region& rhs) { + lhs += rhs; + return std::move(lhs); +} + /// Returns a new region based on input but with indices offset by /// a constant template diff --git a/include/bout/scorepwrapper.hxx b/include/bout/scorepwrapper.hxx index 81761530f0..8b37b2ff61 100644 --- a/include/bout/scorepwrapper.hxx +++ b/include/bout/scorepwrapper.hxx @@ -3,7 +3,6 @@ #include "bout/build_defines.hxx" -#include "bout/msg_stack.hxx" #include #if BOUT_HAS_SCOREP @@ -14,13 +13,25 @@ #define SCOREPLVL 0 #endif +/// The __PRETTY_FUNCTION__ variable is defined by GCC (and some other families) but is +/// not a part of the standard. The __func__ variable *is* a part of the c++11 standard so +/// we'd like to fall back to this if possible. However as these are variables/constants +/// and not macros we can't just check if __PRETTY_FUNCITON__ is defined or not. Instead +/// we need to say if we support this or not by defining BOUT_HAS_PRETTY_FUNCTION (to be +/// implemented in configure) +#if BOUT_HAS_PRETTY_FUNCTION +#define _thefunc_ __PRETTY_FUNCTION__ +#else +#define _thefunc_ __func__ +#endif + /// Instrument a function with scorep /// /// The scorep call is identical for all levels, so just define it here. /// If we don't have scorep support then just define a null function #if BOUT_HAS_SCOREP #define SCOREP_BASE_CALL(...) \ - SCOREP_USER_REGION(__thefunc__, SCOREP_USER_REGION_TYPE_FUNCTION) + SCOREP_USER_REGION(_thefunc_, SCOREP_USER_REGION_TYPE_FUNCTION) #else #define SCOREP_BASE_CALL(...) #endif diff --git a/include/bout/sys/expressionparser.hxx b/include/bout/sys/expressionparser.hxx index b312ce2fb1..8b2442e843 100644 --- a/include/bout/sys/expressionparser.hxx +++ b/include/bout/sys/expressionparser.hxx @@ -29,7 +29,8 @@ #include "bout/unused.hxx" -#include "fmt/core.h" +#include +#include #include #include @@ -239,11 +240,16 @@ private: class ParseException : public std::exception { public: + ParseException(const ParseException&) = default; + ParseException(ParseException&&) = delete; + ParseException& operator=(const ParseException&) = default; + ParseException& operator=(ParseException&&) = delete; ParseException(const std::string& message_) : message(message_) {} - template - ParseException(const S& format, const Args&... args) - : message(fmt::format(format, args...)) {} + template + // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) + ParseException(fmt::format_string format, Args&&... args) + : message(fmt::vformat(format, fmt::make_format_args(args...))) {} ~ParseException() override = default; diff --git a/include/bout/sys/gettext.hxx b/include/bout/sys/gettext.hxx index d67becd54d..d62cd66882 100644 --- a/include/bout/sys/gettext.hxx +++ b/include/bout/sys/gettext.hxx @@ -7,15 +7,47 @@ #if BOUT_HAS_GETTEXT -#include +#include // IWYU pragma: keep + #include #define GETTEXT_PACKAGE "libbout" -#define _(string) dgettext(GETTEXT_PACKAGE, string) +// If we have C++23, we can get fmt to do compile-time checks of our format +// strings, _and_ have gettext do runtime replacement +#if __cpp_if_consteval >= 202106L +constexpr const char* dgettext_wrap(const char* __domainname, const char* __msgid) __THROW + __attribute_format_arg__(2); + +constexpr const char* dgettext_wrap(const char* __domainname, const char* __msgid) { + if consteval { + return __msgid; + } + return dgettext(__domainname, __msgid); +} + +/// Gettext i18n macro for text containing fmt format specifiers +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define _f(string) dgettext_wrap(GETTEXT_PACKAGE, string) #else +// We're pre-C++23, so all our i18n text must be fmt runtime formats +#include "fmt/base.h" // IWYU pragma: keep + +/// Gettext i18n macro for text containing fmt format specifiers +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define _f(string) fmt::runtime(dgettext(GETTEXT_PACKAGE, string)) +#endif + +/// Gettext i18n macro for plain text that doesn't need formatting +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define _(string) dgettext(GETTEXT_PACKAGE, string) + +#else // BOUT_HAS_GETTEXT +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define _f(string) string +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) #define _(string) string #endif // BOUT_HAS_GETTEXT diff --git a/include/bout/sys/timer.hxx b/include/bout/sys/timer.hxx index f3beba27b1..c6a0c9020a 100644 --- a/include/bout/sys/timer.hxx +++ b/include/bout/sys/timer.hxx @@ -7,7 +7,6 @@ #include #include "bout/msg_stack.hxx" -#include "bout/output.hxx" /*! * Timing class for performance benchmarking and diagnosis @@ -133,5 +132,4 @@ public: static void printTimeReport(); }; -#define AUTO_TIME() Timer CONCATENATE(time_, __LINE__)(__thefunc__) #endif // BOUT_TIMER_H diff --git a/include/bout/utils.hxx b/include/bout/utils.hxx index f8d6f08856..eae3b27730 100644 --- a/include/bout/utils.hxx +++ b/include/bout/utils.hxx @@ -5,10 +5,10 @@ * simple but common calculations * ************************************************************************** - * Copyright 2010 - 2025 BOUT++ contributors + * Copyright 2010 - 2026 BOUT++ contributors * * Contact: Ben Dudson, dudson2@llnl.gov - * + * * This file is part of BOUT++. * * BOUT++ is free software: you can redistribute it and/or modify @@ -35,7 +35,6 @@ #include "bout/assert.hxx" #include "bout/bout_types.hxx" #include "bout/boutexception.hxx" -#include "bout/msg_stack.hxx" #include "bout/region.hxx" #include "bout/unused.hxx" @@ -187,6 +186,7 @@ typename std::vector::size_type erase_if(std::vector& c, Pre return r; } #else +using std::erase; using std::erase_if; #endif } // namespace utils @@ -231,6 +231,14 @@ public: data.reallocate(new_size_1 * new_size_2); } + /*! + * Change shape of the container. + * Invalidates contents. + */ + void reshape(std::tuple new_shape) { + reallocate(std::get<0>(new_shape), std::get<1>(new_shape)); + } + Matrix& operator=(const Matrix& other) { n1 = other.n1; n2 = other.n2; @@ -331,6 +339,14 @@ public: data.reallocate(new_size_1 * new_size_2 * new_size_3); } + /*! + * Change shape of the container. + * Invalidates contents. + */ + void reshape(std::tuple new_shape) { + reallocate(std::get<0>(new_shape), std::get<1>(new_shape), std::get<2>(new_shape)); + } + Tensor& operator=(const Tensor& other) { n1 = other.n1; n2 = other.n2; @@ -468,7 +484,7 @@ inline bool is_pow2(int x) { return x && !((x - 1) & x); } /*! * Return the sign of a number \p a - * by testing if a > 0 + * by testing if a > 0 */ template T SIGN(T a) { // Return +1 or -1 (0 -> +1) @@ -495,7 +511,7 @@ inline void checkData(BoutReal f) { } #else /// Ignored with disabled CHECK; Throw an exception if \p f is not finite -inline void checkData(BoutReal UNUSED(f)){}; +inline void checkData(BoutReal UNUSED(f)) {}; #endif /*! @@ -571,7 +587,7 @@ BoutReal stringToReal(const std::string& s); /*! * Convert a string to an int - * + * * Throws BoutException if can't be done */ int stringToInt(const std::string& s); @@ -588,7 +604,7 @@ std::list& strsplit(const std::string& s, char delim, /*! * Split a string on a given delimiter - * + * * @param[in] s The string to split (not modified by call) * @param[in] delim The delimiter to split on (single char) */ @@ -596,7 +612,7 @@ std::list strsplit(const std::string& s, char delim); /*! * Strips leading and trailing spaces from a string - * + * * @param[in] s The string to trim (not modified) * @param[in] c Collection of characters to remove */ @@ -604,7 +620,7 @@ std::string trim(const std::string& s, const std::string& c = " \t\r"); /*! * Strips leading spaces from a string - * + * * @param[in] s The string to trim (not modified) * @param[in] c Collection of characters to remove */ @@ -612,7 +628,7 @@ std::string trimLeft(const std::string& s, const std::string& c = " \t"); /*! * Strips leading spaces from a string - * + * * @param[in] s The string to trim (not modified) * @param[in] c Collection of characters to remove */ @@ -620,7 +636,7 @@ std::string trimRight(const std::string& s, const std::string& c = " \t\r"); /*! * Strips the comments from a string - * + * * @param[in] s The string to trim (not modified) * @param[in] c Collection of characters to remove */ diff --git a/include/bout/vector2d.hxx b/include/bout/vector2d.hxx index bdc375e698..1e19241a8e 100644 --- a/include/bout/vector2d.hxx +++ b/include/bout/vector2d.hxx @@ -13,7 +13,7 @@ * Copyright 2010 B.D.Dudson, S.Farley, M.V.Umansky, X.Q.Xu * * Contact: Ben Dudson, bd512@york.ac.uk - * + * * This file is part of BOUT++. * * BOUT++ is free software: you can redistribute it and/or modify @@ -142,7 +142,7 @@ public: CELL_LOC getLocation() const override; // FieldData virtual functions - bool is3D() const override { return false; } + FieldType field_type() const override { return FieldType::field2d; } int elementSize() const override { return 3; } /// Apply boundary condition to all fields diff --git a/include/bout/vector3d.hxx b/include/bout/vector3d.hxx index 0c71dcffa5..655a85ca73 100644 --- a/include/bout/vector3d.hxx +++ b/include/bout/vector3d.hxx @@ -9,7 +9,7 @@ * Copyright 2010 B.D.Dudson, S.Farley, M.V.Umansky, X.Q.Xu * * Contact: Ben Dudson, bd512@york.ac.uk - * + * * This file is part of BOUT++. * * BOUT++ is free software: you can redistribute it and/or modify @@ -45,10 +45,10 @@ class Vector2D; * ------- * * Vector3D f; - * + * * a.x; // Error! a.x not allocated * - * + * */ class Vector3D : public FieldData { public: @@ -70,7 +70,7 @@ public: ~Vector3D() override; /*! - * The components of the vector. These can be + * The components of the vector. These can be * either co- or contra-variant, depending on * the boolean flag "covariant" */ @@ -92,8 +92,8 @@ public: bool covariant{true}; /*! - * In-place conversion to covariant form. - * + * In-place conversion to covariant form. + * * If already covariant (covariant = true) then does nothing * If contravariant, multiplies by metric tensor g_{ij} */ @@ -104,7 +104,7 @@ public: * * If already contravariant (covariant = false) then does nothing * If covariant, multiplies by metric tensor g^{ij} - * + * */ void toContravariant(); @@ -114,14 +114,14 @@ public: * The first time this is called, a new Vector3D object is created. * Subsequent calls return a pointer to this same object * - * For convenience, a standalone function "ddt" exists, so that + * For convenience, a standalone function "ddt" exists, so that * * ddt(v) is equivalent to *(v.timeDeriv()) - * + * * This does some book-keeping to ensure that the time derivative * of the components is the same as the components of the time derivative * - * ddt(v).x == ddt(v.x) + * ddt(v).x == ddt(v.x) */ Vector3D* timeDeriv(); @@ -172,7 +172,7 @@ public: CELL_LOC getLocation() const override; // FieldData virtual functions - bool is3D() const override { return true; } + FieldType field_type() const override { return FieldType::field3d; } int elementSize() const override { return 3; } void applyBoundary(bool init = false) override; @@ -203,7 +203,7 @@ const Vector3D cross(const Vector3D& lhs, const Vector2D& rhs); /*! * Absolute magnitude (modulus) of a vector |v| - * + * * sqrt( v.x^2 + v.y^2 + v.z^2 ) */ const Field3D abs(const Vector3D& v, const std::string& region = "RGN_ALL"); diff --git a/manual/CMakeLists.txt b/manual/CMakeLists.txt index c8e22b1dcf..87f5101e58 100644 --- a/manual/CMakeLists.txt +++ b/manual/CMakeLists.txt @@ -6,28 +6,37 @@ find_package(Sphinx REQUIRED) set(BOUT_SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/sphinx) set(BOUT_SPHINX_BUILD ${CMAKE_CURRENT_BINARY_DIR}/docs) -add_custom_target(sphinx-html - COMMAND ${SPHINX_EXECUTABLE} -b html ${BOUT_SPHINX_SOURCE} ${BOUT_SPHINX_BUILD} - COMMAND ${CMAKE_COMMAND} -E echo "Generated HTML docs in file://${BOUT_SPHINX_BUILD}/index.html" +set(env_command ${CMAKE_COMMAND} -E env + PYTHONPATH=${BOUT_PYTHONPATH}:$ENV{PYTHONPATH} +) + +add_custom_target( + sphinx-html + COMMAND ${env_command} ${SPHINX_EXECUTABLE} -b html ${BOUT_SPHINX_SOURCE} + ${BOUT_SPHINX_BUILD} + COMMAND ${CMAKE_COMMAND} -E echo + "Generated HTML docs in file://${BOUT_SPHINX_BUILD}/index.html" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generating HTML documentation with Sphinx in ${BOUT_SPHINX_BUILD}" ) -add_custom_target(sphinx-pdf - COMMAND ${SPHINX_EXECUTABLE} -M latexpdf ${BOUT_SPHINX_SOURCE} ${BOUT_SPHINX_BUILD} - COMMAND ${CMAKE_COMMAND} -E echo "Generated PDF docs in file://${BOUT_SPHINX_BUILD}" +add_custom_target( + sphinx-pdf + COMMAND ${env_command} ${SPHINX_EXECUTABLE} -M latexpdf ${BOUT_SPHINX_SOURCE} + ${BOUT_SPHINX_BUILD} + COMMAND ${CMAKE_COMMAND} -E echo + "Generated PDF docs in file://${BOUT_SPHINX_BUILD}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generating PDF documentation with Sphinx in ${BOUT_SPHINX_BUILD}" ) -set_target_properties(sphinx-html sphinx-pdf PROPERTIES - ENVIRONMENT PYTHONPATH=${BOUT_PYTHONPATH}:$ENV{PYTHONPATH} -) - -add_custom_target(docs ALL) +add_custom_target(docs) add_dependencies(docs sphinx-html) -install(DIRECTORY ${BOUT_SPHINX_BUILD}/ +install( + DIRECTORY ${BOUT_SPHINX_BUILD}/ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/doc/bout++/ + EXCLUDE_FROM_ALL + COMPONENT docs PATTERN .* EXCLUDE ) diff --git a/manual/doxygen/Doxyfile b/manual/doxygen/Doxyfile index e7998854b0..973c5fce58 100644 --- a/manual/doxygen/Doxyfile +++ b/manual/doxygen/Doxyfile @@ -840,7 +840,8 @@ EXCLUDE = ../../examples \ ../../manual/ \ ../../bin/ \ ../../externalpackages/googletest/ \ - ../../tests/ + ../../tests/ \ + ../../build/_deps/ # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded @@ -2059,13 +2060,11 @@ PREDEFINED = BACKTRACE \ BOUT_HAS_LAPACK \ BOUT_HAS_NETCDF \ BOUT_HAS_PETSC \ - BOUT_HAS_PRETTY_FUNCTION \ BOUT_HAS_PVODE \ BOUT_HAS_SCOREP \ BOUT_HAS_SLEPC \ BOUT_HAS_SUNDIALS \ BOUT_HAS_UUID_SYSTEM_GENERATOR \ - BOUT_USE_BACKTRACE \ BOUT_USE_COLOR \ BOUT_USE_OPENMP \ BOUT_USE_OUTPUT_DEBUG \ diff --git a/manual/doxygen/Doxyfile_readthedocs b/manual/doxygen/Doxyfile_readthedocs index 73060bb6b4..08de92fc79 100644 --- a/manual/doxygen/Doxyfile_readthedocs +++ b/manual/doxygen/Doxyfile_readthedocs @@ -790,7 +790,8 @@ EXCLUDE = ../../examples \ ../../manual/ \ ../../bin/ \ ../../externalpackages/googletest/ \ - ../../tests/ + ../../tests/ \ + ../sphinx/bout_build/_deps/ # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded @@ -1019,13 +1020,11 @@ PREDEFINED = BACKTRACE \ BOUT_HAS_LAPACK \ BOUT_HAS_NETCDF \ BOUT_HAS_PETSC \ - BOUT_HAS_PRETTY_FUNCTION \ BOUT_HAS_PVODE \ BOUT_HAS_SCOREP \ BOUT_HAS_SLEPC \ BOUT_HAS_SUNDIALS \ BOUT_HAS_UUID_SYSTEM_GENERATOR \ - BOUT_USE_BACKTRACE \ BOUT_USE_COLOR \ BOUT_USE_OPENMP \ BOUT_USE_OUTPUT_DEBUG \ diff --git a/manual/sphinx/conf.py b/manual/sphinx/conf.py index 3aa6e94dc3..69ed52aeed 100755 --- a/manual/sphinx/conf.py +++ b/manual/sphinx/conf.py @@ -72,35 +72,29 @@ def __getattr__(cls, name): sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) print(os.environ) print(sys.argv) - python = sys.argv[0] - pydir = "/".join(python.split("/")[:-2]) os.system("which clang-format") os.system("which clang-format-6.0") - os.system( - "git submodule update --init --recursive ../../externalpackages/mpark.variant" - ) - pwd = "/".join(os.getcwd().split("/")[:-2]) - os.system("git submodule update --init --recursive ../../externalpackages/fmt") + subprocess.run("git submodule update --init --recursive", shell=True) cmake = ( - "cmake . -DBOUT_USE_FFTW=ON" - + " -DBOUT_USE_LAPACK=OFF" - + " -DBOUT_ENABLE_PYTHON=ON" - + " -DBOUT_UPDATE_GIT_SUBMODULE=OFF" - + " -DBOUT_TESTS=OFF" - + " -DBOUT_ALLOW_INSOURCE_BUILD=ON" - + f" -DPython3_ROOT_DIR={pydir}" - + f" -Dmpark_variant_DIR={pwd}/externalpackages/mpark.variant/" - + f" -Dfmt_DIR={pwd}/externalpackages/fmt/" + "cmake -S ../.. " + " -B bout_build" + " -DBOUT_USE_FFTW=ON" + " -DBOUT_USE_LAPACK=OFF" + " -DBOUT_ENABLE_PYTHON=ON" + " -DBOUT_UPDATE_GIT_SUBMODULE=OFF" + " -DBOUT_TESTS=OFF" + " -DBOUT_ALLOW_INSOURCE_BUILD=ON" + f" -DPython3_ROOT_DIR={sys.exec_prefix}" ) - # os.system("mkdir ../../build") - os.system("echo " + cmake) - x = os.system("cd ../.. ;" + cmake) - assert x == 0 - x = os.system("cd ../.. ; make -j 2 -f Makefile") - assert x == 0 + subprocess.run(f"echo {cmake}", shell=True) + subprocess.run(f"{cmake}", shell=True, check=True) + subprocess.run("cmake --build bout_build", shell=True, check=True) + + # Add the build directory to sys.path so that sphinx picks up the built + # Python modules + sys.path.append("bout_build/tools/pylib") -# readthedocs currently runs out of memory if we actually dare to try to do this if has_breathe: # Run doxygen to generate the XML sources if on_readthedocs: @@ -198,7 +192,7 @@ def __getattr__(cls, name): # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ".venv", "venv"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" diff --git a/manual/sphinx/developer_docs/contributing.rst b/manual/sphinx/developer_docs/contributing.rst index 4453f518b6..062415af04 100644 --- a/manual/sphinx/developer_docs/contributing.rst +++ b/manual/sphinx/developer_docs/contributing.rst @@ -173,6 +173,86 @@ give you comments to improve the code. You can make additional changes and push them to the same feature branch and they will be automatically added to the pull request. +Running Tests +~~~~~~~~~~~~~ + +We run many tests and checks automatically on GitHub (collectively called "CI") +for a variety of build configurations. See :ref:`sec-runtestsuite` for how to +run them locally. Running the full test suite can take some time, but it's a +good idea to run at least the unit tests locally when you're making changes: + +.. code-block:: console + + cmake --build build --target check-unit-tests + +Along with the automated tests, we also run things like formatters and static +analysis in CI, which may provide further feedback (see :ref:`sec-coding-style` +for more details). If your PR fails the format check on CI, please install the +formatters and run them according to the next section. Usually this is just: + +.. code-block:: console + + git clang-format next + +which will format just the parts of C++ files that have changed since ``next``. + +Formatting and Linters +~~~~~~~~~~~~~~~~~~~~~~ + +For frequent developers, we strongly recommend installing the formatting and +linting tools locally and building them into your regular workflow. You can +install the majority of our developer tools at once using `uv +`_ from the top of your BOUT++ directory: + +.. code-block:: console + + uv sync --only-dev --inexact + +This will install all the developer tools into a virtual environment (creating +one if necessary, typically under ``.venv``). You can then activate the virtual +environment and use the tools manually, or via integrations in your editor. + +We *strongly* recommend using ``uv`` to install the developer tools, as this +will ensure that you get the *exact* versions used in CI and by other +developers, but you could also use ``pip install --group dev`` if you wish. + +The quickest way to run all the formatters on a PR at once is typically: + +.. code-block:: console + + uv tool run prek run --from-ref next + +This will run all our formatters on files that have changed since ``next``. + +The tools can be updated with: + +.. code-block:: console + + uv lock -U + uv tool run sync-with-uv + + +Install Pre-Commit Hooks (Optional but recommended) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We also have a `prek `_ config that can run some of +these automatically using pre-commit hooks -- that is, when you run ``git +commit``, these tools will run and abort the commit if they change any +files. You then just have to stage the new changes and try again. + +Install our developer tools with ``uv`` as above, and then install the +pre-commit hook: + +.. code-block:: console + + prek install + +That's it! ``prek`` will install the tools into their own separate virtual +environment so they won't interfere with any you may have, and ``git`` will take +care of running them automatically. + +.. _sec-coding-style: + Coding Style ------------ @@ -189,6 +269,25 @@ These guidelines are intended to make code easier to read and therefore easier to understand. Being consistent in coding style also helps comprehension by reducing cognitive load. +We use various tools to enforce code style and quality: + +- `clang-format `_ for formatting + of C++ code +- `clang-tidy `_ for linting + and static analysis of C++ code +- `ruff `_ for formatting and linting of Python code +- `cmake-format `_ for formatting + CMake files +- `sphinx-lint `_ for linting + documentation pages + +You can install all of these tools using: + +.. code-block:: console + + pip install -r requirements_dev.txt + + Comments ~~~~~~~~ diff --git a/manual/sphinx/developer_docs/data_types.rst b/manual/sphinx/developer_docs/data_types.rst index fa8e9e6ea6..f9411dcb39 100644 --- a/manual/sphinx/developer_docs/data_types.rst +++ b/manual/sphinx/developer_docs/data_types.rst @@ -255,9 +255,9 @@ parallelise and vectorise. Some tuning of this is possible, see below for details. It replaces the C-style triple-nested loop:: Field3D f(0.0); - for (int i = mesh->xstart; i < mesh->xend; ++i) { - for (int j = mesh->ystart; j < mesh->yend; ++j) { - for (int k = 0; k < mesh->LocalNz; ++k) { + for (int i = mesh->xstart; i <= mesh->xend; ++i) { + for (int j = mesh->ystart; j <= mesh->yend; ++j) { + for (int k = mesh->zstart; k <= mesh->zend; ++k) { f(i,j,k) = a(i,j,k) + b(i,j,k) } } diff --git a/manual/sphinx/developer_docs/debugging.rst b/manual/sphinx/developer_docs/debugging.rst index 805cb5409a..4f47cd63e3 100644 --- a/manual/sphinx/developer_docs/debugging.rst +++ b/manual/sphinx/developer_docs/debugging.rst @@ -34,14 +34,77 @@ To enable the ``output_debug`` messages, configure BOUT++ with a configure BOUT++ with ``-DENABLE_OUTPUT_DEBUG``. When running BOUT++ add a ``-v -v`` flag to see ``output_debug`` messages. +Backtrace +========= + +BOUT++ can also automatically print a backtrace in the event of a crash. This is +very useful to include if you ever need to report a bug to the developers! The +output looks something like this: + +.. code:: text + + ... + Error encountered: Stack trace (most recent call first): + #0 (filtered) + #1 (filtered) + #2 (filtered) + #3 in BoutMesh::createCommunicators() + at BOUT-dev/src/mesh/impls/bout/boutmesh.cxx:709:64 + 707: } + 708: // Unconditional exception for demo purposes + > 709: throw BoutException("Single null outer SOL not correct\n"); + ^ + 710: MPI_Group_free(&group); + 711: } + #4 in BoutMesh::load() + at BOUT-dev/src/mesh/impls/bout/boutmesh.cxx:575:22 + 573: /// Communicator + 574: + > 575: createCommunicators(); + ^ + 576: output_debug << "Got communicators" << endl; + #5 in BoutInitialise(int&, char**&) + at BOUT-dev/src/bout++.cxx:201:28 + 199: bout::globals::mesh = Mesh::create(); + 200: // Load from sources. Required for Field initialisation + > 201: bout::globals::mesh->load(); + ^ + 202: + 203: // time_report options are used in BoutFinalise, i.e. after we + #6 in main + at BOUT-dev/examples/elm-pb/elm_pb.cxx:2161:1 + 2159: }; + 2160: + > 2161: BOUTMAIN(ELMpb); + ^ + #7 (filtered) + #8 (filtered) + #9 (filtered) + + ====== Exception thrown ====== + Single null outer SOL not correct + + +Debug symbols are required to get the filename/line number and code snippets. If +they are missing in either BOUT++ or the physics model, only the function name +and signature will be included for that part. + +Including debug symbols is a configure time ``CMake`` option, set either: +``-DCMAKE_BUILD_TYPE=Debug`` or ``-DCMAKE_BUILD_TYPE=RelWithDebInfo`` (the +default). + +The formatting of this backtrace is controlled in +`BoutException::getBacktrace()` using the `cpptrace +`_ library. + + Message Stack ============= -The second utility BOUT++ has to help debugging is the message stack -using the `TRACE` (and related `AUTO_TRACE`) macro. These are very -useful for when a bug only occurs after a long time of running, and/or -only occasionally. The ``TRACE`` macro can simply be dropped in -anywhere in the code:: +Another utility BOUT++ has to help debugging is the message stack using the +`TRACE` macro. This is very useful for when a bug only occurs after a long time +of running, and/or only occasionally. The ``TRACE`` macro can simply be dropped +in anywhere in the code:: { TRACE("Some message here"); // message pushed @@ -70,14 +133,15 @@ help find where an error occurred. For example, given this snippet:: we would see something like the following output: .. code:: text - - ====== Back trace ====== + + ====== Exception thrown ====== + Something went wrong + + === Additional information === -> 4. Inner-most scope on line 58 of '/path/to/model.cxx' -> 2. Middle scope on line 53 of '/path/to/model.cxx' -> 1. Outer-most scope on line 51 of '/path/to/model.cxx' - ====== Exception thrown ====== - Something went wrong The third ``TRACE`` message doesn't appear in the output because we've left its scope and it's no longer relevant. @@ -95,52 +159,29 @@ the ``fmt`` syntax also used by the loggers:: TRACE("Value of i={}, some arbitrary {}", i, "string"); -There is also an ``AUTO_TRACE`` macro that automatically captures the -name of the function it's used in. This is used throughout the main -library, especially in functions where numerical issues are likely to -arise. -Backtrace -========= +Time evolution +============== -Lastly, BOUT++ can also automatically print a backtrace in the event -of a crash. This is a compile-time option in the BOUT++ library -(``-DBOUT_ENABLE_BACKTRACE=ON``, the default, requires the -``addr2line`` program to be installed), and debug symbols to be turned -on (``-DCMAKE_BUILD_TYPE=Debug`` or ``=RelWithDebInfo``) in BOUT++ -_and_ the physics model. If debug symbols are only present in part, the -backtrace will be missing names for the other part. +It can be useful to know what happened when the simulation failed. The pvode +solver can dump the state of the simulation, at the time the solver +failed. This information includes the individual terms in the derivative. This +allows to identify which term is causing the issue. Additionally, the +residuum is dumped. This identifies not only which term is causing the issue, +but also where in the domain the solver is struggling. This can be enabled +with:: -The output looks something like this: + solver:type=pvode solver:debug_on_failure=true -.. code:: text +It can be also useful for understanding why the solver is slow. Forcing a +higher min_timestep, the solver will fail to evolve the system as it +encounters the situation, and provides information where it is happening. +This can be done with:: - ... - Error encountered - ====== Exception path ====== - [bt] #10 ./backtrace() [0x40a27e] - _start at /home/abuild/rpmbuild/BUILD/glibc-2.33/csu/../sysdeps/x86_64/start.S:122 - [bt] #9 /lib64/libc.so.6(__libc_start_main+0xd5) [0x7fecbfa28b25] - __libc_start_main at /usr/src/debug/glibc-2.33-4.1.x86_64/csu/../csu/libc-start.c:332 - [bt] #8 ./backtrace() [0x40a467] - main at /path/to/BOUT-dev/build/../examples/backtrace/backtrace.cxx:32 (discriminator 9) - [bt] #7 /path/to/BOUT-dev/build/libbout++.so(_ZN6Solver8setModelEP12PhysicsModel+0xb5) [0x7fecc0ca2e93] - Solver::setModel(PhysicsModel*) at /path/to/BOUT-dev/build/../src/solver/solver.cxx:94 - [bt] #6 /path/to/BOUT-dev/build/libbout++.so(_ZN12PhysicsModel10initialiseEP6Solver+0xc0) [0x7fecc0cad594] - PhysicsModel::initialise(Solver*) at /path/to/BOUT-dev/build/../include/bout/physicsmodel.hxx:93 (discriminator 5) - [bt] #5 ./backtrace() [0x40a986] - Backtrace::init(bool) at /path/to/BOUT-dev/build/../examples/backtrace/backtrace.cxx:27 - [bt] #4 ./backtrace() [0x40a3cf] - f3() at /path/to/BOUT-dev/build/../examples/backtrace/backtrace.cxx:19 - [bt] #3 ./backtrace() [0x40a3be] - f2(int) at /path/to/BOUT-dev/build/../examples/backtrace/backtrace.cxx:15 - [bt] #2 ./backtrace() [0x40a386] - f1() at /path/to/BOUT-dev/build/../examples/backtrace/backtrace.cxx:13 (discriminator 2) - [bt] #1 ./backtrace(_ZN13BoutExceptionC1IA19_cJEEERKT_DpRKT0_+0xba) [0x40ae16] - BoutException::BoutException(char const (&) [19]) at /path/to/BOUT-dev/build/../include/bout/../boutexception.hxx:28 (discriminator 2) - - -This output tends to be much harder to read than the message stack -from ``TRACE`` macros, but the advantage is that it doesn't require -any modifications to the code to use, and can give you more precise -location information. + solver:type=pvode solver:debug_on_failure=true solver:min_timestep=1e2 + +It is also possible to dump at a specific time using the euler solver. +This can be useful for tracking down what is causing differences between two +different versions. It can be used with:: + + solver:type=euler solver:dump_at_time=0 input:error_on_unused_options=false diff --git a/manual/sphinx/figs/figure_creators/LaplacianMatrices.ipynb b/manual/sphinx/figs/figure_creators/LaplacianMatrices.ipynb index c3c39a3c12..4e940b998f 100644 --- a/manual/sphinx/figs/figure_creators/LaplacianMatrices.ipynb +++ b/manual/sphinx/figs/figure_creators/LaplacianMatrices.ipynb @@ -40,10 +40,10 @@ "outputs": [], "source": [ "# Note, these can be max 9 due to the current index convention\n", - "nx = 3 # Not including ghost points\n", + "nx = 3 # Not including ghost points\n", "nz = 3\n", "\n", - "startXIndex = 10 # The x indices are the slowest growing indices\n", + "startXIndex = 10 # The x indices are the slowest growing indices\n", "startZIndex = 1 # The z indices are the fastest growing indices" ] }, @@ -85,13 +85,13 @@ "for z in range(nz):\n", " f.append([])\n", " xStart = startZIndex\n", - " xEnd = startXIndex*(nx+1) # +1 due to ghost point\n", + " xEnd = startXIndex * (nx + 1) # +1 due to ghost point\n", " # +startXIndex in the range as the range does not include endpoint\n", - " for xInd in range(xStart, xEnd+startXIndex, 10):\n", - " ind = str(xInd+z)\n", - " if (xInd+z) < startXIndex:\n", - " ind = '0'+str(xInd+z)\n", - " f[z].append(symbols('f_' + ind))\n", + " for xInd in range(xStart, xEnd + startXIndex, 10):\n", + " ind = str(xInd + z)\n", + " if (xInd + z) < startXIndex:\n", + " ind = \"0\" + str(xInd + z)\n", + " f[z].append(symbols(\"f_\" + ind))\n", "\n", "mesh = Matrix(f[::-1])\n", "display(mesh)" @@ -195,21 +195,21 @@ } ], "source": [ - "xVec=[]\n", - "bVec=[]\n", + "xVec = []\n", + "bVec = []\n", "# Do the inner loop, so start ranges at 1\n", "# (nx+1) to include outer ghost point, +1 in the range as the range does not include endpoint\n", - "for x in range(1, (nx+1)+1):\n", - " for z in range(1,nz+1):\n", - " xVec.append(symbols('x_'+str(x)+'_'+str(z)))\n", - " bVec.append(symbols('b_'+str(x)+'_'+str(z)))\n", + "for x in range(1, (nx + 1) + 1):\n", + " for z in range(1, nz + 1):\n", + " xVec.append(symbols(\"x_\" + str(x) + \"_\" + str(z)))\n", + " bVec.append(symbols(\"b_\" + str(x) + \"_\" + str(z)))\n", "\n", "# Do the inner ghost points\n", "# Must count backwards since we are inserting in the front\n", - "for ind in range(nz,0,-1):\n", - " xVec.insert(0, symbols('x_0_'+str(ind)))\n", - " bVec.insert(0, symbols('b_0_'+str(ind)))\n", - " \n", + "for ind in range(nz, 0, -1):\n", + " xVec.insert(0, symbols(\"x_0_\" + str(ind)))\n", + " bVec.insert(0, symbols(\"b_0_\" + str(ind)))\n", + "\n", "display(Matrix(xVec))\n", "display(Matrix(bVec))" ] @@ -274,7 +274,7 @@ "globInd = []\n", "for rows in range(len(xVec)):\n", " cols = []\n", - " for col in range(rows*len(xVec), (rows+1)*len(xVec)):\n", + " for col in range(rows * len(xVec), (rows + 1) * len(xVec)):\n", " cols.append(col)\n", " globInd.append(cols)\n", "\n", @@ -305,10 +305,14 @@ "source": [ "c = []\n", "for x in range(nx):\n", - " indexStart = (startXIndex+1)+(startXIndex*x) # Multiply by 10 due to index system\n", - " indexEnd = (startXIndex+nz+1)+(startXIndex*x) # Multiply by 10 due to index system\n", + " indexStart = (startXIndex + 1) + (\n", + " startXIndex * x\n", + " ) # Multiply by 10 due to index system\n", + " indexEnd = (startXIndex + nz + 1) + (\n", + " startXIndex * x\n", + " ) # Multiply by 10 due to index system\n", " for ind in range(indexStart, indexEnd):\n", - " c.append(symbols('c_'+str(ind)))" + " c.append(symbols(\"c_\" + str(ind)))" ] }, { @@ -328,11 +332,11 @@ "source": [ "# The inner ghost\n", "innerGhostStart = startZIndex\n", - "innerGhostEnd = nz\n", + "innerGhostEnd = nz\n", "ig = []\n", "# +1 in the range as last point is not included\n", - "for z in range(innerGhostStart, innerGhostEnd+1):\n", - " ig.append(symbols('ig_0_'+str(z)))" + "for z in range(innerGhostStart, innerGhostEnd + 1):\n", + " ig.append(symbols(\"ig_0_\" + str(z)))" ] }, { @@ -345,12 +349,12 @@ "source": [ "# The outer ghost\n", "# nx+1 as we want to go past the last inner x grid point\n", - "outerGhostStart = startXIndex*(nx+1) + startZIndex\n", - "outerGhostEnd = startXIndex*(nx+1) + nz\n", + "outerGhostStart = startXIndex * (nx + 1) + startZIndex\n", + "outerGhostEnd = startXIndex * (nx + 1) + nz\n", "og = []\n", "# +1 in the range as last point is not included\n", - "for z in range(outerGhostStart, outerGhostEnd+1):\n", - " og.append(symbols('og_'+str(z)))" + "for z in range(outerGhostStart, outerGhostEnd + 1):\n", + " og.append(symbols(\"og_\" + str(z)))" ] }, { @@ -396,21 +400,27 @@ "for x in range(nx):\n", " # The indices referring to the matrix index\n", " # The last -1 is there as the matrix indices count from 0\n", - " startRow = (nz+1)+(x*nz)-1 # Starting at row+1 after inner ghost point sub-matrix\n", - " endRow = (nz+1)+(x*nz)+(nz-1)-1 # Ending row-1 before the last z-index (last z will be wrapped around)\n", + " startRow = (\n", + " (nz + 1) + (x * nz) - 1\n", + " ) # Starting at row+1 after inner ghost point sub-matrix\n", + " endRow = (\n", + " (nz + 1) + (x * nz) + (nz - 1) - 1\n", + " ) # Ending row-1 before the last z-index (last z will be wrapped around)\n", " # +1 in range as last point is not included\n", - " rows = range(startRow, endRow+1)\n", - " cols = range(startRow+1, endRow+1) # Column is shifted +1 from the diagonal\n", - " \n", + " rows = range(startRow, endRow + 1)\n", + " cols = range(startRow + 1, endRow + 1) # Column is shifted +1 from the diagonal\n", + "\n", " # The indices referring to the spatial point in the grid\n", " # The last \"+1\" is fue to the fact that the column is shifted +1 from the diagonal\n", - " startInd = (startXIndex+startZIndex) + (startXIndex*x) + 1\n", - " endInd = (startXIndex+startZIndex) + (nz-1) + (startXIndex*x) + 1 # Wrap around last point\n", + " startInd = (startXIndex + startZIndex) + (startXIndex * x) + 1\n", + " endInd = (\n", + " (startXIndex + startZIndex) + (nz - 1) + (startXIndex * x) + 1\n", + " ) # Wrap around last point\n", " # +1 in range as last point is not included\n", - " inds = range(startInd, endInd+1)\n", - " \n", + " inds = range(startInd, endInd + 1)\n", + "\n", " for rInd, cInd, ind in zip(rows, cols, inds):\n", - " InvM[rInd, cInd] = symbols('zp_'+str(ind))" + " InvM[rInd, cInd] = symbols(\"zp_\" + str(ind))" ] }, { @@ -423,15 +433,17 @@ "source": [ "# The wrap around\n", "# The index referring to the spatial point in the grid\n", - "startInd = startXIndex+startZIndex\n", + "startInd = startXIndex + startZIndex\n", "# The indices referring to the matrix index\n", - "# Last -1 as the matrix indices are counted from 0 \n", - "startRow = (nz+1) + (nz-1) - 1 # nz+1 below from the ghost sub matrix, nz-1 below after that\n", - "startCol = (nz+1)-1 # nz+1 left of the ghost sub matrix\n", + "# Last -1 as the matrix indices are counted from 0\n", + "startRow = (\n", + " (nz + 1) + (nz - 1) - 1\n", + ") # nz+1 below from the ghost sub matrix, nz-1 below after that\n", + "startCol = (nz + 1) - 1 # nz+1 left of the ghost sub matrix\n", "for wrap in range(nx):\n", - " row = startRow+wrap*nz\n", - " col = startCol+wrap*nz\n", - " InvM[row, col] = symbols('zp_'+str(startInd+startXIndex*wrap))" + " row = startRow + wrap * nz\n", + " col = startCol + wrap * nz\n", + " InvM[row, col] = symbols(\"zp_\" + str(startInd + startXIndex * wrap))" ] }, { @@ -452,20 +464,26 @@ "for x in range(nx):\n", " # The indices referring to the matrix index\n", " # The last -1 is there as the matrix indices count from 0\n", - " startRow = (nz+1)+(x*nz)-1 # Starting at row+1 after inner ghost point sub-matrix\n", - " endRow = (nz+1)+(x*nz)+(nz-1)-1 # Ending row-1 before the last z-index (last z will be wrapped around)\n", + " startRow = (\n", + " (nz + 1) + (x * nz) - 1\n", + " ) # Starting at row+1 after inner ghost point sub-matrix\n", + " endRow = (\n", + " (nz + 1) + (x * nz) + (nz - 1) - 1\n", + " ) # Ending row-1 before the last z-index (last z will be wrapped around)\n", " # +1 in range as last point is not included\n", - " rows = range(startRow+1, endRow+1) # Row is shifted +1 from the diagonal\n", - " cols = range(startRow, endRow+1)\n", - " \n", + " rows = range(startRow + 1, endRow + 1) # Row is shifted +1 from the diagonal\n", + " cols = range(startRow, endRow + 1)\n", + "\n", " # The indices referring to the spatial point in the grid\n", - " startInd = (startXIndex+startZIndex) + (startXIndex*x)\n", - " endInd = (startXIndex+startZIndex) + (nz-1) + (startXIndex*x) # Wrap around last point\n", + " startInd = (startXIndex + startZIndex) + (startXIndex * x)\n", + " endInd = (\n", + " (startXIndex + startZIndex) + (nz - 1) + (startXIndex * x)\n", + " ) # Wrap around last point\n", " # +1 in range as last point is not included\n", - " inds = range(startInd, endInd+1)\n", - " \n", + " inds = range(startInd, endInd + 1)\n", + "\n", " for rInd, cInd, ind in zip(rows, cols, inds):\n", - " InvM[rInd, cInd] = symbols('zm_'+str(ind))" + " InvM[rInd, cInd] = symbols(\"zm_\" + str(ind))" ] }, { @@ -478,15 +496,19 @@ "source": [ "# The wrap around\n", "# The index referring to the spatial point in the grid\n", - "startInd = startXIndex+startZIndex+(nz-1) # +(nz-1) as this will be the last z point for the current x\n", + "startInd = (\n", + " startXIndex + startZIndex + (nz - 1)\n", + ") # +(nz-1) as this will be the last z point for the current x\n", "# The indices referring to the matrix index\n", - "# Last -1 as the matrix indices are counted from 0 \n", - "startRow = (nz+1)-1 # nz+1 below the ghost sub matrix\n", - "startCol = (nz+1) + (nz-1) - 1 # nz+1 left from the ghost sub matrix, nz-1 left after that\n", + "# Last -1 as the matrix indices are counted from 0\n", + "startRow = (nz + 1) - 1 # nz+1 below the ghost sub matrix\n", + "startCol = (\n", + " (nz + 1) + (nz - 1) - 1\n", + ") # nz+1 left from the ghost sub matrix, nz-1 left after that\n", "for wrap in range(nx):\n", - " row = startRow+wrap*nz\n", - " col = startCol+wrap*nz\n", - " InvM[row, col] = symbols('zm_'+str(startInd+startXIndex*wrap))" + " row = startRow + wrap * nz\n", + " col = startCol + wrap * nz\n", + " InvM[row, col] = symbols(\"zm_\" + str(startInd + startXIndex * wrap))" ] }, { @@ -505,23 +527,29 @@ "outputs": [], "source": [ "# Indices referring to the spatial points in the grid\n", - "startInd = startXIndex*2 + startZIndex # *2 as we start at the second inner x-index\n", - "endInd = startInd + (startZIndex*nz) # *nz as this is the last z-index in the current x-index\n", + "startInd = startXIndex * 2 + startZIndex # *2 as we start at the second inner x-index\n", + "endInd = startInd + (\n", + " startZIndex * nz\n", + ") # *nz as this is the last z-index in the current x-index\n", "\n", "for x in range(nx):\n", " # The indices referring to the matrix index\n", " # The last -1 as the matrix indices counts from 0\n", - " startRow = (nz+1)+(x*nz)-1 # Starting at row+1 after inner ghost point sub-matrix\n", - " endRow = (nz+1)+(x*nz)+(nz)-1 # Ending at the row referring to the last z-index\n", + " startRow = (\n", + " (nz + 1) + (x * nz) - 1\n", + " ) # Starting at row+1 after inner ghost point sub-matrix\n", + " endRow = (\n", + " (nz + 1) + (x * nz) + (nz) - 1\n", + " ) # Ending at the row referring to the last z-index\n", " # Not +1 in range as we do not want to include last point\n", - " rows = range(startRow, endRow)\n", - " cols = range(startRow+nz, endRow+nz) # Start at first index after last z-index\n", - " \n", + " rows = range(startRow, endRow)\n", + " cols = range(startRow + nz, endRow + nz) # Start at first index after last z-index\n", + "\n", " # Indices referring to the spatial points in the grid\n", - " inds = range(startInd+startXIndex*x, endInd+startXIndex*x)\n", - " \n", + " inds = range(startInd + startXIndex * x, endInd + startXIndex * x)\n", + "\n", " for rInd, cInd, ind in zip(rows, cols, inds):\n", - " InvM[rInd, cInd] = symbols('xp_'+str(ind))" + " InvM[rInd, cInd] = symbols(\"xp_\" + str(ind))" ] }, { @@ -534,22 +562,22 @@ "source": [ "# x+1 for inner ghost point\n", "# Indices referring to the spatial points in the grid\n", - "startInd = startXIndex + startZIndex # First inner point for first z\n", - "endInd = startInd + (startZIndex*nz) # First inner point for last z\n", + "startInd = startXIndex + startZIndex # First inner point for first z\n", + "endInd = startInd + (startZIndex * nz) # First inner point for last z\n", "\n", "# The indices referring to the matrix index\n", "# The last -1 as the matrix indices counts from 0\n", - "startRow = startZIndex-1 # Starting at first row\n", - "endRow = startZIndex+nz-1 # Ending at the row referring to the last z-index\n", + "startRow = startZIndex - 1 # Starting at first row\n", + "endRow = startZIndex + nz - 1 # Ending at the row referring to the last z-index\n", "# Not +1 in range as we do not want to include last point\n", - "rows = range(startRow, endRow)\n", - "cols = range(startRow+nz, endRow+nz) # Start at first index after last z-index\n", - " \n", + "rows = range(startRow, endRow)\n", + "cols = range(startRow + nz, endRow + nz) # Start at first index after last z-index\n", + "\n", "# Indices referring to the spatial points in the grid\n", - "inds = range(startInd, endInd)\n", - " \n", + "inds = range(startInd, endInd)\n", + "\n", "for rInd, cInd, ind in zip(rows, cols, inds):\n", - " InvM[rInd, cInd] = symbols('igxp_'+str(ind))" + " InvM[rInd, cInd] = symbols(\"igxp_\" + str(ind))" ] }, { @@ -569,25 +597,25 @@ "source": [ "# Indices referring to the spatial points in the grid\n", "startInd = startZIndex\n", - "endInd = startInd + nz\n", + "endInd = startInd + nz\n", "\n", "for x in range(nx):\n", " # The indices referring to the matrix index\n", " # Note that x starts counting from zero, so we must add 1 to x in the rows\n", - " startRow = ((x+1)*nz) # Starting at row+1 after inner ghost point sub-matrix\n", - " endRow = ((x+1)*nz)+(nz) # Ending at the row referring to the last z-index\n", + " startRow = (x + 1) * nz # Starting at row+1 after inner ghost point sub-matrix\n", + " endRow = ((x + 1) * nz) + (nz) # Ending at the row referring to the last z-index\n", " # Not +1 in range as we do not want to include last point\n", - " rows = range(startRow, endRow)\n", - " cols = range(startRow-nz, endRow-nz) # Start at first index after last z-index\n", - " \n", + " rows = range(startRow, endRow)\n", + " cols = range(startRow - nz, endRow - nz) # Start at first index after last z-index\n", + "\n", " # Indices referring to the spatial points in the grid\n", - " inds = range(startInd+startXIndex*x, endInd+startXIndex*x)\n", - " \n", + " inds = range(startInd + startXIndex * x, endInd + startXIndex * x)\n", + "\n", " for rInd, cInd, ind in zip(rows, cols, inds):\n", " if (ind) < startXIndex:\n", - " ind = '0'+str(ind)\n", + " ind = \"0\" + str(ind)\n", "\n", - " InvM[rInd, cInd] = symbols('xm_'+str(ind))" + " InvM[rInd, cInd] = symbols(\"xm_\" + str(ind))" ] }, { @@ -600,22 +628,24 @@ "source": [ "# x-1 for inner ghost point\n", "# Indices referring to the spatial points in the grid\n", - "startInd = startXIndex*nx + startZIndex # Last inner point for first z\n", - "endInd = startInd + (startZIndex*nz) # Last inner point for last z\n", + "startInd = startXIndex * nx + startZIndex # Last inner point for first z\n", + "endInd = startInd + (startZIndex * nz) # Last inner point for last z\n", "\n", "# The indices referring to the matrix index\n", "# The last -1 as the matrix indices counts from 0\n", - "startRow = len(xVec)-nz-1 # Starting at last inner point row\n", - "endRow = len(xVec)-1 # Ending at the last row\n", + "startRow = len(xVec) - nz - 1 # Starting at last inner point row\n", + "endRow = len(xVec) - 1 # Ending at the last row\n", "# +1 in range as last point is not included\n", - "rows = range(startRow+1, endRow+1)\n", - "cols = range(startRow-nz+1, endRow-nz+1) # Start at first index after last z-index\n", - " \n", + "rows = range(startRow + 1, endRow + 1)\n", + "cols = range(\n", + " startRow - nz + 1, endRow - nz + 1\n", + ") # Start at first index after last z-index\n", + "\n", "# Indices referring to the spatial points in the grid\n", - "inds = range(startInd, endInd)\n", - " \n", + "inds = range(startInd, endInd)\n", + "\n", "for rInd, cInd, ind in zip(rows, cols, inds):\n", - " InvM[rInd, cInd] = symbols('ogxm_'+str(ind))" + " InvM[rInd, cInd] = symbols(\"ogxm_\" + str(ind))" ] }, { diff --git a/manual/sphinx/figs/figure_creators/bout_runners_folder_structure.py b/manual/sphinx/figs/figure_creators/bout_runners_folder_structure.py index 8a95a13436..5a4a2a0843 100644 --- a/manual/sphinx/figs/figure_creators/bout_runners_folder_structure.py +++ b/manual/sphinx/figs/figure_creators/bout_runners_folder_structure.py @@ -10,10 +10,10 @@ mmag@fysik.dtu.dk """ -__authors__ = 'Michael Loeiten' -__email__ = 'mmag@fysik.dtu.dk' -__version__ = '1.0' -__date__ = '21.01.2016' +__authors__ = "Michael Loeiten" +__email__ = "mmag@fysik.dtu.dk" +__version__ = "1.0" +__date__ = "21.01.2016" import pygraphviz as pgv @@ -21,113 +21,118 @@ tree = pgv.AGraph() # Appendable lists -files = [] +files = [] dead_ends = [] # Default node attributes -tree.node_attr['shape']='box' -tree.node_attr['style']='bold' +tree.node_attr["shape"] = "box" +tree.node_attr["style"] = "bold" # Adding nodes and edges -l0 = 'project' -l1 = ['data', 'source\nfiles', 'driver.py'] +l0 = "project" +l1 = ["data", "source\nfiles", "driver.py"] # Append the files files.append(l1[1]) files.append(l1[2]) # Add the boxes to the mother node for box in l1: - tree.add_edge(l0,box) + tree.add_edge(l0, box) -l2 = ['solver1', 'solver2',\ - 'BOUT.inp', 'run_log.txt'] +l2 = ["solver1", "solver2", "BOUT.inp", "run_log.txt"] # Append the files files.append(l2[2]) files.append(l2[3]) # Add the boxes to the mother node for box in l2: - tree.add_edge('data', box) -tree.add_edge('solver2', 'solver2/...') + tree.add_edge("data", box) +tree.add_edge("solver2", "solver2/...") # Append the dead_end -de = l2[1] + '/...' +de = l2[1] + "/..." dead_ends.append(de) -l3 = ['method1', 'method2', 'solver1/...'] +l3 = ["method1", "method2", "solver1/..."] for box in l3: - tree.add_edge('solver1', box) -tree.add_edge('method2', 'method2/...') + tree.add_edge("solver1", box) +tree.add_edge("method2", "method2/...") # Append the dead_end de = l3[2] dead_ends.append(de) -de = l3[1] + '/...' +de = l3[1] + "/..." dead_ends.append(de) -l4 = ['nout\ntimestep1', 'nout\ntimestep2', 'method1/...'] +l4 = ["nout\ntimestep1", "nout\ntimestep2", "method1/..."] for box in l4: - tree.add_edge('method1', box) -tree.add_edge('nout\ntimestep2', 'nout\ntimestep2/...') + tree.add_edge("method1", box) +tree.add_edge("nout\ntimestep2", "nout\ntimestep2/...") # Append the dead_end de = l4[2] dead_ends.append(de) -de = l4[1] + '/...' +de = l4[1] + "/..." dead_ends.append(de) -l5 = ['mesh1', 'mesh2', 'nout\ntimestep1/...'] +l5 = ["mesh1", "mesh2", "nout\ntimestep1/..."] for box in l5: - tree.add_edge('nout\ntimestep1', box) -tree.add_edge('mesh2', 'mesh2/...') + tree.add_edge("nout\ntimestep1", box) +tree.add_edge("mesh2", "mesh2/...") # Append the dead_end de = l5[2] dead_ends.append(de) -de = l5[1] + '/...' +de = l5[1] + "/..." dead_ends.append(de) -l6 = ['additional1', 'additional2', 'mesh1/...'] +l6 = ["additional1", "additional2", "mesh1/..."] for box in l6: - tree.add_edge('mesh1', box) -tree.add_edge('additional2', 'additional2/...') + tree.add_edge("mesh1", box) +tree.add_edge("additional2", "additional2/...") # Append the dead_end de = l6[2] dead_ends.append(de) -de = l6[1] + '/...' +de = l6[1] + "/..." dead_ends.append(de) -l7 = ['grid_file1', 'grid_file2', 'additional1/...'] +l7 = ["grid_file1", "grid_file2", "additional1/..."] for box in l7: - tree.add_edge('additional1', box) -tree.add_edge('grid_file2', 'grid_file2/...') + tree.add_edge("additional1", box) +tree.add_edge("grid_file2", "grid_file2/...") # Append the dead_end de = l7[2] dead_ends.append(de) -de = l7[1] + '/...' +de = l7[1] + "/..." dead_ends.append(de) -l8 = ['BOUT.inp\n(copy)', 'BOUT.log', 'BOUT.dmp',\ - 'BOUT.restart', '(source_files\n(copy))', '(grid_file\n(copy))'] +l8 = [ + "BOUT.inp\n(copy)", + "BOUT.log", + "BOUT.dmp", + "BOUT.restart", + "(source_files\n(copy))", + "(grid_file\n(copy))", +] # Add l8 to the files list for cur_file in l8: files.append(cur_file) # Append them to the mother node for box in l8: - tree.add_edge('grid_file1', box) + tree.add_edge("grid_file1", box) # Change colors for the files for the_file in files: - member=tree.get_node(the_file) -# member.attr['fontcolor'] = 'limegreen' - member.attr['color'] = 'limegreen' + member = tree.get_node(the_file) + # member.attr['fontcolor'] = 'limegreen' + member.attr["color"] = "limegreen" # Change colors for the dead_ends for dead_end in dead_ends: - member=tree.get_node(dead_end) -# member.attr['fontcolor'] = 'darksalmon' - member.attr['color'] = 'darksalmon' + member = tree.get_node(dead_end) + # member.attr['fontcolor'] = 'darksalmon' + member.attr["color"] = "darksalmon" # Print the graph print(tree.string()) # Set layout -tree.layout('dot') +tree.layout("dot") # Write to file -tree.draw('folder_tree.svg') +tree.draw("folder_tree.svg") diff --git a/manual/sphinx/user_docs/boundary_options.rst b/manual/sphinx/user_docs/boundary_options.rst index a3cdf0078b..8493049516 100644 --- a/manual/sphinx/user_docs/boundary_options.rst +++ b/manual/sphinx/user_docs/boundary_options.rst @@ -575,7 +575,7 @@ and implemented in ``boundary_standard.cxx`` void BoundaryNeumann::apply(Field3D &f) { for(bndry->first(); !bndry->isDone(); bndry->next()) - for(int z=0;zLocalNz;z++) + for(int z= mesh->zstart; z <= mesh->zend;z++) f[bndry->x][bndry->y][z] = f[bndry->x - bndry->bx][bndry->y - bndry->by][z]; } diff --git a/manual/sphinx/user_docs/bout_options.rst b/manual/sphinx/user_docs/bout_options.rst index 85a8a17d59..df244cd7c1 100644 --- a/manual/sphinx/user_docs/bout_options.rst +++ b/manual/sphinx/user_docs/bout_options.rst @@ -554,42 +554,42 @@ may be useful anyway. See :ref:`sec-output` for more details. Input and Output ---------------- -The output (dump) files with time-history are controlled by settings -in a section called “output”. Restart files contain a single -time-slice, and are controlled by a section called “restart”. The -options available are listed in table :numref:`tab-outputopts`. +The output (dump) files with time-history are controlled by settings in a +section called ``"output"``. Restart files contain a single time-slice, and are +controlled by a section called ``"restart"``. The options available are listed +in table :numref:`tab-outputopts`. .. _tab-outputopts: .. table:: Output file options - +-------------+----------------------------------------------------+--------------+ - | Option | Description | Default | - | | | value | - +-------------+----------------------------------------------------+--------------+ - | enabled | Writing is enabled | true | - +-------------+----------------------------------------------------+--------------+ - | type | File type e.g. "netcdf" or "adios" | "netcdf" | - +-------------+----------------------------------------------------+--------------+ - | prefix | File name prefix | "BOUT.dmp" | - +-------------+----------------------------------------------------+--------------+ - | path | Directory to write the file into | ``datadir`` | - +-------------+----------------------------------------------------+--------------+ - | floats | Write floats rather than doubles | false | - +-------------+----------------------------------------------------+--------------+ - | flush | Flush the file to disk after each write | true | - +-------------+----------------------------------------------------+--------------+ - | guards | Output guard cells | true | - +-------------+----------------------------------------------------+--------------+ - | openclose | Re-open the file for each write, and close after | true | - +-------------+----------------------------------------------------+--------------+ + +----------------------+-----------------------------------------+----------------+ + | Option | Description | Default value | + +======================+=========================================+================+ + | ``append`` | Append to existing file if true, | ``false`` | + | | otherwise overwrite | | + +----------------------+-----------------------------------------+----------------+ + | ``enabled`` | Writing is enabled | ``true`` | + +----------------------+-----------------------------------------+----------------+ + | ``flush_frequency`` | How many output timesteps between | ``1`` | + | | writing output to disk (NetCDF only) | | + +----------------------+-----------------------------------------+----------------+ + | ``prefix`` | File name prefix | ``"BOUT.dmp"`` | + +----------------------+-----------------------------------------+----------------+ + | ``path`` | Directory to write the file into | ``datadir`` | + +----------------------+-----------------------------------------+----------------+ + | ``type`` | File type, either ``"netcdf"`` or | ``"netcdf"`` | + | | ``"adios"`` | | + +----------------------+-----------------------------------------+----------------+ | -**enabled** is useful mainly for doing performance or scaling tests, where you -want to exclude I/O from the timings. **floats** can be used to reduce the size -of the output files: files are stored as double by default, but setting -**floats = true** changes the output to single-precision floats. - +- ``enabled`` is useful mainly for doing performance or scaling tests, where you + want to exclude I/O from the timings. +- If you find that IO is taking more and more time as your simulation goes on, + try setting ``flush_frequency`` to a larger value such as ``10``. This can + workaround an issue with NetCDF where subsequent writes take longer and + longer. However, larger values risk losing more data in the event of a crash + or the simulation being killed early. Implementation -------------- @@ -889,7 +889,7 @@ Fields can also be stored and written:: Options fields; fields["f2d"] = Field2D(1.0); fields["f3d"] = Field3D(2.0); - bout::OptionsIO::create("fields.nc").write(fields); + bout::OptionsIO::create("fields.nc")->write(fields); This allows the input settings and evolving variables to be combined into a single tree (see above on joining trees) and written @@ -950,7 +950,7 @@ automatically set the ``"time_dimension"`` attribute:: data["field"] = Field3D(3.0); // Append data to file - bout::OptionsIO({{"file", "time.nc"}, {"append", true}})->write(data); + bout::OptionsIO::create({{"file", "time.nc"}, {"append", true}})->write(data); .. note:: By default, `bout::OptionsIO::write` will only write variables with a ``"time_dimension"`` of ``"t"``. You can write diff --git a/manual/sphinx/user_docs/installing.rst b/manual/sphinx/user_docs/installing.rst index 10f5d9b9f1..a493f932c4 100644 --- a/manual/sphinx/user_docs/installing.rst +++ b/manual/sphinx/user_docs/installing.rst @@ -332,10 +332,13 @@ The default build configuration options try to be sensible for new users and developers, but there are a few you probably want to set manually for production runs or for debugging: -* ``CMAKE_BUILD_TYPE``: The default is ``RelWithDebInfo``, which - builds an optimised executable with debug symbols included. Change - this to ``Release`` to remove the debug symbols, or ``Debug`` for an - unoptimised build, but better debug experience +* ``CMAKE_BUILD_TYPE``: The default is ``RelWithDebInfo``, which builds an + optimised executable with debug symbols included. This is generally the most + useful, except for developers, who may wish to use ``Debug`` for an + unoptimised build, but better debug experience. There are a couple of other + choices (``Release`` and ``MinSizeRel``) which also produce optimised + executables, but without debug symbols, which is only really useful for + producing smaller binaries. * ``CHECK``: This sets the level of internal runtime checking done in the BOUT++ library, and ranges from 0 to 4 (inclusive). By default, @@ -380,16 +383,32 @@ to read and write this high-performance parallel file format. Bundled Dependencies ~~~~~~~~~~~~~~~~~~~~ -BOUT++ bundles some dependencies, currently `mpark.variant -`_, `fmt `_ and -`googletest `_. If you wish to -use an existing installation of ``mpark.variant``, you can set -``-DBOUT_USE_SYSTEM_MPARK_VARIANT=ON``, and supply the installation -path using ``mpark_variant_ROOT`` via the command line or environment -variable if it is installed in a non standard loction. Similarly for -``fmt``, using ``-DBOUT_USE_SYSTEM_FMT=ON`` and ``fmt_ROOT`` -respectively. To turn off both, you can set -``-DBOUT_USE_GIT_SUBMODULE=OFF``. +BOUT++ bundles some dependencies, currently: + +- `mpark.variant `_ +- `fmt `_ +- `cpptrace `_ +- ``googletest `_ (for unit tests) + +Aside from ``googletest``, the others are required dependencies and can either +be built as part of the BOUT++ build, or provided externally. If you wish to use +existing installations of some of these, set the following flags: + ++--------------------+-----------------------------------+------------------------+ +| Name | Flag for external installation | Library path | ++====================+===================================+========================+ +| ``mpark.variant`` | ``BOUT_USE_SYSTEM_MPARK_VARIANT`` | ``mpark_variant_ROOT`` | ++--------------------+-----------------------------------+------------------------+ +| ``fmt`` | ``BOUT_USE_SYSTEM_FMT`` | ``fmt_ROOT`` | ++--------------------+-----------------------------------+------------------------+ +| ``cpptrace`` | ``BOUT_USE_SYSTEM_CPPTRACE`` | ``cpptrace_ROOT`` | ++--------------------+-----------------------------------+------------------------+ + +You can also set ``-DBOUT_USE_GIT_SUBMODULE=OFF`` to not use any of the bundled +versions. + +If the libraries are in non-standard locations, you may also need to supply the +relevant library path flags. The recommended way to use ``googletest`` is to compile it at the same time as your project, therefore there is no option to use an external diff --git a/manual/sphinx/user_docs/invertable_operator.rst b/manual/sphinx/user_docs/invertable_operator.rst index 92b7efd288..8f92796c9c 100644 --- a/manual/sphinx/user_docs/invertable_operator.rst +++ b/manual/sphinx/user_docs/invertable_operator.rst @@ -48,7 +48,6 @@ the ``operator()`` call:: // Drop C term for now Field3D operator()(const Field3D &input) { - TRACE("myLaplacian::operator()"); Timer timer("invertable_operator_operate"); Field3D result = A * input + D * Delp2(input); @@ -68,7 +67,6 @@ A more complete example is :: // Drop C term for now Field3D operator()(const Field3D &input) { - TRACE("myLaplacian::operator()"); Timer timer("invertable_operator_operate"); Field3D result = A * input + D * Delp2(input); diff --git a/manual/sphinx/user_docs/python_boutpp.rst b/manual/sphinx/user_docs/python_boutpp.rst index 939a817ffe..e79facbf55 100644 --- a/manual/sphinx/user_docs/python_boutpp.rst +++ b/manual/sphinx/user_docs/python_boutpp.rst @@ -160,7 +160,7 @@ A real example - check derivative contributions: phi[-1,:,z]=phi_arr with open(path+"/equilibrium/phi_eq.dat","rb") as inf: phi_arr=np.fromfile(inf,dtype=np.double) - bm="BRACKET_ARAKAWA_OLD" + bm="BRACKET_ARAKAWA" for tind in range(len(times)): vort = Field3D.fromCollect("vort" ,path=path,tind=tind,info=False) diff --git a/manual/sphinx/user_docs/time_integration.rst b/manual/sphinx/user_docs/time_integration.rst index c1526c14e2..37bf9a6abf 100644 --- a/manual/sphinx/user_docs/time_integration.rst +++ b/manual/sphinx/user_docs/time_integration.rst @@ -361,16 +361,394 @@ And the adaptive timestepping options: Backward Euler - SNES --------------------- -The `beuler` or `snes` solver type (either name can be used) is -intended mainly for solving steady-state problems, so integrates in -time using a stable but low accuracy method (Backward Euler). It uses -PETSc's SNES solvers to solve the nonlinear system at each timestep, -and adjusts the internal timestep to keep the number of SNES -iterations within a given range. +The `beuler` or `snes` solver type (either name can be used) is a PETSc-based implicit +solver for finding steady-state solutions to systems of partial differential equations. +It supports multiple solution strategies including backward Euler timestepping, +direct Newton iteration, and Pseudo-Transient Continuation (PTC) with Switched +Evolution Relaxation (SER). + +Basic Configuration +~~~~~~~~~~~~~~~~~~~ + +The SNES solver is configured through the ``[solver]`` section of the input file: + +.. code-block:: ini + + [solver] + type = snes + + # Nonlinear solver settings + snes_type = newtonls # anderson, newtonls, newtontr, nrichardson + atol = 1e-7 # Absolute tolerance + rtol = 1e-6 # Relative tolerance + stol = 1e-12 # Solution change tolerance + max_nonlinear_iterations = 20 # Maximum SNES iterations per solve + + # Linear solver settings + ksp_type = fgmres # Linear solver: gmres, bicgstab, etc. + maxl = 20 # Maximum linear iterations + pc_type = ilu # Preconditioner: ilu, bjacobi, hypre, etc. + +Timestepping Modes +~~~~~~~~~~~~~~~~~~ + +The solver supports several timestepping strategies controlled by ``equation_form``: + +**Backward Euler (default)** + Standard implicit backward Euler method. Good for general timestepping. + + .. code-block:: ini + + equation_form = rearranged_backward_euler # Default + + This method has low accuracy in time but its dissipative properties + are helpful when evolving to steady state solutions. + +**Direct Newton** + Solves the steady-state problem F(u) = 0 directly without timestepping. + + .. code-block:: ini + + equation_form = direct_newton + + This method is unlikely to converge unless the system is very close + to steady state. + +**Pseudo-Transient Continuation** + Uses pseudo-time to guide the solution to steady state. Recommended for + highly nonlinear problems where Newton's method fails. + + .. code-block:: ini + + equation_form = pseudo_transient + + This uses the same form as rearranged_backward_euler, but the time step + can be different for each cell. + +Adaptive Timestepping +~~~~~~~~~~~~~~~~~~~~~ + +When ``equation_form = rearranged_backward_euler`` (default), the +solver uses global timestepping with adaptive timestep control based +on nonlinear iteration count. + +.. code-block:: ini + + [solver] + type = snes + equation_form = rearranged_backward_euler + + # Initial and maximum timesteps + timestep = 1.0 # Initial timestep + max_timestep = 1e10 # Upper limit on timestep + dt_min_reset = 1e-6 # Reset the solver when timestep < this + + # Timestep adaptation + timestep_control = pid_nonlinear_its + target_its = 7 # Target number of nonlinear iterations + kP = 0.7 # Proportional gain + kI = 0.3 # Integral gain + kD = 0.2 # Derivative gain + +This uses a PID controller that adjusts the timestep to maintain approximately ``target_its`` +nonlinear iterations per solve. + +Residual Ratio +^^^^^^^^^^^^^^ + +This adjusts the timestep using the ratio of global residuals and a timestep factor: + +.. math:: + + dt_n = r dt_{n-1} \frac{||F(X_{n-1})||}{||F(X_{n})|| + +so that as the residual falls the timestep :math:`dt` is increased. +The :math:`r` parameter is input option ``timestep_factor`` that has +default value 1.1. + +.. code-block:: ini + + [solver] + timestep_control = residual_ratio # Use global residual + timestep_factor = 1.1 # Constant timestep factor + +Threshold Controller +^^^^^^^^^^^^^^^^^^^^ + +An alternative adaptive strategy uses thresholds in nonlinear iterations +to adjust the timestep: + +.. code-block:: ini + + [solver] + timestep_control = threshold_nonlinear_its + lower_its = 3 # Increase dt if iterations < this + upper_its = 10 # Decrease dt if iterations > this + timestep_factor_on_lower_its = 1.4 # Growth factor + timestep_factor_on_upper_its = 0.9 # Reduction factor + timestep_factor_on_failure = 0.5 # Reduction on convergence failure + +The adjustments are less smooth than the default PID method, but the +timestep is changed less frequently. This may enable the Jacobian and +preconditioner to be used for more iterations. + +Output trigger +~~~~~~~~~~~~~~ + +The default behavior is to save outputs at a regular time interval, as +BOUT++ solvers do. This is desirable when performing time-dependent +simulations, but for simulations that are trying to get to steady +state a better measure of progress is reduction of the global residual +(norm of the time-derivatives of the system). + +.. code-block:: ini + + [solver] + output_trigger = residual_ratio # Trigger an output based on the ratio of residuals + output_residual_ratio = 0.5 # Output when global residual is multiplied by this + +With this choice, each output has a global residual that is less than +or equal to `output_residual_ratio` times the last output global +residual. This provides a way of measuring progress to steady state +that is independent of time integration accuracy. + +Pseudo-Transient Continuation and Switched Evolution Relaxation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When ``equation_form = pseudo_transient`` the solver uses +Pseudo-Transient Continuation (PTC). This is a robust numerical +technique for solving steady-state problems that are too nonlinear for +direct Newton iteration. Instead of solving the steady-state system +**F(u) = 0** directly, PTC solves a modified time-dependent problem: + +.. math:: + + M(u) \frac{\partial u}{\partial \tau} + F(u) = 0 + +where :math:`\tau` is a pseudo-time variable (not physical time) and :math:`M(u)` +is a preconditioning matrix. As :math:`\tau \to \infty`, the solution converges +to the steady state **F(u) = 0**. + +The key advantage of PTC is that it transforms a difficult root-finding problem +into a sequence of easier initial value problems. Poor initial guesses that would +cause Newton's method to diverge can still reach the solution via a stable +pseudo-transient path. + +The Switched Evolution Relaxation (SER) method is a spatially adaptive +variant of PTC that allows each cell to use a different +pseudo-timestep :math:`\Delta\tau_i`. The timestep in each cell adapts +based on the local residual, allowing the algorithm to take large +timesteps in well-behaved regions (fast convergence), while taking +small timesteps in difficult regions (stable advancement). The the +same :math:`\Delta\tau_i` is used for all equations (density, +momentum, energy etc.) within each cell. This maintains coupling +between temperature, pressure, and composition through the equation of +state. + +**Key parameters:** + +``pseudo_max_ratio`` (default: 2.0) + Maximum allowed ratio of timesteps between neighboring cells. This prevents + sharp spatial gradients in convergence rate. + +**Example PTC configuration:** + +.. code-block:: ini + + [solver] + type = snes + equation_form = pseudo_transient + + timestep = 1.0 # Initial timestep + + # SER parameters + timestep_control = pid_nonlinear_its # Scale timesteps based on iterations + pseudo_max_ratio = 2.0 # Limit neighbor timestep ratio + + # Tolerances + atol = 1e-7 + rtol = 1e-6 + stol = 1e-12 + +SER timestep strategy +^^^^^^^^^^^^^^^^^^^^^ + +After each nonlinear solve the timesteps in each cell are adjusted. +The strategy used depends on the ``pseudo_strategy`` option: + +**inverse_residual** (default) + +If ``pseudo_strategy = inverse_residual`` then the timestep is inversely +proportional to the RMS residual in each cell. +``pseudo_alpha`` (default: 100 × atol × timestep) +Controls the relationship between residual and timestep. The local timestep +is computed as: + +.. math:: + + \Delta\tau_i = \frac{\alpha}{||R_i||} + +Larger values allow more aggressive timestepping. The default is to use +a fixed ``pseudo_alpha`` but a better strategy is to enable the PID controller +that adjusts this parameter based on the nonlinear solver convergence. + +The timestep is limited to be between ``dt_min_reset`` and +``max_timestep``. In addition the timestep is limited between 0.67 × +previous timestep and 1.5 × previous timestep, to limit sudden changes +in timestep. + +In practice this strategy seems to work well, though problems could +arise when residuals become very small. + +**history_based** + +When ``pseudo_strategy = history_based`` the history of residuals +within each cell is used to adjust the timestep. The key parameters +are: + +``pseudo_growth_factor`` (default: 1.1) + Factor by which timestep increases when residual decreases successfully. + +``pseudo_reduction_factor`` (default: 0.5) + Factor by which timestep decreases when residual increases (step rejected). + +This method may be less susceptible to fluctuations when residuals +become small, but tends to be slower to converge when residuals are +large. + +**hybrid** + +When ``pseudo_strategy = hybrid`` the ``inverse_residual`` and +``history_based`` strategies are combined: When the residuals are +large the ``inverse_residual`` method is used, and when residuals +become small the method switches to ``history_based``. + +PID Controller +^^^^^^^^^^^^^^ + +When using the PTC method the PID controller can be used to dynamically +adjust ``pseudo_alpha`` depending on the nonlinearity of the system: + +.. code-block:: ini + + [solver] + timestep_control = pid_nonlinear_its # Scale global timestep using PID controller + target_its = 7 # Target number of nonlinear iterations + kP = 0.7 # Proportional gain + kI = 0.3 # Integral gain + kD = 0.2 # Derivative gain + +The PID controller adjusts ``pseudo_alpha``, scaling all cell +timesteps together, to maintain approximately ``target_its`` nonlinear +iterations per solve. + +With this enabled the solver uses the number of nonlinear iterations +to scale timesteps globally, and residuals to scale timesteps locally. +Note that the PID controller has no effect on the ``history_based`` +strategy because that strategy does not use ``pseudo_alpha``. + +Jacobian Finite Difference with Coloring +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The default and recommended approach for most problems: + +.. code-block:: ini + + [solver] + use_coloring = true # Enable (default) + lag_jacobian = 5 # Reuse Jacobian for this many iterations + + # Stencil shape (determines Jacobian sparsity pattern) + stencil:taxi = 2 # Taxi-cab distance (default) + stencil:square = 0 # Square stencil extent + stencil:cross = 0 # Cross stencil extent + +The coloring algorithm exploits the sparse structure of the Jacobian to reduce +the number of function evaluations needed for finite differencing. + +Jacobian coloring stencil +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The stencil used to create the Jacobian colouring can be varied, +depending on which numerical operators are in use. It is important to +note that the coloring won't work for every problem: It assumes that +each evolving quantity is coupled to all other evolving quantities on +the same grid cell, and on all the neighbouring grid cells. If the RHS +function includes Fourier transforms, or matrix inversions +(e.g. potential solves) then these will introduce longer-range +coupling and the Jacobian calculation will give spurious +results. Generally the method will then fail to converge. Two +solutions are to a) switch to matrix-free (``matrix_free=true``), +or b) solve the matrix inversion as a constraint. + + +``solver:stencil:cross = N`` +e.g. for N == 2 + +.. code-block:: bash + + * + * + * * x * * + * + * + + +``solver:stencil:square = N`` +e.g. for N == 2 + +.. code-block:: bash + + * * * * * + * * * * * + * * x * * + * * * * * + * * * * * + +``solver:stencil:taxi = N`` +e.g. for N == 2 + +.. code-block:: bash + + * + * * * + * * x * * + * * * + * + +Setting ``solver:force_symmetric_coloring = true``, will make sure +that the jacobian colouring matrix is symmetric. This will often +include a few extra non-zeros that the stencil will miss otherwise + +Diagnostics and Monitoring +--------------------------- + +.. code-block:: ini + + [solver] + diagnose = true # Print iteration info to screen + diagnose_failures = true # Detailed diagnostics on failures + +When ``equation_form = pseudo_transient``, the solver saves additional diagnostic fields: + +- ``snes_pseudo_residual``: Local residual in each cell +- ``snes_pseudo_timestep``: Local pseudo-timestep in each cell +- ``snes_pseudo_alpha``: Global timestep scaling + +These can be visualized to understand convergence behavior and identify +problematic regions. + +Summary of solver options +~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------+---------------+----------------------------------------------------+ | Option | Default |Description | +===========================+===============+====================================================+ +| pseudo_time | false | Pseudo-Transient Continuation (PTC) method, using | +| | | a different timestep for each cell. | ++---------------------------+---------------+----------------------------------------------------+ +| pseudo_max_ratio | 2. | Maximum timestep ratio between neighboring cells | ++---------------------------+---------------+----------------------------------------------------+ | snes_type | newtonls | PETSc SNES nonlinear solver (try anderson, qn) | +---------------------------+---------------+----------------------------------------------------+ | ksp_type | gmres | PETSc KSP linear solver | @@ -427,16 +805,7 @@ The default `newtonls` SNES type can be very effective if combined with Jacobian coloring: The coloring enables the Jacobian to be calculated relatively efficiently; once a Jacobian matrix has been calculated, effective preconditioners can be used to speed up -convergence. It is important to note that the coloring assumes a star -stencil and so won't work for every problem: It assumes that each -evolving quantity is coupled to all other evolving quantities on the -same grid cell, and on all the neighbouring grid cells. If the RHS -function includes Fourier transforms, or matrix inversions -(e.g. potential solves) then these will introduce longer-range -coupling and the Jacobian calculation will give spurious -results. Generally the method will then fail to converge. Two -solutions are to a) switch to matrix-free (``matrix_free=true``), or b) -solve the matrix inversion as a constraint. +convergence. The `SNES type `_ @@ -456,50 +825,7 @@ Preconditioner types: Enable with command-line args ``-pc_type hypre -pc_hypre_type euclid -pc_hypre_euclid_levels k`` where ``k`` is the level (1-8 typically). -Jacobian coloring stencil -~~~~~~~~~~~~~~~~~~~~~~~~~ - -The stencil used to create the Jacobian colouring can be varied, -depending on which numerical operators are in use. - - -``solver:stencil:cross = N`` -e.g. for N == 2 - -.. code-block:: bash - - * - * - * * x * * - * - * - - -``solver:stencil:square = N`` -e.g. for N == 2 - -.. code-block:: bash - - * * * * * - * * * * * - * * x * * - * * * * * - * * * * * - -``solver:stencil:taxi = N`` -e.g. for N == 2 - -.. code-block:: bash - - * - * * * - * * x * * - * * * - * -Setting ``solver:force_symmetric_coloring = true``, will make sure -that the jacobian colouring matrix is symmetric. This will often -include a few extra non-zeros that the stencil will miss otherwise ODE integration --------------- diff --git a/manual/sphinx/user_docs/variable_init.rst b/manual/sphinx/user_docs/variable_init.rst index 4ac13e0ead..1baaba5e84 100644 --- a/manual/sphinx/user_docs/variable_init.rst +++ b/manual/sphinx/user_docs/variable_init.rst @@ -81,17 +81,19 @@ following values are also already defined: .. _tab-initexprvals: .. table:: Initialisation expression values - +--------+------------------------------------------------------------------------------------+ - | Name | Description | - +========+====================================================================================+ - | x | :math:`x` position between :math:`0` and :math:`1` | - +--------+------------------------------------------------------------------------------------+ - | y | :math:`y` angle-like position, definition depends on topology of grid | - +--------+------------------------------------------------------------------------------------+ - | z | :math:`z` position between :math:`0` and :math:`2\pi` (excluding the last point) | - +--------+------------------------------------------------------------------------------------+ - | pi π | :math:`3.1415\ldots` | - +--------+------------------------------------------------------------------------------------+ + +----------------+------------------------------------------------------------------------------------+ + | Name | Description | + +================+====================================================================================+ + | x | :math:`x` position between :math:`0` and :math:`1` | + +----------------+------------------------------------------------------------------------------------+ + | y | :math:`y` angle-like position, definition depends on topology of grid | + +----------------+------------------------------------------------------------------------------------+ + | z | :math:`z` position between :math:`0` and :math:`2\pi` (excluding the last point) | + +----------------+------------------------------------------------------------------------------------+ + | pi π | :math:`3.1415\ldots` | + +----------------+------------------------------------------------------------------------------------+ + | is_periodic_y | :math:`1` in core region where Y is periodic. :math:`0` otherwise | + +----------------+------------------------------------------------------------------------------------+ By default, :math:`x` is defined as ``(i+0.5) / (nx - 2*MXG)``, where ``MXG`` diff --git a/pyproject.toml b/pyproject.toml index a87e8bb674..f7ccf83cff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,3 +4,25 @@ requires = [] # Defined by PEP 517: build-backend = "backend" backend-path = ["tools/pylib/_boutpp_build/"] + +[dependency-groups] +dev = [ + "cmake~=4.2", + "clang-format~=22.0", + "clang-tidy~=22.0", + "clangd~=22.0", + "clangd-tidy~=1.1", + "cmakelang~=0.6", + "prek>=0.3.5", + "pyyaml~=6.0", + "ruff~=0.15", + "sphinx-lint~=1.0", + "sync-with-uv~=0.5.0", +] +maint = [ + "pygithub~=2.8", + "ruamel-yaml~=0.19", + "Unidecode~=1.3", +] + +[tool.uv.workspace] diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000000..ba0eeb91e2 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,6 @@ +clang-format~=22.0 +clang-tidy~=22.0 +cmakelang~=0.6 +pre-commit>=4.5.1 +ruff +sphinx-lint~=1.0 diff --git a/requirements_maint.txt b/requirements_maint.txt index b1d9db5ff8..be76672fd4 100644 --- a/requirements_maint.txt +++ b/requirements_maint.txt @@ -1,3 +1,3 @@ -pygithub~=2.8 -ruamel-yaml~=0.18 +pygithub~=2.9 +ruamel-yaml~=0.19 Unidecode~=1.3 diff --git a/src/bout++.cxx b/src/bout++.cxx index 48ab96298b..cad3edf6c3 100644 --- a/src/bout++.cxx +++ b/src/bout++.cxx @@ -27,21 +27,28 @@ #include "bout/build_config.hxx" -const char DEFAULT_DIR[] = "data"; +static constexpr auto DEFAULT_DIR = "data"; #define GLOBALORIGIN #include "bout++-time.hxx" +#include "bout/array.hxx" #include "bout/boundary_factory.hxx" +#include "bout/bout_types.hxx" #include "bout/boutcomm.hxx" #include "bout/boutexception.hxx" +#include "bout/build_defines.hxx" #include "bout/coordinates_accessor.hxx" +#include "bout/dcomplex.hxx" +#include "bout/globals.hxx" #include "bout/hyprelib.hxx" #include "bout/interpolation_xz.hxx" #include "bout/interpolation_z.hxx" #include "bout/invert/laplacexz.hxx" #include "bout/invert_laplace.hxx" #include "bout/invert_parderiv.hxx" +#include "bout/mask.hxx" +#include "bout/monitor.hxx" #include "bout/mpi_wrapper.hxx" #include "bout/msg_stack.hxx" #include "bout/openmpwrap.hxx" @@ -52,7 +59,9 @@ const char DEFAULT_DIR[] = "data"; #include "bout/rkscheme.hxx" #include "bout/slepclib.hxx" #include "bout/solver.hxx" +#include "bout/sys/gettext.hxx" #include "bout/sys/timer.hxx" +#include "bout/utils.hxx" #include "bout/version.hxx" #define BOUT_NO_USING_NAMESPACE_BOUTGLOBALS @@ -63,11 +72,22 @@ const char DEFAULT_DIR[] = "data"; #include "bout/adios_object.hxx" #endif +#include #include +#include +#include #include +#include +#include #include #include +#include +#include +#include +#include +#include +#include #include #include @@ -137,78 +157,72 @@ int BoutInitialise(int& argc, char**& argv) { try { args = parseCommandLineArgs(argc, argv); } catch (const BoutException& e) { - output_error << _("Bad command line arguments:\n") << e.what() << std::endl; + output_error.write("{:s}{:s}\n", _("Bad command line arguments:\n"), e.what()); return 1; } - try { - checkDataDirectoryIsAccessible(args.data_dir); + checkDataDirectoryIsAccessible(args.data_dir); - // Set the command-line arguments - SlepcLib::setArgs(argc, argv); // SLEPc initialisation - PetscLib::setArgs(argc, argv); // PETSc initialisation - Solver::setArgs(argc, argv); // Solver initialisation - BoutComm::setArgs(argc, argv); // MPI initialisation + // Set the command-line arguments + SlepcLib::setArgs(argc, argv); // SLEPc initialisation + PetscLib::setArgs(argc, argv); // PETSc initialisation + Solver::setArgs(argc, argv); // Solver initialisation + BoutComm::setArgs(argc, argv); // MPI initialisation - const int MYPE = BoutComm::rank(); + const int MYPE = BoutComm::rank(); - setupBoutLogColor(args.color_output, MYPE); + setupBoutLogColor(args.color_output, MYPE); - setupOutput(args.data_dir, args.log_file, args.verbosity, MYPE); + setupOutput(args.data_dir, args.log_file, args.verbosity, MYPE); - savePIDtoFile(args.data_dir, MYPE); + savePIDtoFile(args.data_dir, MYPE); #if BOUT_HAS_ADIOS2 - bout::ADIOSInit(BoutComm::get()); + bout::ADIOSInit(BoutComm::get()); #endif - // Print the different parts of the startup info - printStartupHeader(MYPE, BoutComm::size()); - printCompileTimeOptions(); - printCommandLineArguments(args.original_argv); - - // Load settings file - OptionsReader* reader = OptionsReader::getInstance(); - // Ideally we'd use the long options for `datadir` and - // `optionfile` here, but we'd need to call parseCommandLine - // _first_ in order to do that and set the source, etc., but we - // need to call that _second_ in order to override the input file - reader->read(Options::getRoot(), "{}", (args.data_dir / args.opt_file).string()); - - // Get options override from command-line - reader->parseCommandLine(Options::getRoot(), args.argv); - - // Get the variables back out so they count as having been used - // when checking for unused options. They normally _do_ get used, - // but it's possible that only happens in BoutFinalise, which is - // too late for that check. - const auto datadir = Options::root()["datadir"].withDefault(DEFAULT_DIR); - [[maybe_unused]] const auto optionfile = - Options::root()["optionfile"].withDefault(args.opt_file); - const auto settingsfile = - Options::root()["settingsfile"].withDefault(args.set_file); - - setRunStartInfo(Options::root()); - - if (MYPE == 0) { - writeSettingsFile(Options::root(), datadir, settingsfile); - } - - bout::globals::mpi = new MpiWrapper(); + // Print the different parts of the startup info + printStartupHeader(MYPE, BoutComm::size()); + printCompileTimeOptions(); + printCommandLineArguments(args.original_argv); + + // Load settings file + OptionsReader* reader = OptionsReader::getInstance(); + // Ideally we'd use the long options for `datadir` and + // `optionfile` here, but we'd need to call parseCommandLine + // _first_ in order to do that and set the source, etc., but we + // need to call that _second_ in order to override the input file + reader->read(Options::getRoot(), "{}", (args.data_dir / args.opt_file).string()); + + // Get options override from command-line + reader->parseCommandLine(Options::getRoot(), args.argv); + + // Get the variables back out so they count as having been used + // when checking for unused options. They normally _do_ get used, + // but it's possible that only happens in BoutFinalise, which is + // too late for that check. + const auto datadir = Options::root()["datadir"].withDefault(DEFAULT_DIR); + [[maybe_unused]] const auto optionfile = + Options::root()["optionfile"].withDefault(args.opt_file); + const auto settingsfile = + Options::root()["settingsfile"].withDefault(args.set_file); + + setRunStartInfo(Options::root()); + + if (MYPE == 0) { + writeSettingsFile(Options::root(), datadir, settingsfile); + } - // Create the mesh - bout::globals::mesh = Mesh::create(); - // Load from sources. Required for Field initialisation - bout::globals::mesh->load(); + bout::globals::mpi = new MpiWrapper(); - // time_report options are used in BoutFinalise, i.e. after we - // check for unused options - Options::root()["time_report"].setConditionallyUsed(); + // Create the mesh + bout::globals::mesh = Mesh::create(); + // Load from sources. Required for Field initialisation + bout::globals::mesh->load(); - } catch (const BoutException& e) { - output_error.write(_("Error encountered during initialisation: {:s}\n"), e.what()); - throw; - } + // time_report options are used in BoutFinalise, i.e. after we + // check for unused options + Options::root()["time_report"].setConditionallyUsed(); return 0; } @@ -329,8 +343,8 @@ template // Now we can print all the options used in constructing our // type. Note that this does require all the options are used in the // constructor, and not in a `init` method or similar - std::cout << fmt::format("Input options for {} '{}':\n\n", Factory::type_name, type); - std::cout << fmt::format("{:id}\n", help_options); + fmt::println("Input options for {:s} '{}':\n", Factory::type_name, type); + fmt::println("{:id}", help_options); std::exit(EXIT_SUCCESS); } @@ -350,7 +364,7 @@ void handleFactoryHelp(const std::string& current_arg, int i, int argc, char** a if (current_arg == help_arg) { if (i + 1 >= argc) { - throw BoutException(_("Usage is {} {} \n"), argv[0], help_arg); + throw BoutException(_f("Usage is {} {} \n"), argv[0], help_arg); } printTypeOptions(argv[i + 1]); } @@ -364,9 +378,10 @@ auto parseCommandLineArgs(int argc, char** argv) -> CommandLineArgs { if (current_arg == "-h" || current_arg == "--help") { // Print help message -- note this will be displayed once per processor as we've not // started MPI yet. - output.write(_("Usage: {:s} [-d ] [-f ] [restart " - "[append]] [VAR=VALUE]\n"), - argv[0]); + output.write( + _f("Usage: {:s} [-d ] [-f ] [restart " + "[append]] [VAR=VALUE]\n"), + argv[0]); output.write( _("\n" " -d \t\tLook in for input/output files\n" @@ -379,33 +394,33 @@ auto parseCommandLineArgs(int argc, char** argv) -> CommandLineArgs { output.write(_(" -c, --color\t\t\tColor output using bout-log-color\n")); #endif output.write( - _(" --print-config\t\tPrint the compile-time configuration\n" - " --list-solvers\t\tList the available time solvers\n" - " --help-solver \tPrint help for the given time solver\n" - " --list-laplacians\t\tList the available Laplacian inversion solvers\n" - " --help-laplacian \tPrint help for the given Laplacian " - "inversion solver\n" - " --list-laplacexz\t\tList the available LaplaceXZ inversion solvers\n" - " --help-laplacexz \tPrint help for the given LaplaceXZ " - "inversion solver\n" - " --list-invertpars\t\tList the available InvertPar solvers\n" - " --help-invertpar \tPrint help for the given InvertPar solver\n" - " --list-rkschemes\t\tList the available Runge-Kutta schemes\n" - " --help-rkscheme \tPrint help for the given Runge-Kutta scheme\n" - " --list-meshes\t\t\tList the available Meshes\n" - " --help-mesh \t\tPrint help for the given Mesh\n" - " --list-xzinterpolations\tList the available XZInterpolations\n" - " --help-xzinterpolation \tPrint help for the given " - "XZInterpolation\n" - " --list-zinterpolations\tList the available ZInterpolations\n" - " --help-zinterpolation \tPrint help for the given " - "ZInterpolation\n" - " -h, --help\t\t\tThis message\n" - " restart [append]\t\tRestart the simulation. If append is specified, " - "append to the existing output files, otherwise overwrite them\n" - " VAR=VALUE\t\t\tSpecify a VALUE for input parameter VAR\n" - "\nFor all possible input parameters, see the user manual and/or the " - "physics model source (e.g. {:s}.cxx)\n"), + _f(" --print-config\t\tPrint the compile-time configuration\n" + " --list-solvers\t\tList the available time solvers\n" + " --help-solver \tPrint help for the given time solver\n" + " --list-laplacians\t\tList the available Laplacian inversion solvers\n" + " --help-laplacian \tPrint help for the given Laplacian " + "inversion solver\n" + " --list-laplacexz\t\tList the available LaplaceXZ inversion solvers\n" + " --help-laplacexz \tPrint help for the given LaplaceXZ " + "inversion solver\n" + " --list-invertpars\t\tList the available InvertPar solvers\n" + " --help-invertpar \tPrint help for the given InvertPar solver\n" + " --list-rkschemes\t\tList the available Runge-Kutta schemes\n" + " --help-rkscheme \tPrint help for the given Runge-Kutta scheme\n" + " --list-meshes\t\t\tList the available Meshes\n" + " --help-mesh \t\tPrint help for the given Mesh\n" + " --list-xzinterpolations\tList the available XZInterpolations\n" + " --help-xzinterpolation \tPrint help for the given " + "XZInterpolation\n" + " --list-zinterpolations\tList the available ZInterpolations\n" + " --help-zinterpolation \tPrint help for the given " + "ZInterpolation\n" + " -h, --help\t\t\tThis message\n" + " restart [append]\t\tRestart the simulation. If append is specified, " + "append to the existing output files, otherwise overwrite them\n" + " VAR=VALUE\t\t\tSpecify a VALUE for input parameter VAR\n" + "\nFor all possible input parameters, see the user manual and/or the " + "physics model source (e.g. {:s}.cxx)\n"), argv[0]); std::exit(EXIT_SUCCESS); @@ -436,7 +451,7 @@ auto parseCommandLineArgs(int argc, char** argv) -> CommandLineArgs { if (string(argv[i]) == "-d") { // Set data directory if (i + 1 >= argc) { - throw BoutException(_("Usage is {:s} -d \n"), argv[0]); + throw BoutException(_f("Usage is {:s} -d \n"), argv[0]); } args.data_dir = argv[++i]; @@ -445,7 +460,7 @@ auto parseCommandLineArgs(int argc, char** argv) -> CommandLineArgs { } else if (string(argv[i]) == "-f") { // Set options file if (i + 1 >= argc) { - throw BoutException(_("Usage is {:s} -f \n"), argv[0]); + throw BoutException(_f("Usage is {:s} -f \n"), argv[0]); } args.opt_file = argv[++i]; @@ -454,7 +469,7 @@ auto parseCommandLineArgs(int argc, char** argv) -> CommandLineArgs { } else if (string(argv[i]) == "-o") { // Set options file if (i + 1 >= argc) { - throw BoutException(_("Usage is {:s} -o \n"), argv[0]); + throw BoutException(_f("Usage is {:s} -o \n"), argv[0]); } args.set_file = argv[++i]; @@ -463,7 +478,7 @@ auto parseCommandLineArgs(int argc, char** argv) -> CommandLineArgs { } else if ((string(argv[i]) == "-l") || (string(argv[i]) == "--log")) { // Set log file if (i + 1 >= argc) { - throw BoutException(_("Usage is {:s} -l \n"), argv[0]); + throw BoutException(_f("Usage is {:s} -l \n"), argv[0]); } args.log_file = argv[++i]; @@ -501,10 +516,10 @@ auto parseCommandLineArgs(int argc, char** argv) -> CommandLineArgs { void checkDataDirectoryIsAccessible(const std::string& data_dir) { if (std::filesystem::exists(data_dir)) { if (!std::filesystem::is_directory(data_dir)) { - throw BoutException(_("DataDir \"{:s}\" is not a directory\n"), data_dir); + throw BoutException(_f("DataDir \"{:s}\" is not a directory\n"), data_dir); } } else { - throw BoutException(_("DataDir \"{:s}\" does not exist or is not accessible\n"), + throw BoutException(_f("DataDir \"{:s}\" does not exist or is not accessible\n"), data_dir); } } @@ -516,7 +531,7 @@ void savePIDtoFile(const std::string& data_dir, int MYPE) { pid_file.open(filename.str(), std::ios::out | std::ios::trunc); if (not pid_file.is_open()) { - throw BoutException(_("Could not create PID file {:s}"), filename.str()); + throw BoutException(_f("Could not create PID file {:s}"), filename.str()); } pid_file << getpid() << "\n"; @@ -524,17 +539,17 @@ void savePIDtoFile(const std::string& data_dir, int MYPE) { } void printStartupHeader(int MYPE, int NPES) { - output_progress.write(_("BOUT++ version {:s}\n"), bout::version::full); - output_progress.write(_("Revision: {:s}\n"), bout::version::revision); + output_progress.write(_f("BOUT++ version {:s}\n"), bout::version::full); + output_progress.write(_f("Revision: {:s}\n"), bout::version::revision); #ifdef MD5SUM output_progress.write("MD5 checksum: {:s}\n", BUILDFLAG(MD5SUM)); #endif - output_progress.write(_("Code compiled on {:s} at {:s}\n\n"), boutcompiledate, + output_progress.write(_f("Code compiled on {:s} at {:s}\n\n"), boutcompiledate, boutcompiletime); output_info.write("B.Dudson (University of York), M.Umansky (LLNL) 2007\n"); output_info.write("Based on BOUT by Xueqiao Xu, 1999\n\n"); - output_info.write(_("Processor number: {:d} of {:d}\n\n"), MYPE, NPES); + output_info.write(_f("Processor number: {:d} of {:d}\n\n"), MYPE, NPES); output_info.write("pid: {:d}\n\n", getpid()); } @@ -544,55 +559,51 @@ void printCompileTimeOptions() { using namespace bout::build; - output_info.write(_("\tRuntime error checking {}"), is_enabled(check_level > 0)); + output_info.write(_f("\tRuntime error checking {}"), is_enabled(check_level > 0)); if (check_level > 0) { - output_info.write(_(", level {}"), check_level); + output_info.write(_f(", level {}"), check_level); } output_info.write("\n"); #ifdef PNCDF - output_info.write(_("\tParallel NetCDF support enabled\n")); + output_info.write(_f("\tParallel NetCDF support enabled\n")); #else - output_info.write(_("\tParallel NetCDF support disabled\n")); + output_info.write(_f("\tParallel NetCDF support disabled\n")); #endif - output_info.write(_("\tMetrics mode is {}\n"), use_metric_3d ? "3D" : "2D"); - output_info.write(_("\tFFT support {}\n"), is_enabled(has_fftw)); - output_info.write(_("\tNatural language support {}\n"), is_enabled(has_gettext)); - output_info.write(_("\tLAPACK support {}\n"), is_enabled(has_lapack)); + output_info.write(_f("\tMetrics mode is {}\n"), use_metric_3d ? "3D" : "2D"); + output_info.write(_f("\tFFT support {}\n"), is_enabled(has_fftw)); + output_info.write(_f("\tNatural language support {}\n"), is_enabled(has_gettext)); + output_info.write(_f("\tLAPACK support {}\n"), is_enabled(has_lapack)); // Horrible nested ternary to set this at compile time constexpr auto netcdf_flavour = has_netcdf ? (has_legacy_netcdf ? " (Legacy)" : " (NetCDF4)") : ""; - output_info.write(_("\tNetCDF support {}{}\n"), is_enabled(has_netcdf), netcdf_flavour); - output_info.write(_("\tADIOS2 support {}\n"), is_enabled(has_adios2)); - output_info.write(_("\tPETSc support {}\n"), is_enabled(has_petsc)); - output_info.write(_("\tPretty function name support {}\n"), - is_enabled(has_pretty_function)); - output_info.write(_("\tPVODE support {}\n"), is_enabled(has_pvode)); - output_info.write(_("\tScore-P support {}\n"), is_enabled(has_scorep)); - output_info.write(_("\tSLEPc support {}\n"), is_enabled(has_slepc)); - output_info.write(_("\tSUNDIALS support {}\n"), is_enabled(has_sundials)); - output_info.write(_("\tBacktrace in exceptions {}\n"), is_enabled(use_backtrace)); - output_info.write(_("\tColour in logs {}\n"), is_enabled(use_color)); - output_info.write(_("\tOpenMP parallelisation {}, using {} threads\n"), + output_info.write(_f("\tNetCDF support {}{}\n"), is_enabled(has_netcdf), + netcdf_flavour); + output_info.write(_f("\tADIOS2 support {}\n"), is_enabled(has_adios2)); + output_info.write(_f("\tPETSc support {}\n"), is_enabled(has_petsc)); + output_info.write(_f("\tPVODE support {}\n"), is_enabled(has_pvode)); + output_info.write(_f("\tScore-P support {}\n"), is_enabled(has_scorep)); + output_info.write(_f("\tSLEPc support {}\n"), is_enabled(has_slepc)); + output_info.write(_f("\tSUNDIALS support {}\n"), is_enabled(has_sundials)); + output_info.write(_f("\tBacktrace in exceptions {}\n"), is_enabled(use_backtrace)); + output_info.write(_f("\tColour in logs {}\n"), is_enabled(use_color)); + output_info.write(_f("\tOpenMP parallelisation {}, using {} threads\n"), is_enabled(use_openmp), omp_get_max_threads()); - output_info.write(_("\tExtra debug output {}\n"), is_enabled(use_output_debug)); - output_info.write(_("\tFloating-point exceptions {}\n"), is_enabled(use_sigfpe)); - output_info.write(_("\tSignal handling support {}\n"), is_enabled(use_signal)); - output_info.write(_("\tField name tracking {}\n"), is_enabled(use_track)); - output_info.write(_("\tMessage stack {}\n"), is_enabled(use_msgstack)); + output_info.write(_f("\tExtra debug output {}\n"), is_enabled(use_output_debug)); + output_info.write(_f("\tFloating-point exceptions {}\n"), is_enabled(use_sigfpe)); + output_info.write(_f("\tSignal handling support {}\n"), is_enabled(use_signal)); + output_info.write(_f("\tField name tracking {}\n"), is_enabled(use_track)); + output_info.write(_f("\tMessage stack {}\n"), is_enabled(use_msgstack)); // The stringify is needed here as BOUT_FLAGS_STRING may already contain quoted strings // which could cause problems (e.g. terminate strings). - output_info.write(_("\tCompiled with flags : {:s}\n"), STRINGIFY(BOUT_FLAGS_STRING)); + output_info.write(_f("\tCompiled with flags : {:s}\n"), STRINGIFY(BOUT_FLAGS_STRING)); } void printCommandLineArguments(const std::vector& original_argv) { - output_info.write(_("\tCommand line options for this run : ")); - for (auto& arg : original_argv) { - output_info << arg << " "; - } - output_info.write("\n"); + output_info.write("{:s}{}\n", _("\tCommand line options for this run : "), + fmt::join(original_argv, " ")); } bool setupBoutLogColor(bool color_output, int MYPE) { @@ -626,7 +637,8 @@ bool setupBoutLogColor(bool color_output, int MYPE) { } if (!success) { // Failed . Probably not important enough to stop the simulation - std::cerr << _("Could not run bout-log-color. Make sure it is in your PATH\n"); + fmt::print(stderr, "{:s}", + _("Could not run bout-log-color. Make sure it is in your PATH\n")); } return success; } @@ -646,7 +658,7 @@ void setupOutput(const std::string& data_dir, const std::string& log_file, int v /// Open an output file to echo everything to /// On processor 0 anything written to output will go to stdout and the file if (output.open("{:s}/{:s}.{:d}", data_dir, log_file, MYPE)) { - throw BoutException(_("Could not open {:s}/{:s}.{:d} for writing"), data_dir, + throw BoutException(_f("Could not open {:s}/{:s}.{:d} for writing"), data_dir, log_file, MYPE); } } @@ -696,7 +708,6 @@ void addBuildFlagsToOptions(Options& options) { options["has_umpire"].force(bout::build::has_umpire); options["has_caliper"].force(bout::build::has_caliper); options["has_raja"].force(bout::build::has_raja); - options["has_pretty_function"].force(bout::build::has_pretty_function); options["has_pvode"].force(bout::build::has_pvode); options["has_scorep"].force(bout::build::has_scorep); options["has_slepc"].force(bout::build::has_slepc); @@ -740,7 +751,7 @@ int BoutFinalise(bool write_settings) { writeSettingsFile(options, data_dir, set_file); } } catch (const BoutException& e) { - output_error << _("Error whilst writing settings") << e.what() << endl; + output_error.write("{} {}\n", _("Error whilst writing settings"), e.what()); } } @@ -897,7 +908,7 @@ int BoutMonitor::call(Solver* solver, BoutReal t, [[maybe_unused]] int iter, int BoutReal t_remain = mpi_start_time + wall_limit - bout::globals::mpi->MPI_Wtime(); if (t_remain < run_data.wtime * 2) { // Less than 2 time-steps left - output_warn.write(_("Only {:e} seconds ({:.2f} steps) left. Quitting\n"), t_remain, + output_warn.write(_f("Only {:e} seconds ({:.2f} steps) left. Quitting\n"), t_remain, t_remain / run_data.wtime); user_requested_exit = true; } else { diff --git a/src/field/field.cxx b/src/field/field.cxx index 797df6c405..e9b01f0bcb 100644 --- a/src/field/field.cxx +++ b/src/field/field.cxx @@ -27,7 +27,6 @@ #include #include #include -#include #include #include diff --git a/src/field/field2d.cxx b/src/field/field2d.cxx index c8b9ebb689..a269306c3a 100644 --- a/src/field/field2d.cxx +++ b/src/field/field2d.cxx @@ -4,7 +4,7 @@ * Class for 2D X-Y profiles * ************************************************************************** - * Copyright 2010 - 2025 BOUT++ developers + * Copyright 2010 - 2026 BOUT++ developers * * Contact: Ben Dudson, dudson2@llnl.gov * @@ -25,53 +25,47 @@ * **************************************************************************/ -#include "bout/build_config.hxx" - -#include -#include - -#include // for mesh - -#include - -#include +#include "bout/bout_types.hxx" +#include "bout/build_defines.hxx" +#include "bout/unused.hxx" +#include +#include #include #include - +#include #include +#include +#include // for mesh #include -#include - #include -#include -#include +#include +#include +#include -Field2D::Field2D(Mesh* localmesh, CELL_LOC location_in, DirectionTypes directions_in) +Field2D::Field2D(Mesh* localmesh, CELL_LOC location_in, DirectionTypes directions_in, + std::optional UNUSED(regionID)) : Field(localmesh, location_in, directions_in) { - - if (fieldmesh) { + if (fieldmesh != nullptr) { + // Note: Even if fieldmesh is not null, LocalNx and LocalNy may + // not be initialised. nx = fieldmesh->LocalNx; ny = fieldmesh->LocalNy; } - #if BOUT_USE_TRACK name = ""; #endif } Field2D::Field2D(const Field2D& f) : Field(f), data(f.data) { - TRACE("Field2D(Field2D&)"); - -#if BOUT_USE_TRACK - name = f.name; -#endif - - if (fieldmesh) { + if (fieldmesh != nullptr) { nx = fieldmesh->LocalNx; ny = fieldmesh->LocalNy; } +#if BOUT_USE_TRACK + name = f.name; +#endif } Field2D::Field2D(BoutReal val, Mesh* localmesh) : Field2D(localmesh) { *this = val; } @@ -94,13 +88,16 @@ Field2D::~Field2D() { delete deriv; } Field2D& Field2D::allocate() { if (data.empty()) { - if (!fieldmesh) { + if (fieldmesh == nullptr) { // fieldmesh was not initialized when this field was initialized, so use - // the global mesh and set some members to default values + // the global mesh fieldmesh = bout::globals::mesh; - nx = fieldmesh->LocalNx; - ny = fieldmesh->LocalNy; } + // Get size from the mesh. + nx = fieldmesh->LocalNx; + ny = fieldmesh->LocalNy; + ASSERT1(nx > 0); + ASSERT1(ny > 0); data.reallocate(nx * ny); #if CHECK > 2 invalidateGuards(*this); @@ -145,8 +142,6 @@ Field2D& Field2D::operator=(const Field2D& rhs) { return (*this); // skip this assignment } - TRACE("Field2D: Assignment from Field2D"); - Field::operator=(rhs); // Copy the data and data sizes @@ -165,8 +160,6 @@ Field2D& Field2D::operator=(Field2D&& rhs) noexcept { return (*this); // skip this assignment } - TRACE("Field2D: Move assignment from Field2D"); - // Move the data and data sizes nx = rhs.nx; ny = rhs.ny; @@ -185,7 +178,6 @@ Field2D& Field2D::operator=(const BoutReal rhs) { name = ""; #endif - TRACE("Field2D = BoutReal"); allocate(); BOUT_FOR(i, getRegion("RGN_ALL")) { (*this)[i] = rhs; } @@ -196,7 +188,6 @@ Field2D& Field2D::operator=(const BoutReal rhs) { ///////////////////// BOUNDARY CONDITIONS ////////////////// void Field2D::applyBoundary(bool init) { - TRACE("Field2D::applyBoundary()"); #if CHECK > 0 if (init) { @@ -218,7 +209,6 @@ void Field2D::applyBoundary(bool init) { } void Field2D::applyBoundary(BoutReal time) { - TRACE("Field2D::applyBoundary(time)"); #if CHECK > 0 if (not isBoundarySet()) { @@ -234,7 +224,6 @@ void Field2D::applyBoundary(BoutReal time) { } void Field2D::applyBoundary(const std::string& condition) { - TRACE("Field2D::applyBoundary(condition)"); checkData(*this); @@ -268,7 +257,7 @@ void Field2D::applyBoundary(const std::string& condition) { } void Field2D::applyBoundary(const std::string& region, const std::string& condition) { - TRACE("Field2D::applyBoundary(string, string)"); + checkData(*this); /// Get the boundary factory (singleton) @@ -310,7 +299,6 @@ void Field2D::applyBoundary(const std::string& region, const std::string& condit } void Field2D::applyTDerivBoundary() { - TRACE("Field2D::applyTDerivBoundary()"); checkData(*this); ASSERT1(deriv != nullptr); @@ -322,7 +310,6 @@ void Field2D::applyTDerivBoundary() { } void Field2D::setBoundaryTo(const Field2D& f2d) { - TRACE("Field2D::setBoundary(const Field2D&)"); checkData(f2d); diff --git a/src/field/field3d.cxx b/src/field/field3d.cxx index 4c5ffa9748..7cbf135e25 100644 --- a/src/field/field3d.cxx +++ b/src/field/field3d.cxx @@ -4,10 +4,10 @@ * Class for 3D X-Y-Z scalar fields * ************************************************************************** - * Copyright 2010 - 2025 BOUT++ developers + * Copyright 2010 - 2026 BOUT++ developers * * Contact: Ben Dudson, dudson2@llnl.gov - * + * * This file is part of BOUT++. * * BOUT++ is free software: you can redistribute it and/or modify @@ -25,12 +25,20 @@ * **************************************************************************/ -#include "bout/build_config.hxx" +#include "bout/array.hxx" +#include "bout/bout_types.hxx" +#include "bout/build_defines.hxx" +#include "bout/field2d.hxx" #include #include #include +#include +#include +#include +#include +#include #include "bout/parallel_boundary_op.hxx" #include "bout/parallel_boundary_region.hxx" @@ -43,18 +51,20 @@ #include #include #include -#include #include #include +#include "fmt/format.h" + /// Constructor -Field3D::Field3D(Mesh* localmesh, CELL_LOC location_in, DirectionTypes directions_in) - : Field(localmesh, location_in, directions_in) { +Field3D::Field3D(Mesh* localmesh, CELL_LOC location_in, DirectionTypes directions_in, + std::optional regionID) + : Field(localmesh, location_in, directions_in), regionID{regionID} { #if BOUT_USE_TRACK name = ""; #endif - if (fieldmesh) { + if (fieldmesh != nullptr) { nx = fieldmesh->LocalNx; ny = fieldmesh->LocalNy; nz = fieldmesh->LocalNz; @@ -66,42 +76,27 @@ Field3D::Field3D(Mesh* localmesh, CELL_LOC location_in, DirectionTypes direction Field3D::Field3D(const Field3D& f) : Field(f), data(f.data), yup_fields(f.yup_fields), ydown_fields(f.ydown_fields), regionID(f.regionID) { - - TRACE("Field3D(Field3D&)"); - - if (fieldmesh) { + if (fieldmesh != nullptr) { nx = fieldmesh->LocalNx; ny = fieldmesh->LocalNy; nz = fieldmesh->LocalNz; } } -Field3D::Field3D(const Field2D& f) : Field(f) { - - TRACE("Field3D: Copy constructor from Field2D"); - - nx = fieldmesh->LocalNx; - ny = fieldmesh->LocalNy; - nz = fieldmesh->LocalNz; +Field3D::Field3D(const Field2D& f) + : Field(f), nx(fieldmesh->LocalNx), ny(fieldmesh->LocalNy), nz(fieldmesh->LocalNz) { *this = f; } Field3D::Field3D(const BoutReal val, Mesh* localmesh) : Field3D(localmesh) { - - TRACE("Field3D: Copy constructor from value"); - *this = val; } Field3D::Field3D(Array data_in, Mesh* localmesh, CELL_LOC datalocation, DirectionTypes directions_in) - : Field(localmesh, datalocation, directions_in), data(std::move(data_in)) { - TRACE("Field3D: Copy constructor from Array and Mesh"); - - nx = fieldmesh->LocalNx; - ny = fieldmesh->LocalNy; - nz = fieldmesh->LocalNz; + : Field(localmesh, datalocation, directions_in), nx(fieldmesh->LocalNx), + ny(fieldmesh->LocalNy), nz(fieldmesh->LocalNz), data(std::move(data_in)) { ASSERT1(data.size() == nx * ny * nz); } @@ -110,14 +105,17 @@ Field3D::~Field3D() { delete deriv; } Field3D& Field3D::allocate() { if (data.empty()) { - if (!fieldmesh) { + if (fieldmesh == nullptr) { // fieldmesh was not initialized when this field was initialized, so use // the global mesh and set some members to default values fieldmesh = bout::globals::mesh; - nx = fieldmesh->LocalNx; - ny = fieldmesh->LocalNy; - nz = fieldmesh->LocalNz; } + nx = fieldmesh->LocalNx; + ny = fieldmesh->LocalNy; + nz = fieldmesh->LocalNz; + ASSERT1(nx > 0); + ASSERT1(ny > 0); + ASSERT1(nz > 0); data.reallocate(nx * ny * nz); #if CHECK > 2 invalidateGuards(*this); @@ -137,7 +135,6 @@ Field3D* Field3D::timeDeriv() { } void Field3D::splitParallelSlices() { - TRACE("Field3D::splitParallelSlices"); if (hasParallelSlices()) { return; @@ -149,10 +146,10 @@ void Field3D::splitParallelSlices() { yup_fields.emplace_back(fieldmesh); ydown_fields.emplace_back(fieldmesh); } + resetRegionParallel(); } void Field3D::clearParallelSlices() { - TRACE("Field3D::clearParallelSlices"); if (!hasParallelSlices()) { return; @@ -234,7 +231,7 @@ const Region& Field3D::getRegion2D(const std::string& region_name) const } /*************************************************************** - * OPERATORS + * OPERATORS ***************************************************************/ /////////////////// ASSIGNMENT //////////////////// @@ -245,7 +242,7 @@ Field3D& Field3D::operator=(const Field3D& rhs) { return (*this); // skip this assignment } - TRACE("Field3D: Assignment from Field3D"); + track(rhs, "operator="); // Copy base slice Field::operator=(rhs); @@ -266,7 +263,7 @@ Field3D& Field3D::operator=(const Field3D& rhs) { } Field3D& Field3D::operator=(Field3D&& rhs) { - TRACE("Field3D: Assignment from Field3D"); + track(rhs, "operator="); // Move parallel slices or delete existing ones. yup_fields = std::move(rhs.yup_fields); @@ -287,7 +284,7 @@ Field3D& Field3D::operator=(Field3D&& rhs) { } Field3D& Field3D::operator=(const Field2D& rhs) { - TRACE("Field3D = Field2D"); + track(rhs, "operator="); /// Check that the data is allocated ASSERT1(rhs.isAllocated()); @@ -314,7 +311,6 @@ Field3D& Field3D::operator=(const Field2D& rhs) { } void Field3D::operator=(const FieldPerp& rhs) { - TRACE("Field3D = FieldPerp"); ASSERT1_FIELDS_COMPATIBLE(*this, rhs); /// Check that the data is allocated @@ -333,7 +329,7 @@ void Field3D::operator=(const FieldPerp& rhs) { } Field3D& Field3D::operator=(const BoutReal val) { - TRACE("Field3D = BoutReal"); + track(val, "operator="); // Delete existing parallel slices. We don't copy parallel slices, so any // that currently exist will be incorrect. @@ -347,15 +343,14 @@ Field3D& Field3D::operator=(const BoutReal val) { return *this; } -Field3D& Field3D::calcParallelSlices() { +void Field3D::calcParallelSlices() { + ASSERT1(areCalcParallelSlicesAllowed()); getCoordinates()->getParallelTransform().calcParallelSlices(*this); - return *this; } ///////////////////// BOUNDARY CONDITIONS ////////////////// void Field3D::applyBoundary(bool init) { - TRACE("Field3D::applyBoundary()"); #if CHECK > 0 if (init) { @@ -378,7 +373,6 @@ void Field3D::applyBoundary(bool init) { } void Field3D::applyBoundary(BoutReal t) { - TRACE("Field3D::applyBoundary()"); #if CHECK > 0 if (not isBoundarySet()) { @@ -395,7 +389,6 @@ void Field3D::applyBoundary(BoutReal t) { } void Field3D::applyBoundary(const std::string& condition) { - TRACE("Field3D::applyBoundary(condition)"); checkData(*this); @@ -413,7 +406,7 @@ void Field3D::applyBoundary(const std::string& condition) { } void Field3D::applyBoundary(const std::string& region, const std::string& condition) { - TRACE("Field3D::applyBoundary(string, string)"); + checkData(*this); /// Get the boundary factory (singleton) @@ -439,7 +432,6 @@ void Field3D::applyBoundary(const std::string& region, const std::string& condit } void Field3D::applyTDerivBoundary() { - TRACE("Field3D::applyTDerivBoundary()"); checkData(*this); ASSERT1(deriv != nullptr); @@ -451,7 +443,6 @@ void Field3D::applyTDerivBoundary() { } void Field3D::setBoundaryTo(const Field3D& f3d) { - TRACE("Field3D::setBoundary(const Field3D&)"); checkData(f3d); @@ -475,8 +466,6 @@ void Field3D::setBoundaryTo(const Field3D& f3d) { void Field3D::applyParallelBoundary() { - TRACE("Field3D::applyParallelBoundary()"); - checkData(*this); if (isFci()) { ASSERT1(hasParallelSlices()); @@ -493,8 +482,6 @@ void Field3D::applyParallelBoundary() { void Field3D::applyParallelBoundary(BoutReal t) { - TRACE("Field3D::applyParallelBoundary(t)"); - checkData(*this); if (isFci()) { ASSERT1(hasParallelSlices()); @@ -511,8 +498,6 @@ void Field3D::applyParallelBoundary(BoutReal t) { void Field3D::applyParallelBoundary(const std::string& condition) { - TRACE("Field3D::applyParallelBoundary(condition)"); - checkData(*this); if (isFci()) { ASSERT1(hasParallelSlices()); @@ -535,8 +520,6 @@ void Field3D::applyParallelBoundary(const std::string& condition) { void Field3D::applyParallelBoundary(const std::string& region, const std::string& condition) { - TRACE("Field3D::applyParallelBoundary(region, condition)"); - checkData(*this); if (isFci()) { ASSERT1(hasParallelSlices()); @@ -562,8 +545,6 @@ void Field3D::applyParallelBoundary(const std::string& region, void Field3D::applyParallelBoundary(const std::string& region, const std::string& condition, Field3D* f) { - TRACE("Field3D::applyParallelBoundary(region, condition, f)"); - checkData(*this); if (isFci()) { ASSERT1(hasParallelSlices()); @@ -599,7 +580,7 @@ Field3D operator-(const Field3D& f) { return -1.0 * f; } //////////////// NON-MEMBER FUNCTIONS ////////////////// Field3D pow(const Field3D& lhs, const Field2D& rhs, const std::string& rgn) { - TRACE("pow(Field3D, Field2D)"); + // Check if the inputs are allocated checkData(lhs); checkData(rhs); @@ -615,7 +596,6 @@ Field3D pow(const Field3D& lhs, const Field2D& rhs, const std::string& rgn) { } FieldPerp pow(const Field3D& lhs, const FieldPerp& rhs, const std::string& rgn) { - TRACE("pow(Field3D, FieldPerp)"); checkData(lhs); checkData(rhs); @@ -635,8 +615,8 @@ FieldPerp pow(const Field3D& lhs, const FieldPerp& rhs, const std::string& rgn) // Friend functions Field3D filter(const Field3D& var, int N0, const std::string& rgn) { - TRACE("filter(Field3D, int)"); + bout::fft::assertZSerial(*var.getMesh(), "`filter`"); checkData(var); int ncz = var.getNz(); @@ -681,8 +661,8 @@ Field3D filter(const Field3D& var, int N0, const std::string& rgn) { // Fourier filter in z with zmin Field3D lowPass(const Field3D& var, int zmax, bool keep_zonal, const std::string& rgn) { - TRACE("lowPass(Field3D, {}, {})", zmax, keep_zonal); + bout::fft::assertZSerial(*var.getMesh(), "`lowPass`"); checkData(var); int ncz = var.getNz(); @@ -727,11 +707,11 @@ Field3D lowPass(const Field3D& var, int zmax, bool keep_zonal, const std::string return result; } -/* +/* * Use FFT to shift by an angle in the Z direction */ void shiftZ(Field3D& var, int jx, int jy, double zangle) { - TRACE("shiftZ"); + bout::fft::assertZSerial(*var.getMesh(), "`shiftZ`"); checkData(var); var.allocate(); // Ensure that var is unique Mesh* localmesh = var.getMesh(); @@ -800,7 +780,6 @@ void checkData(const Field3D& f, const std::string& region) { #endif Field2D DC(const Field3D& f, const std::string& rgn) { - TRACE("DC(Field3D)"); checkData(f); @@ -864,3 +843,72 @@ Field3D::getValidRegionWithDefault(const std::string& region_name) const { void Field3D::setRegion(const std::string& region_name) { regionID = fieldmesh->getRegionID(region_name); } + +void Field3D::resetRegionParallel() { + if (isFci()) { + for (int i = 0; i < fieldmesh->ystart; ++i) { + yup_fields[i].setRegion(fmt::format("RGN_YPAR_{:+d}", i + 1)); + ydown_fields[i].setRegion(fmt::format("RGN_YPAR_{:+d}", -i - 1)); + } + } +} + +Field3D& Field3D::enableTracking(const std::string& name, + std::weak_ptr _tracking) { + tracking = std::move(_tracking); + tracking_state = 1; + selfname = name; + return *this; +} + +template +void Field3D::_track(const T& change, std::string operation) { + if (tracking_state == 0) { + return; + } + auto locked = tracking.lock(); + if (locked == nullptr) { + return; + } + const std::string outname{fmt::format("track_{:s}_{:d}", selfname, tracking_state++)}; + + locked->set(outname, change, "tracking"); + + const std::string trace = cpptrace::generate_trace().to_string(); + + // Workaround for bug in gcc9.4 +#if BOUT_USE_TRACK + const std::string changename = change.name; +#endif + (*locked)[outname].setAttributes({ + {"operation", operation}, +#if BOUT_USE_TRACK + {"rhs.name", changename}, +#endif + {"trace", trace}, + }); +} + +template void +Field3D::_track>(const Field3D&, + std::string); +template void Field3D::_track(const Field2D&, std::string); +template void Field3D::_track<>(const FieldPerp&, std::string); + +void Field3D::_track(const BoutReal& change, std::string operation) { + if (tracking_state == 0) { + return; + } + auto locked = tracking.lock(); + if (locked == nullptr) { + return; + } + const std::string trace = cpptrace::generate_trace().to_string(); + const std::string outname{fmt::format("track_{:s}_{:d}", selfname, tracking_state++)}; + locked->set(outname, change, "tracking"); + (*locked)[outname].setAttributes({ + {"operation", operation}, + {"rhs.name", "BoutReal"}, + {"trace", trace}, + }); +} diff --git a/src/field/field_data.cxx b/src/field/field_data.cxx index 1c3f646bb1..8f6a808482 100644 --- a/src/field/field_data.cxx +++ b/src/field/field_data.cxx @@ -15,7 +15,6 @@ namespace bout { /// Throws if checks are enabled and trying to use a staggered /// location on a non-staggered mesh CELL_LOC normaliseLocation(CELL_LOC location, Mesh* mesh) { - AUTO_TRACE(); // CELL_DEFAULT always means CELL_CENTRE if (location == CELL_DEFAULT) { @@ -211,7 +210,6 @@ Mesh* FieldData::getMesh() const { } FieldData& FieldData::setLocation(CELL_LOC new_location) { - AUTO_TRACE(); location = bout::normaliseLocation(new_location, getMesh()); @@ -223,10 +221,7 @@ FieldData& FieldData::setLocation(CELL_LOC new_location) { return *this; } -CELL_LOC FieldData::getLocation() const { - AUTO_TRACE(); - return location; -} +CELL_LOC FieldData::getLocation() const { return location; } Coordinates* FieldData::getCoordinates() const { auto fieldCoordinates_shared = fieldCoordinates.lock(); diff --git a/src/field/field_factory.cxx b/src/field/field_factory.cxx index b8f5437b24..793ea9f278 100644 --- a/src/field/field_factory.cxx +++ b/src/field/field_factory.cxx @@ -227,6 +227,9 @@ FieldFactory::FieldFactory(Mesh* localmesh, Options* opt) // Where switch function addGenerator("where", std::make_shared(nullptr, nullptr, nullptr)); + // Periodic in the Y direction? + addGenerator("is_periodic_y", std::make_shared()); + // Variables from the grid file read_grid_variables(*this, *fieldmesh, nonconst_options); } @@ -238,7 +241,6 @@ Field2D FieldFactory::create2D(const std::string& value, const Options* opt, Field2D FieldFactory::create2D(FieldGeneratorPtr gen, Mesh* localmesh, CELL_LOC loc, BoutReal t) const { - AUTO_TRACE(); if (localmesh == nullptr) { if (fieldmesh == nullptr) { @@ -270,7 +272,6 @@ Field3D FieldFactory::create3D(const std::string& value, const Options* opt, Field3D FieldFactory::create3D(FieldGeneratorPtr gen, Mesh* localmesh, CELL_LOC loc, BoutReal t) const { - AUTO_TRACE(); if (localmesh == nullptr) { if (fieldmesh == nullptr) { @@ -321,7 +322,6 @@ FieldPerp FieldFactory::createPerp(const std::string& value, const Options* opt, FieldPerp FieldFactory::createPerp(FieldGeneratorPtr gen, Mesh* localmesh, CELL_LOC loc, BoutReal t) const { - AUTO_TRACE(); if (localmesh == nullptr) { if (fieldmesh == nullptr) { diff --git a/src/field/fieldgenerators.cxx b/src/field/fieldgenerators.cxx index 1f21f5c262..7e728664f8 100644 --- a/src/field/fieldgenerators.cxx +++ b/src/field/fieldgenerators.cxx @@ -1,6 +1,8 @@ #include "fieldgenerators.hxx" +#include + #include #include diff --git a/src/field/fieldgenerators.hxx b/src/field/fieldgenerators.hxx index 9699b38210..6866808df9 100644 --- a/src/field/fieldgenerators.hxx +++ b/src/field/fieldgenerators.hxx @@ -7,15 +7,20 @@ #ifndef BOUT_FIELDGENERATORS_H #define BOUT_FIELDGENERATORS_H +#include #include #include #include +#include #include #include +#include #include +#include #include #include +#include ////////////////////////////////////////////////////////// // Generators from values @@ -43,7 +48,7 @@ template class FieldGenOneArg : public FieldGenerator { ///< Template for single-argument function public: FieldGenOneArg(FieldGeneratorPtr g, const std::string& name = "function") - : gen(g), name(name) {} + : gen(std::move(g)), name(name) {} FieldGeneratorPtr clone(const std::list args) override { if (args.size() != 1) { throw ParseException("Incorrect number of arguments to {:s}. Expecting 1, got {:d}", @@ -70,7 +75,7 @@ class FieldGenTwoArg : public FieldGenerator { ///< Template for two-argument fu public: FieldGenTwoArg(FieldGeneratorPtr a, FieldGeneratorPtr b, const std::string& name = "function") - : A(a), B(b), name(name) {} + : A(std::move(a)), B(std::move(b)), name(name) {} FieldGeneratorPtr clone(const std::list args) override { if (args.size() != 2) { throw ParseException("Incorrect number of arguments to {:s}. Expecting 2, got {:d}", @@ -93,11 +98,13 @@ private: /// Arc (Inverse) tangent. Either one or two argument versions class FieldATan : public FieldGenerator { public: - FieldATan(FieldGeneratorPtr a, FieldGeneratorPtr b = nullptr) : A(a), B(b) {} + FieldATan(FieldGeneratorPtr a, FieldGeneratorPtr b = nullptr) + : A(std::move(a)), B(std::move(b)) {} FieldGeneratorPtr clone(const std::list args) override { if (args.size() == 1) { return std::make_shared(args.front()); - } else if (args.size() == 2) { + } + if (args.size() == 2) { return std::make_shared(args.front(), args.back()); } throw ParseException( @@ -148,7 +155,7 @@ public: FieldMin() = default; FieldMin(const std::list args) : input(args) {} FieldGeneratorPtr clone(const std::list args) override { - if (args.size() == 0) { + if (args.empty()) { throw ParseException("min function must have some inputs"); } return std::make_shared(args); @@ -157,10 +164,7 @@ public: auto it = input.begin(); BoutReal result = (*it)->generate(pos); for (; it != input.end(); it++) { - BoutReal val = (*it)->generate(pos); - if (val < result) { - result = val; - } + result = std::min(result, (*it)->generate(pos)); } return result; } @@ -175,7 +179,7 @@ public: FieldMax() = default; FieldMax(const std::list args) : input(args) {} FieldGeneratorPtr clone(const std::list args) override { - if (args.size() == 0) { + if (args.empty()) { throw ParseException("max function must have some inputs"); } return std::make_shared(args); @@ -184,10 +188,7 @@ public: auto it = input.begin(); BoutReal result = (*it)->generate(pos); for (; it != input.end(); it++) { - BoutReal val = (*it)->generate(pos); - if (val > result) { - result = val; - } + result = std::max(result, (*it)->generate(pos)); } return result; } @@ -206,7 +207,7 @@ class FieldClamp : public FieldGenerator { public: FieldClamp() = default; FieldClamp(FieldGeneratorPtr value, FieldGeneratorPtr low, FieldGeneratorPtr high) - : value(value), low(low), high(high) {} + : value(std::move(value)), low(std::move(low)), high(std::move(high)) {} FieldGeneratorPtr clone(const std::list args) override { if (args.size() != 3) { throw ParseException( @@ -242,7 +243,7 @@ private: /// Generator to round to the nearest integer class FieldRound : public FieldGenerator { public: - FieldRound(FieldGeneratorPtr g) : gen(g) {} + FieldRound(FieldGeneratorPtr g) : gen(std::move(g)) {} FieldGeneratorPtr clone(const std::list args) override { if (args.size() != 1) { @@ -394,8 +395,9 @@ public: } FieldGeneratorPtr clone(const std::list args) override { - if (args.size() != 0) { - throw ParseException("Variable '{}' takes no arguments but got {:d}", args.size()); + if (!args.empty()) { + throw ParseException("Variable '{}' takes no arguments but got {:d}", + variable->name, args.size()); } return std::make_shared>(variable); } @@ -403,4 +405,29 @@ public: std::string str() const override { return variable->name; } }; +/// Function that evaluates to 1 when Y is periodic (i.e. in the core), 0 otherwise +/// Note: Assumes symmetricGlobalX +class FieldPeriodicY : public FieldGenerator { +public: + FieldPeriodicY() = default; + FieldGeneratorPtr clone(const std::list UNUSED(args)) override { + return std::make_shared(); + } + BoutReal generate(const bout::generator::Context& ctx) override { + const Mesh* mesh = ctx.getMesh(); + const BoutReal local_inner_boundary = + 0.5 * (mesh->GlobalX(mesh->xstart - 1) + mesh->GlobalX(mesh->xstart)); + const BoutReal local_outer_boundary = + 0.5 * (mesh->GlobalX(mesh->xend + 1) + mesh->GlobalX(mesh->xend)); + const int local_index = mesh->xstart + + int(((ctx.x() - local_inner_boundary) + / (local_outer_boundary - local_inner_boundary)) + * (mesh->xend - mesh->xstart + 1)); + if (mesh->periodicY(local_index)) { + return 1.0; + } + return 0.0; + } +}; + #endif // BOUT_FIELDGENERATORS_H diff --git a/src/field/fieldperp.cxx b/src/field/fieldperp.cxx index ca9bdc0397..d5f0d93706 100644 --- a/src/field/fieldperp.cxx +++ b/src/field/fieldperp.cxx @@ -5,7 +5,7 @@ * Copyright 2010 - 2025 BOUT++ developers * * Contact: Ben Dudson, dudson2@llnl.gov - * + * * This file is part of BOUT++. * * BOUT++ is free software: you can redistribute it and/or modify @@ -23,19 +23,21 @@ * **************************************************************************/ +#include "bout/unused.hxx" #include #include #include +#include +#include #include #include #include -#include #include FieldPerp::FieldPerp(Mesh* localmesh, CELL_LOC location_in, int yindex_in, - DirectionTypes directions) + DirectionTypes directions, std::optional UNUSED(regionID)) : Field(localmesh, location_in, directions), yindex(yindex_in) { if (fieldmesh) { nx = fieldmesh->LocalNx; @@ -51,7 +53,6 @@ FieldPerp::FieldPerp(Array data_in, Mesh* localmesh, CELL_LOC location int yindex_in, DirectionTypes directions) : Field(localmesh, location_in, directions), yindex(yindex_in), nx(fieldmesh->LocalNx), nz(fieldmesh->LocalNz), data(std::move(data_in)) { - TRACE("FieldPerp: Copy constructor from Array and Mesh"); ASSERT1(data.size() == nx * nz); } @@ -77,7 +78,7 @@ FieldPerp& FieldPerp::allocate() { } /*************************************************************** - * ASSIGNMENT + * ASSIGNMENT ***************************************************************/ FieldPerp& FieldPerp::operator=(const FieldPerp& rhs) { @@ -97,7 +98,6 @@ FieldPerp& FieldPerp::operator=(const FieldPerp& rhs) { } FieldPerp& FieldPerp::operator=(const BoutReal rhs) { - TRACE("FieldPerp = BoutReal"); allocate(); diff --git a/src/field/gen_fieldops.jinja b/src/field/gen_fieldops.jinja index ecd4e628cc..5476242d50 100644 --- a/src/field/gen_fieldops.jinja +++ b/src/field/gen_fieldops.jinja @@ -129,9 +129,16 @@ } {% endif %} + {% if lhs == "Field3D" %} + track(rhs, "operator{{operator}}="); + {% endif %} + checkData(*this); } else { + {% if lhs == "Field3D" %} + track(rhs, "operator{{operator}}="); + {% endif %} (*this) = (*this) {{operator}} {{rhs.name}}; } return *this; diff --git a/src/field/gen_fieldops.py b/src/field/gen_fieldops.py index 29631ff7aa..3e07d6fec4 100755 --- a/src/field/gen_fieldops.py +++ b/src/field/gen_fieldops.py @@ -11,7 +11,6 @@ """ - from __future__ import print_function from builtins import object diff --git a/src/field/generated_fieldops.cxx b/src/field/generated_fieldops.cxx index 6b778acee3..24bfde425c 100644 --- a/src/field/generated_fieldops.cxx +++ b/src/field/generated_fieldops.cxx @@ -42,9 +42,12 @@ Field3D& Field3D::operator*=(const Field3D& rhs) { BOUT_FOR(index, this->getRegion("RGN_ALL")) { (*this)[index] *= rhs[index]; } + track(rhs, "operator*="); + checkData(*this); } else { + track(rhs, "operator*="); (*this) = (*this) * rhs; } return *this; @@ -86,9 +89,12 @@ Field3D& Field3D::operator/=(const Field3D& rhs) { BOUT_FOR(index, this->getRegion("RGN_ALL")) { (*this)[index] /= rhs[index]; } + track(rhs, "operator/="); + checkData(*this); } else { + track(rhs, "operator/="); (*this) = (*this) / rhs; } return *this; @@ -130,9 +136,12 @@ Field3D& Field3D::operator+=(const Field3D& rhs) { BOUT_FOR(index, this->getRegion("RGN_ALL")) { (*this)[index] += rhs[index]; } + track(rhs, "operator+="); + checkData(*this); } else { + track(rhs, "operator+="); (*this) = (*this) + rhs; } return *this; @@ -174,9 +183,12 @@ Field3D& Field3D::operator-=(const Field3D& rhs) { BOUT_FOR(index, this->getRegion("RGN_ALL")) { (*this)[index] -= rhs[index]; } + track(rhs, "operator-="); + checkData(*this); } else { + track(rhs, "operator-="); (*this) = (*this) - rhs; } return *this; @@ -226,9 +238,12 @@ Field3D& Field3D::operator*=(const Field2D& rhs) { } } + track(rhs, "operator*="); + checkData(*this); } else { + track(rhs, "operator*="); (*this) = (*this) * rhs; } return *this; @@ -280,9 +295,12 @@ Field3D& Field3D::operator/=(const Field2D& rhs) { } } + track(rhs, "operator/="); + checkData(*this); } else { + track(rhs, "operator/="); (*this) = (*this) / rhs; } return *this; @@ -332,9 +350,12 @@ Field3D& Field3D::operator+=(const Field2D& rhs) { } } + track(rhs, "operator+="); + checkData(*this); } else { + track(rhs, "operator+="); (*this) = (*this) + rhs; } return *this; @@ -384,9 +405,12 @@ Field3D& Field3D::operator-=(const Field2D& rhs) { } } + track(rhs, "operator-="); + checkData(*this); } else { + track(rhs, "operator-="); (*this) = (*this) - rhs; } return *this; @@ -504,9 +528,12 @@ Field3D& Field3D::operator*=(const BoutReal rhs) { BOUT_FOR(index, this->getRegion("RGN_ALL")) { (*this)[index] *= rhs; } + track(rhs, "operator*="); + checkData(*this); } else { + track(rhs, "operator*="); (*this) = (*this) * rhs; } return *this; @@ -546,9 +573,12 @@ Field3D& Field3D::operator/=(const BoutReal rhs) { const auto tmp = 1.0 / rhs; BOUT_FOR(index, this->getRegion("RGN_ALL")) { (*this)[index] *= tmp; } + track(rhs, "operator/="); + checkData(*this); } else { + track(rhs, "operator/="); (*this) = (*this) / rhs; } return *this; @@ -586,9 +616,12 @@ Field3D& Field3D::operator+=(const BoutReal rhs) { BOUT_FOR(index, this->getRegion("RGN_ALL")) { (*this)[index] += rhs; } + track(rhs, "operator+="); + checkData(*this); } else { + track(rhs, "operator+="); (*this) = (*this) + rhs; } return *this; @@ -626,9 +659,12 @@ Field3D& Field3D::operator-=(const BoutReal rhs) { BOUT_FOR(index, this->getRegion("RGN_ALL")) { (*this)[index] -= rhs; } + track(rhs, "operator-="); + checkData(*this); } else { + track(rhs, "operator-="); (*this) = (*this) - rhs; } return *this; diff --git a/src/field/initialprofiles.cxx b/src/field/initialprofiles.cxx index ded5aa9882..6a437dbe02 100644 --- a/src/field/initialprofiles.cxx +++ b/src/field/initialprofiles.cxx @@ -1,23 +1,10 @@ /************************************************************************** * Sets initial profiles * - * ChangeLog - * ========= - * - * 2011-02-12 Ben Dudson - * * Changed to use new options system. For now the structure of the - * options is the same, but this could be modified more easily in future - * - * 2010-05-12 Ben Dudson - * - * * Changed random numbers to use a hash of the parameters - * so that the phase doesn't vary with number of processors or grid size - * User can vary phase to give a different random sequence - * ************************************************************************** - * Copyright 2010 B.D.Dudson, S.Farley, M.V.Umansky, X.Q.Xu + * Copyright 2010 - 2026 BOUT++ contributors * - * Contact: Ben Dudson, bd512@york.ac.uk + * Contact: Ben Dudson, dudson2@llnl.gov * * This file is part of BOUT++. * @@ -42,21 +29,17 @@ #include #include #include -#include void initial_profile(const std::string& name, Field3D& var) { - AUTO_TRACE(); Mesh* localmesh = var.getMesh(); Options* varOpts = Options::getRoot()->getSection(name); - FieldFactory f(localmesh); - std::string function; VAROPTION(varOpts, function, "0.0"); - var = f.create3D(function, varOpts, nullptr, var.getLocation()); + var = FieldFactory::get()->create3D(function, varOpts, localmesh, var.getLocation()); // Optionally scale the variable BoutReal scale; @@ -65,18 +48,15 @@ void initial_profile(const std::string& name, Field3D& var) { } void initial_profile(const std::string& name, Field2D& var) { - AUTO_TRACE(); Mesh* localmesh = var.getMesh(); Options* varOpts = Options::getRoot()->getSection(name); - FieldFactory f(localmesh); - std::string function; VAROPTION(varOpts, function, "0.0"); - var = f.create2D(function, varOpts, nullptr, var.getLocation()); + var = FieldFactory::get()->create2D(function, varOpts, localmesh, var.getLocation()); // Optionally scale the variable BoutReal scale; @@ -85,7 +65,6 @@ void initial_profile(const std::string& name, Field2D& var) { } void initial_profile(const std::string& name, Vector2D& var) { - AUTO_TRACE(); if (var.covariant) { initial_profile(name + "_x", var.x); @@ -99,7 +78,6 @@ void initial_profile(const std::string& name, Vector2D& var) { } void initial_profile(const std::string& name, Vector3D& var) { - AUTO_TRACE(); if (var.covariant) { initial_profile(name + "_x", var.x); diff --git a/src/field/vecops.cxx b/src/field/vecops.cxx index 5f34e2af02..1677187fcd 100644 --- a/src/field/vecops.cxx +++ b/src/field/vecops.cxx @@ -29,7 +29,6 @@ #include #include -#include #include #include #include @@ -39,7 +38,7 @@ **************************************************************************/ Vector2D Grad(const Field2D& f, CELL_LOC outloc, const std::string& method) { - TRACE("Grad( Field2D )"); + SCOREP0(); CELL_LOC outloc_x, outloc_y, outloc_z; if (outloc == CELL_VSHIFT) { @@ -68,7 +67,7 @@ Vector2D Grad(const Field2D& f, CELL_LOC outloc, const std::string& method) { } Vector3D Grad(const Field3D& f, CELL_LOC outloc, const std::string& method) { - TRACE("Grad( Field3D )"); + SCOREP0(); CELL_LOC outloc_x, outloc_y, outloc_z; if (outloc == CELL_VSHIFT) { @@ -97,7 +96,7 @@ Vector3D Grad(const Field3D& f, CELL_LOC outloc, const std::string& method) { } Vector3D Grad_perp(const Field3D& f, CELL_LOC outloc, const std::string& method) { - TRACE("Grad_perp( Field3D )"); + SCOREP0(); ASSERT1(outloc == CELL_DEFAULT || outloc == f.getLocation()); @@ -119,7 +118,7 @@ Vector3D Grad_perp(const Field3D& f, CELL_LOC outloc, const std::string& method) } Vector2D Grad_perp(const Field2D& f, CELL_LOC outloc, const std::string& method) { - AUTO_TRACE(); + SCOREP0(); ASSERT1(outloc == CELL_DEFAULT || outloc == f.getLocation()); @@ -145,7 +144,7 @@ Vector2D Grad_perp(const Field2D& f, CELL_LOC outloc, const std::string& method) Coordinates::FieldMetric Div(const Vector2D& v, CELL_LOC outloc, const std::string& method) { - TRACE("Div( Vector2D )"); + SCOREP0(); if (outloc == CELL_DEFAULT) { outloc = v.getLocation(); @@ -170,7 +169,7 @@ Coordinates::FieldMetric Div(const Vector2D& v, CELL_LOC outloc, } Field3D Div(const Vector3D& v, CELL_LOC outloc, const std::string& method) { - TRACE("Div( Vector3D )"); + SCOREP0(); if (outloc == CELL_DEFAULT) { outloc = v.getLocation(); @@ -209,7 +208,7 @@ Field3D Div(const Vector3D& v, CELL_LOC outloc, const std::string& method) { Coordinates::FieldMetric Div(const Vector2D& v, const Field2D& f, CELL_LOC outloc, const std::string& method) { - TRACE("Div( Vector2D, Field2D )"); + SCOREP0(); if (outloc == CELL_DEFAULT) { outloc = v.getLocation(); @@ -236,7 +235,6 @@ Coordinates::FieldMetric Div(const Vector2D& v, const Field2D& f, CELL_LOC outlo Field3D Div(const Vector3D& v, const Field3D& f, CELL_LOC outloc, const std::string& method) { - TRACE("Div( Vector3D, Field3D )"); if (outloc == CELL_DEFAULT) { outloc = v.getLocation(); @@ -265,8 +263,6 @@ Field3D Div(const Vector3D& v, const Field3D& f, CELL_LOC outloc, Vector2D Curl(const Vector2D& v) { - TRACE("Curl( Vector2D )"); - ASSERT1(v.getLocation() != CELL_VSHIFT); Mesh* localmesh = v.getMesh(); auto metric = v.x.getCoordinates(); @@ -292,7 +288,7 @@ Vector2D Curl(const Vector2D& v) { } Vector3D Curl(const Vector3D& v) { - TRACE("Curl( Vector3D )"); + SCOREP0(); ASSERT1(v.getLocation() != CELL_VSHIFT); @@ -323,7 +319,7 @@ Vector3D Curl(const Vector3D& v) { * Upwinding operators **************************************************************************/ Coordinates::FieldMetric V_dot_Grad(const Vector2D& v, const Field2D& f) { - TRACE("V_dot_Grad( Vector2D , Field2D )"); + SCOREP0(); // Get contravariant components of v @@ -334,7 +330,7 @@ Coordinates::FieldMetric V_dot_Grad(const Vector2D& v, const Field2D& f) { } Field3D V_dot_Grad(const Vector2D& v, const Field3D& f) { - TRACE("V_dot_Grad( Vector2D , Field3D )"); + SCOREP0(); // Get contravariant components of v @@ -345,7 +341,7 @@ Field3D V_dot_Grad(const Vector2D& v, const Field3D& f) { } Field3D V_dot_Grad(const Vector3D& v, const Field2D& f) { - TRACE("V_dot_Grad( Vector3D , Field2D )"); + SCOREP0(); // Get contravariant components of v @@ -356,7 +352,7 @@ Field3D V_dot_Grad(const Vector3D& v, const Field2D& f) { } Field3D V_dot_Grad(const Vector3D& v, const Field3D& f) { - TRACE("V_dot_Grad( Vector3D , Field3D )"); + SCOREP0(); // Get contravariant components of v @@ -370,7 +366,7 @@ Field3D V_dot_Grad(const Vector3D& v, const Field3D& f) { // operation (addition) between the two input types. template R V_dot_Grad(const T& v, const F& a) { - AUTO_TRACE(); + SCOREP0(); ASSERT1(v.getLocation() == a.getLocation()); ASSERT1(v.getLocation() != CELL_VSHIFT); diff --git a/src/field/vector2d.cxx b/src/field/vector2d.cxx index 74d3a88a22..ab2d993549 100644 --- a/src/field/vector2d.cxx +++ b/src/field/vector2d.cxx @@ -429,7 +429,7 @@ CELL_LOC Vector2D::getLocation() const { Vector2D& Vector2D::setLocation(CELL_LOC loc) { SCOREP0(); - TRACE("Vector2D::setLocation"); + if (loc == CELL_DEFAULT) { loc = CELL_CENTRE; } diff --git a/src/field/vector3d.cxx b/src/field/vector3d.cxx index 56124595b3..7fb4a88918 100644 --- a/src/field/vector3d.cxx +++ b/src/field/vector3d.cxx @@ -551,7 +551,7 @@ CELL_LOC Vector3D::getLocation() const { Vector3D& Vector3D::setLocation(CELL_LOC loc) { SCOREP0(); - TRACE("Vector3D::setLocation"); + if (loc == CELL_DEFAULT) { loc = CELL_CENTRE; } diff --git a/src/invert/fft_fftw.cxx b/src/invert/fft_fftw.cxx index 05977e5f3f..2c633da6d5 100644 --- a/src/invert/fft_fftw.cxx +++ b/src/invert/fft_fftw.cxx @@ -27,8 +27,10 @@ #include "bout/build_defines.hxx" +#include #include #include +#include #include #include @@ -36,7 +38,6 @@ #include #include -#include #include #if BOUT_USE_OPENMP @@ -46,6 +47,12 @@ #include #endif // BOUT_HAS_FFTW +#if BOUT_CHECK_LEVEL > 0 +#include + +#include +#endif + namespace bout { namespace fft { @@ -527,5 +534,14 @@ Array irfft(const Array& in, int length) { return out; } +#if BOUT_CHECK_LEVEL > 0 +void assertZSerial(const Mesh& mesh, std::string_view name) { + if (mesh.getNZPE() != 1) { + throw BoutException("{} uses FFTs which are currently incompatible with multiple " + "processors in Z (using {})", + name, mesh.getNZPE()); + } +} +#endif } // namespace fft } // namespace bout diff --git a/src/invert/laplace/common_transform.cxx b/src/invert/laplace/common_transform.cxx new file mode 100644 index 0000000000..98571624a1 --- /dev/null +++ b/src/invert/laplace/common_transform.cxx @@ -0,0 +1,381 @@ +#include "common_transform.hxx" + +#include "bout/array.hxx" +#include "bout/bout_types.hxx" +#include "bout/constants.hxx" +#include "bout/dcomplex.hxx" +#include "bout/fft.hxx" +#include "bout/field2d.hxx" +#include "bout/field3d.hxx" +#include "bout/fieldperp.hxx" +#include "bout/invert_laplace.hxx" +#include "bout/mesh.hxx" +#include "bout/openmpwrap.hxx" +#include "bout/utils.hxx" + +#include + +FFTTransform::FFTTransform(const Mesh& mesh, int nmode, int xs, int xe, int ys, int ye, + int zs, int ze, int inbndry, int outbndry, + bool inner_boundary_set_on_first_x, + bool outer_boundary_set_on_last_x, bool zero_DC) + : zlength(getUniform(mesh.getCoordinatesConst()->zlength())), nmode(nmode), xs(xs), + xe(xe), ys(ys), ye(ye), zs(zs), ze(ze), nx(xe - xs + 1), ny(ye - ys + 1), + nz(ze - zs + 1), nxny(nx * ny), nsys(nmode * ny), inbndry(inbndry), + outbndry(outbndry), inner_boundary_set_on_first_x(inner_boundary_set_on_first_x), + outer_boundary_set_on_last_x(outer_boundary_set_on_last_x), zero_DC(zero_DC), + localmesh(&mesh) {} + +auto FFTTransform::forward(const Laplacian& laplacian, const Field3D& rhs, + const Field3D& x0, const Field2D& Acoef, const Field2D& C1coef, + const Field2D& C2coef, const Field2D& Dcoef) const + -> Matrices { + + Matrices result(nsys, nx); + + BOUT_OMP_PERF(parallel) + { + /// Create a local thread-scope working array + // ZFFT routine expects input of this length + auto k1d = Array((nz / 2) + 1); + + // Loop over X and Y indices, including boundaries but not guard cells + // (unless periodic in x) + BOUT_OMP_PERF(for) + for (int ind = 0; ind < nxny; ++ind) { + const int ix = xs + (ind / ny); + const int iy = ys + (ind % ny); + + // Take FFT in Z direction, apply shift, and put result in k1d + + if (((ix < inbndry) and inner_boundary_set_on_first_x) + || ((localmesh->LocalNx - ix - 1 < outbndry) + and outer_boundary_set_on_last_x)) { + // Use the values in x0 in the boundary + rfft(&(x0(ix, iy, zs)), nz, std::begin(k1d)); + } else { + rfft(&(rhs(ix, iy, zs)), nz, std::begin(k1d)); + } + + // Copy into array, transposing so kz is first index + for (int kz = 0; kz < nmode; kz++) { + result.bcmplx(((iy - ys) * nmode) + kz, ix - xs) = k1d[kz]; + } + } + + // Get elements of the tridiagonal matrix + // including boundary conditions + BOUT_OMP_PERF(for nowait) + for (int ind = 0; ind < nsys; ind++) { + const int iy = ys + (ind / nmode); + const int kz = ind % nmode; + + const BoutReal kwave = kz * 2.0 * PI / zlength; // wave number is 1/[rad] + laplacian.tridagMatrix(&result.a(ind, 0), &result.b(ind, 0), &result.c(ind, 0), + &result.bcmplx(ind, 0), iy, kz, kwave, &Acoef, &C1coef, + &C2coef, &Dcoef, false); + } + } + return result; +} + +auto FFTTransform::backward(const Field3D& rhs, const Matrix& xcmplx3D) const + -> Field3D { + Field3D x{emptyFrom(rhs)}; + + // FFT back to real space + BOUT_OMP_PERF(parallel) + { + /// Create a local thread-scope working array + // ZFFT routine expects input of this length + auto k1d = Array((nz / 2) + 1); + + BOUT_OMP_PERF(for nowait) + for (int ind = 0; ind < nxny; ++ind) { // Loop over X and Y + const int ix = xs + (ind / ny); + const int iy = ys + (ind % ny); + + if (zero_DC) { + k1d[0] = 0.; + } + + for (int kz = static_cast(zero_DC); kz < nmode; kz++) { + k1d[kz] = xcmplx3D(((iy - ys) * nmode) + kz, ix - xs); + } + + for (int kz = nmode; kz < (nz / 2) + 1; kz++) { + k1d[kz] = 0.0; // Filtering out all higher harmonics + } + + irfft(std::begin(k1d), nz, &(x(ix, iy, zs))); + } + } + + return x; +} + +auto FFTTransform::forward(const Laplacian& laplacian, const FieldPerp& rhs, + const FieldPerp& x0, const Field2D& Acoef, + const Field2D& C1coef, const Field2D& C2coef, + const Field2D& Dcoef) const -> Matrices { + + Matrices result(nmode, nx); + const int jy = rhs.getIndex(); + + BOUT_OMP_PERF(parallel) + { + /// Create a local thread-scope working array + // ZFFT routine expects input of this length + auto k1d = Array((nz / 2) + 1); + + // Loop over X indices, including boundaries but not guard + // cells (unless periodic in x) + BOUT_OMP_PERF(for) + for (int ix = xs; ix <= xe; ix++) { + // Take FFT in Z direction, apply shift, and put result in k1d + + if (((ix < inbndry) and inner_boundary_set_on_first_x) + || ((localmesh->LocalNx - ix - 1 < outbndry) + and outer_boundary_set_on_last_x)) { + // Use the values in x0 in the boundary + rfft(&(x0(ix, zs)), nz, std::begin(k1d)); + } else { + rfft(&(rhs(ix, zs)), nz, std::begin(k1d)); + } + + // Copy into array, transposing so kz is first index + for (int kz = 0; kz < nmode; kz++) { + result.bcmplx(kz, ix - xs) = k1d[kz]; + } + } + + // Get elements of the tridiagonal matrix + // including boundary conditions + BOUT_OMP_PERF(for nowait) + for (int kz = 0; kz < nmode; kz++) { + const BoutReal kwave = kz * 2.0 * PI / zlength; // wave number is 1/[rad] + laplacian.tridagMatrix(&result.a(kz, 0), &result.b(kz, 0), &result.c(kz, 0), + &result.bcmplx(kz, 0), jy, kz, kwave, &Acoef, &C1coef, + &C2coef, &Dcoef, false); + } + } + return result; +} + +auto FFTTransform::backward(const FieldPerp& rhs, const Matrix& xcmplx) const + -> FieldPerp { + FieldPerp x{emptyFrom(rhs)}; + + // FFT back to real space + BOUT_OMP_PERF(parallel) + { + /// Create a local thread-scope working array + // ZFFT routine expects input of this length + auto k1d = Array((nz / 2) + 1); + + BOUT_OMP_PERF(for nowait) + for (int ix = xs; ix <= xe; ++ix) { + if (zero_DC) { + k1d[0] = 0.; + } + + for (int kz = static_cast(zero_DC); kz < nmode; kz++) { + k1d[kz] = xcmplx(kz, ix - xs); + } + + for (int kz = nmode; kz < (nz / 2) + 1; kz++) { + k1d[kz] = 0.0; // Filtering out all higher harmonics + } + + irfft(std::begin(k1d), nz, &(x(ix, zs))); + } + } + + checkData(x); + + return x; +} + +DSTTransform::DSTTransform(const Mesh& mesh, int nmode, int xs, int xe, int ys, int ye, + int zs, int ze, int inbndry, int outbndry, + bool inner_boundary_set_on_first_x, + bool outer_boundary_set_on_last_x, bool zero_DC) + : zlength(getUniform(mesh.getCoordinatesConst()->zlength())), nmode(nmode), xs(xs), + xe(xe), ys(ys), ye(ye), zs(zs), ze(ze), nx(xe - xs + 1), ny(ye - ys + 1), + nz(ze - zs + 1), nxny(nx * ny), nsys(nmode * ny), inbndry(inbndry), + outbndry(outbndry), inner_boundary_set_on_first_x(inner_boundary_set_on_first_x), + outer_boundary_set_on_last_x(outer_boundary_set_on_last_x), zero_DC(zero_DC), + localmesh(&mesh) {} + +auto DSTTransform::forward(const Laplacian& laplacian, const Field3D& rhs, + const Field3D& x0, const Field2D& Acoef, const Field2D& C1coef, + const Field2D& C2coef, const Field2D& Dcoef) const + -> Matrices { + + Matrices result(nsys, nx); + + BOUT_OMP_PERF(parallel) + { + /// Create a local thread-scope working array + // ZDST routine expects input of this length + auto k1d = Array(nz); + + // Loop over X and Y indices, including boundaries but not guard cells + // (unless periodic in x) + BOUT_OMP_PERF(for) + for (int ind = 0; ind < nxny; ++ind) { + const int ix = xs + (ind / ny); + const int iy = ys + (ind % ny); + + // Take DST in Z direction, apply shift, and put result in k1d + + if (((ix < inbndry) and inner_boundary_set_on_first_x) + || ((localmesh->LocalNx - ix - 1 < outbndry) + and outer_boundary_set_on_last_x)) { + // Use the values in x0 in the boundary + bout::fft::DST(&(x0(ix, iy, zs + 1)), nz - 2, std::begin(k1d)); + } else { + bout::fft::DST(&(rhs(ix, iy, zs + 1)), nz - 2, std::begin(k1d)); + } + + // Copy into array, transposing so kz is first index + for (int kz = 0; kz < nmode; kz++) { + result.bcmplx(((iy - ys) * nmode) + kz, ix - xs) = k1d[kz]; + } + } + + // Get elements of the tridiagonal matrix + // including boundary conditions + BOUT_OMP_PERF(for nowait) + for (int ind = 0; ind < nsys; ind++) { + const int iy = ys + (ind / nmode); + const int kz = ind % nmode; + + const BoutReal kwave = kz * 2.0 * PI / zlength; // wave number is 1/[rad] + laplacian.tridagMatrix(&result.a(ind, 0), &result.b(ind, 0), &result.c(ind, 0), + &result.bcmplx(ind, 0), iy, kz, kwave, &Acoef, &C1coef, + &C2coef, &Dcoef, false); + } + } + return result; +} + +auto DSTTransform::backward(const Field3D& rhs, const Matrix& xcmplx3D) const + -> Field3D { + Field3D x{emptyFrom(rhs)}; + + // DST back to real space + BOUT_OMP_PERF(parallel) + { + /// Create a local thread-scope working array + // ZDST routine expects input of this length + auto k1d = Array((nz / 2) + 1); + + BOUT_OMP_PERF(for nowait) + for (int ind = 0; ind < nxny; ++ind) { // Loop over X and Y + const int ix = xs + (ind / ny); + const int iy = ys + (ind % ny); + + if (zero_DC) { + k1d[0] = 0.; + } + + for (int kz = static_cast(zero_DC); kz < nmode; kz++) { + k1d[kz] = xcmplx3D(((iy - ys) * nmode) + kz, ix - xs); + } + + for (int kz = nmode; kz < (nz / 2) + 1; kz++) { + k1d[kz] = 0.0; // Filtering out all higher harmonics + } + + bout::fft::DST_rev(std::begin(k1d), nz - 2, &(x(ix, iy, zs + 1))); + + x(ix, iy, zs) = -x(ix, iy, zs + 2); + x(ix, iy, ze) = -x(ix, iy, ze - 2); + } + } + + return x; +} + +auto DSTTransform::forward(const Laplacian& laplacian, const FieldPerp& rhs, + const FieldPerp& x0, const Field2D& Acoef, + const Field2D& C1coef, const Field2D& C2coef, + const Field2D& Dcoef) const -> Matrices { + + Matrices result(nmode, nx); + const int jy = rhs.getIndex(); + + BOUT_OMP_PERF(parallel) + { + /// Create a local thread-scope working array + // ZDST routine expects input of this length + auto k1d = Array((nz / 2) + 1); + + // Loop over X indices, including boundaries but not guard + // cells (unless periodic in x) + BOUT_OMP_PERF(for) + for (int ix = xs; ix <= xe; ix++) { + // Take DST in Z direction, apply shift, and put result in k1d + + if (((ix < inbndry) and inner_boundary_set_on_first_x) + || ((localmesh->LocalNx - ix - 1 < outbndry) + and outer_boundary_set_on_last_x)) { + // Use the values in x0 in the boundary + rfft(&(x0(ix, zs)), nz, std::begin(k1d)); + } else { + rfft(&(rhs(ix, zs)), nz, std::begin(k1d)); + } + + // Copy into array, transposing so kz is first index + for (int kz = 0; kz < nmode; kz++) { + result.bcmplx(kz, ix - xs) = k1d[kz]; + } + } + + // Get elements of the tridiagonal matrix + // including boundary conditions + BOUT_OMP_PERF(for nowait) + for (int kz = 0; kz < nmode; kz++) { + const BoutReal kwave = kz * 2.0 * PI / zlength; // wave number is 1/[rad] + laplacian.tridagMatrix(&result.a(kz, 0), &result.b(kz, 0), &result.c(kz, 0), + &result.bcmplx(kz, 0), jy, kz, kwave, &Acoef, &C1coef, + &C2coef, &Dcoef, false); + } + } + return result; +} + +auto DSTTransform::backward(const FieldPerp& rhs, const Matrix& xcmplx) const + -> FieldPerp { + FieldPerp x{emptyFrom(rhs)}; + + // DST back to real space + BOUT_OMP_PERF(parallel) + { + /// Create a local thread-scope working array + // ZDST routine expects input of this length + auto k1d = Array((nz / 2) + 1); + + BOUT_OMP_PERF(for nowait) + for (int ix = xs; ix < xe; ++ix) { + if (zero_DC) { + k1d[0] = 0.; + } + + for (int kz = static_cast(zero_DC); kz < nmode; kz++) { + k1d[kz] = xcmplx(kz, ix - xs); + } + + for (int kz = nmode; kz < (nz / 2) + 1; kz++) { + k1d[kz] = 0.0; // Filtering out all higher harmonics + } + + irfft(std::begin(k1d), nz, &(x(ix, zs))); + } + } + + checkData(x); + + return x; +} diff --git a/src/invert/laplace/common_transform.hxx b/src/invert/laplace/common_transform.hxx new file mode 100644 index 0000000000..a05ca2feb3 --- /dev/null +++ b/src/invert/laplace/common_transform.hxx @@ -0,0 +1,132 @@ +#pragma once + +#include "bout/bout_types.hxx" +#include "bout/dcomplex.hxx" +#include "bout/utils.hxx" + +class Laplacian; +class Field2D; +class Field3D; +class FieldPerp; +class Mesh; + +/// Helper class to do FFTs for `LaplaceCyclic`, `LaplacePCR`, `LaplacePCR_THOMAS` +class FFTTransform { + /// Physical length of the Z domain + BoutReal zlength; + /// Number of unfiltered Fourier modes + int nmode; + + int xs; + int xe; + int ys; + int ye; + int zs; + int ze; + + int nx; + int ny; + /// Number of (interior) points in Z + int nz; + int nxny; + /// Number of systems to solve = number of unfiltered Fourier modes times number of y + /// points + int nsys; + /// Number of inner boundary cells + int inbndry; + /// Number of outer boundary cells + int outbndry; + + bool inner_boundary_set_on_first_x; + bool outer_boundary_set_on_last_x; + + bool zero_DC; + + const Mesh* localmesh; + +public: + FFTTransform(const Mesh& mesh, int nmode, int xs, int xe, int ys, int ye, int zs, + int ze, int inbndry, int outbndry, bool inner_boundary_set_on_first_x, + bool outer_boundary_set_on_last_x, bool zero_DC); + + struct Matrices { + Matrix a; + Matrix b; + Matrix c; + Matrix bcmplx; + + Matrices(int nsys, int nx) + : a(nsys, nx), b(nsys, nx), c(nsys, nx), bcmplx(nsys, nx) {} + }; + + auto forward(const Laplacian& laplacian, const Field3D& rhs, const Field3D& x0, + const Field2D& Acoef, const Field2D& C1coef, const Field2D& C2coef, + const Field2D& Dcoef) const -> Matrices; + auto backward(const Field3D& rhs, const Matrix& xcmplx3D) const -> Field3D; + + auto forward(const Laplacian& laplacian, const FieldPerp& rhs, const FieldPerp& x0, + const Field2D& Acoef, const Field2D& C1coef, const Field2D& C2coef, + const Field2D& Dcoef) const -> Matrices; + auto backward(const FieldPerp& rhs, const Matrix& xcmplx) const -> FieldPerp; +}; + +/// Helper class to do discrete sin transforms (DSTs) for `LaplaceCyclic`, +/// `LaplacePCR`, `LaplacePCR_THOMAS` +class DSTTransform { + /// Physical length of the Z domain + BoutReal zlength; + /// Number of unfiltered Fourier modes + int nmode; + + int xs; + int xe; + int ys; + int ye; + int zs; + int ze; + + int nx; + int ny; + /// Number of (interior) points in Z + int nz; + int nxny; + /// Number of systems to solve = number of unfiltered Fourier modes times number of y + /// points + int nsys; + /// Number of inner boundary cells + int inbndry; + /// Number of outer boundary cells + int outbndry; + + bool inner_boundary_set_on_first_x; + bool outer_boundary_set_on_last_x; + + bool zero_DC; + + const Mesh* localmesh; + +public: + DSTTransform(const Mesh& mesh, int nmode, int xs, int xe, int ys, int ye, int zs, + int ze, int inbndry, int outbndry, bool inner_boundary_set_on_first_x, + bool outer_boundary_set_on_last_x, bool zero_DC); + + struct Matrices { + Matrix a; + Matrix b; + Matrix c; + Matrix bcmplx; + + Matrices(int nsys, int nx) + : a(nsys, nx), b(nsys, nx), c(nsys, nx), bcmplx(nsys, nx) {} + }; + + auto forward(const Laplacian& laplacian, const Field3D& rhs, const Field3D& x0, + const Field2D& Acoef, const Field2D& C1coef, const Field2D& C2coef, + const Field2D& Dcoef) const -> Matrices; + auto backward(const Field3D& rhs, const Matrix& xcmplx3D) const -> Field3D; + + auto forward(const Laplacian& laplacian, const FieldPerp& rhs, const FieldPerp& x0, + const Field2D& Acoef, const Field2D& C1coef, const Field2D& C2coef, + const Field2D& Dcoef) const -> Matrices; + auto backward(const FieldPerp& rhs, const Matrix& xcmplx) const -> FieldPerp; +}; diff --git a/src/invert/laplace/impls/cyclic/cyclic_laplace.cxx b/src/invert/laplace/impls/cyclic/cyclic_laplace.cxx index 5ce4e540b7..0ee6fbe0cd 100644 --- a/src/invert/laplace/impls/cyclic/cyclic_laplace.cxx +++ b/src/invert/laplace/impls/cyclic/cyclic_laplace.cxx @@ -38,8 +38,14 @@ #if not BOUT_USE_METRIC_3D #include "cyclic_laplace.hxx" -#include "bout/assert.hxx" -#include "bout/bout_types.hxx" + +#include "../../common_transform.hxx" + +#include "bout/array.hxx" +#include "bout/dcomplex.hxx" +#include +#include +#include #include #include #include @@ -49,12 +55,15 @@ #include #include +#include #include LaplaceCyclic::LaplaceCyclic(Options* opt, const CELL_LOC loc, Mesh* mesh_in, Solver* UNUSED(solver)) : Laplacian(opt, loc, mesh_in), Acoef(0.0), C1coef(1.0), C2coef(1.0), Dcoef(1.0) { + bout::fft::assertZSerial(*localmesh, "`cyclic` inversion"); + Acoef.setLocation(location); C1coef.setLocation(location); C2coef.setLocation(location); @@ -113,15 +122,11 @@ FieldPerp LaplaceCyclic::solve(const FieldPerp& rhs, const FieldPerp& x0) { ASSERT1(rhs.getLocation() == location); ASSERT1(x0.getLocation() == location); - FieldPerp x{emptyFrom(rhs)}; // Result - - int jy = rhs.getIndex(); // Get the Y index - x.setIndex(jy); - // Get the width of the boundary // If the flags to assign that only one guard cell should be used is set - int inbndry = localmesh->xstart, outbndry = localmesh->xstart; + int inbndry = localmesh->xstart; + int outbndry = localmesh->xstart; if (isGlobalFlagSet(INVERT_BOTH_BNDRY_ONE) || (localmesh->xstart < 2)) { inbndry = outbndry = 1; } @@ -133,176 +138,53 @@ FieldPerp LaplaceCyclic::solve(const FieldPerp& rhs, const FieldPerp& x0) { } if (dst) { - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array( - localmesh->LocalNz); // ZFFT routine expects input of this length - - // Loop over X indices, including boundaries but not guard cells. (unless periodic - // in x) - BOUT_OMP_PERF(for) - for (int ix = xs; ix <= xe; ix++) { - // Take DST in Z direction and put result in k1d - - if (((ix < inbndry) && isInnerBoundaryFlagSetOnFirstX(INVERT_SET)) - || ((localmesh->LocalNx - ix - 1 < outbndry) - && isOuterBoundaryFlagSetOnLastX(INVERT_SET))) { - // Use the values in x0 in the boundary - DST(x0[ix] + 1, localmesh->LocalNz - 2, std::begin(k1d)); - } else { - DST(rhs[ix] + 1, localmesh->LocalNz - 2, std::begin(k1d)); - } - - // Copy into array, transposing so kz is first index - for (int kz = 0; kz < nmode; kz++) { - bcmplx(kz, ix - xs) = k1d[kz]; - } - } + const DSTTransform transform( + *localmesh, nmode, xs, xe, 0, 0, localmesh->zstart, localmesh->zend, inbndry, + outbndry, isInnerBoundaryFlagSetOnFirstX(INVERT_SET), + isOuterBoundaryFlagSetOnLastX(INVERT_SET), isGlobalFlagSet(INVERT_ZERO_DC)); - // Get elements of the tridiagonal matrix - // including boundary conditions - BoutReal zlen = getUniform(coords->dz) * (localmesh->LocalNz - 3); - BOUT_OMP_PERF(for nowait) - for (int kz = 0; kz < nmode; kz++) { - // wave number is 1/[rad]; DST has extra 2. - BoutReal kwave = kz * 2.0 * PI / (2. * zlen); - - tridagMatrix(&a(kz, 0), &b(kz, 0), &c(kz, 0), &bcmplx(kz, 0), jy, - kz, // wave number index - kwave, // kwave (inverse wave length) - &Acoef, &C1coef, &C2coef, &Dcoef, - false, // Don't include guard cells in arrays - false); // Z domain not periodic - } - } + auto matrices = transform.forward(*this, rhs, x0, Acoef, C1coef, C2coef, Dcoef); // Solve tridiagonal systems cr->setCoefs(a, b, c); cr->solve(bcmplx, xcmplx); - // FFT back to real space - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - - // ZFFT routine expects input of this length - auto k1d = Array(localmesh->LocalNz); - - BOUT_OMP_PERF(for nowait) - for (int ix = xs; ix <= xe; ix++) { - for (int kz = 0; kz < nmode; kz++) { - k1d[kz] = xcmplx(kz, ix - xs); - } - - for (int kz = nmode; kz < (localmesh->LocalNz); kz++) { - k1d[kz] = 0.0; // Filtering out all higher harmonics - } - - DST_rev(std::begin(k1d), localmesh->LocalNz - 2, x[ix] + 1); - - x(ix, 0) = -x(ix, 2); - x(ix, localmesh->LocalNz - 1) = -x(ix, localmesh->LocalNz - 3); - } - } - } else { - const BoutReal zlength = getUniform(coords->zlength()); - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - // ZFFT routine expects input of this length - auto k1d = Array((localmesh->LocalNz) / 2 + 1); - - // Loop over X indices, including boundaries but not guard - // cells (unless periodic in x) - BOUT_OMP_PERF(for) - for (int ix = xs; ix <= xe; ix++) { - // Take FFT in Z direction, apply shift, and put result in k1d - - if (((ix < inbndry) && isInnerBoundaryFlagSetOnFirstX(INVERT_SET)) - || ((localmesh->LocalNx - ix - 1 < outbndry) - && isOuterBoundaryFlagSetOnLastX(INVERT_SET))) { - // Use the values in x0 in the boundary - rfft(x0[ix], localmesh->LocalNz, std::begin(k1d)); - } else { - rfft(rhs[ix], localmesh->LocalNz, std::begin(k1d)); - } - - // Copy into array, transposing so kz is first index - for (int kz = 0; kz < nmode; kz++) { - bcmplx(kz, ix - xs) = k1d[kz]; - } - } - - // Get elements of the tridiagonal matrix - // including boundary conditions - BOUT_OMP_PERF(for nowait) - for (int kz = 0; kz < nmode; kz++) { - BoutReal kwave = kz * 2.0 * PI / zlength; // wave number is 1/[rad] - tridagMatrix(&a(kz, 0), &b(kz, 0), &c(kz, 0), &bcmplx(kz, 0), jy, - kz, // True for the component constant (DC) in Z - kwave, // Z wave number - &Acoef, &C1coef, &C2coef, &Dcoef, - false); // Don't include guard cells in arrays - } - } - - // Solve tridiagonal systems - cr->setCoefs(a, b, c); - cr->solve(bcmplx, xcmplx); + return transform.backward(rhs, xcmplx); + } - if (localmesh->periodicX) { - // Subtract X average of kz=0 mode - BoutReal local[2] = { - 0.0, // index 0 = sum of coefficients - static_cast(xe - xs + 1) // number of grid cells - }; - for (int ix = xs; ix <= xe; ix++) { - local[0] += xcmplx(0, ix - xs).real(); - } - BoutReal global[2]; - MPI_Allreduce(local, global, 2, MPI_DOUBLE, MPI_SUM, localmesh->getXcomm()); - BoutReal avg = global[0] / global[1]; - for (int ix = xs; ix <= xe; ix++) { - xcmplx(0, ix - xs) -= avg; - } + const FFTTransform transform( + *localmesh, nmode, xs, xe, 0, 0, localmesh->zstart, localmesh->zend, inbndry, + outbndry, isInnerBoundaryFlagSetOnFirstX(INVERT_SET), + isOuterBoundaryFlagSetOnLastX(INVERT_SET), isGlobalFlagSet(INVERT_ZERO_DC)); + + auto matrices = transform.forward(*this, rhs, x0, Acoef, C1coef, C2coef, Dcoef); + + // Solve tridiagonal systems + cr->setCoefs(matrices.a, matrices.b, matrices.c); + cr->solve(matrices.bcmplx, xcmplx); + + if (localmesh->periodicX) { + // Subtract X average of kz=0 mode + std::array local = { + 0.0, // index 0 = sum of coefficients + static_cast(xe - xs + 1) // number of grid cells + }; + for (int ix = xs; ix <= xe; ix++) { + local[0] += xcmplx(0, ix - xs).real(); } - - // FFT back to real space - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - // ZFFT routine expects input of this length - auto k1d = Array((localmesh->LocalNz) / 2 + 1); - - const bool zero_DC = isGlobalFlagSet(INVERT_ZERO_DC); - - BOUT_OMP_PERF(for nowait) - for (int ix = xs; ix <= xe; ix++) { - if (zero_DC) { - k1d[0] = 0.; - } - - for (int kz = static_cast(zero_DC); kz < nmode; kz++) { - k1d[kz] = xcmplx(kz, ix - xs); - } - - for (int kz = nmode; kz < (localmesh->LocalNz) / 2 + 1; kz++) { - k1d[kz] = 0.0; // Filtering out all higher harmonics - } - - irfft(std::begin(k1d), localmesh->LocalNz, x[ix]); - } + std::array global{}; + MPI_Allreduce(local.data(), global.data(), 2, MPI_DOUBLE, MPI_SUM, + localmesh->getXcomm()); + const BoutReal avg = global[0] / global[1]; + for (int ix = xs; ix <= xe; ix++) { + xcmplx(0, ix - xs) -= avg; } } - checkData(x); - - return x; + return transform.backward(rhs, xcmplx); } Field3D LaplaceCyclic::solve(const Field3D& rhs, const Field3D& x0) { - TRACE("LaplaceCyclic::solve(Field3D, Field3D)"); ASSERT1(rhs.getLocation() == location); ASSERT1(x0.getLocation() == location); @@ -315,7 +197,8 @@ Field3D LaplaceCyclic::solve(const Field3D& rhs, const Field3D& x0) { // Get the width of the boundary // If the flags to assign that only one guard cell should be used is set - int inbndry = localmesh->xstart, outbndry = localmesh->xstart; + int inbndry = localmesh->xstart; + int outbndry = localmesh->xstart; if (isGlobalFlagSet(INVERT_BOTH_BNDRY_ONE) || (localmesh->xstart < 2)) { inbndry = outbndry = 1; } @@ -329,7 +212,8 @@ Field3D LaplaceCyclic::solve(const Field3D& rhs, const Field3D& x0) { int nx = xe - xs + 1; // Number of X points on this processor // Get range of Y indices - int ys = localmesh->ystart, ye = localmesh->yend; + int ys = localmesh->ystart; + int ye = localmesh->yend; if (localmesh->hasBndryLowerY()) { if (include_yguards) { @@ -348,215 +232,61 @@ Field3D LaplaceCyclic::solve(const Field3D& rhs, const Field3D& x0) { const int ny = (ye - ys + 1); // Number of Y points const int nsys = nmode * ny; // Number of systems of equations to solve - const int nxny = nx * ny; // Number of points in X-Y // This is just to silence static analysis ASSERT0(ny > 0); - auto a3D = Matrix(nsys, nx); - auto b3D = Matrix(nsys, nx); - auto c3D = Matrix(nsys, nx); - auto xcmplx3D = Matrix(nsys, nx); - auto bcmplx3D = Matrix(nsys, nx); if (dst) { - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - // ZFFT routine expects input of this length - auto k1d = Array(localmesh->LocalNz); - - // Loop over X and Y indices, including boundaries but not guard cells. - // (unless periodic in x) - BOUT_OMP_PERF(for) - for (int ind = 0; ind < nxny; ++ind) { - // ind = (ix - xs)*(ye - ys + 1) + (iy - ys) - int ix = xs + ind / ny; - int iy = ys + ind % ny; - - // Take DST in Z direction and put result in k1d - - if (((ix < inbndry) && isInnerBoundaryFlagSetOnFirstX(INVERT_SET)) - || ((localmesh->LocalNx - ix - 1 < outbndry) - && isOuterBoundaryFlagSetOnLastX(INVERT_SET))) { - // Use the values in x0 in the boundary - DST(x0(ix, iy) + 1, localmesh->LocalNz - 2, std::begin(k1d)); - } else { - DST(rhs(ix, iy) + 1, localmesh->LocalNz - 2, std::begin(k1d)); - } - - // Copy into array, transposing so kz is first index - for (int kz = 0; kz < nmode; kz++) { - bcmplx3D((iy - ys) * nmode + kz, ix - xs) = k1d[kz]; - } - } + const DSTTransform transform( + *localmesh, nmode, xs, xe, ys, ye, localmesh->zstart, localmesh->zend, inbndry, + outbndry, isInnerBoundaryFlagSetOnFirstX(INVERT_SET), + isOuterBoundaryFlagSetOnLastX(INVERT_SET), isGlobalFlagSet(INVERT_ZERO_DC)); - // Get elements of the tridiagonal matrix - // including boundary conditions - const BoutReal zlen = getUniform(coords->dz) * (localmesh->LocalNz - 3); - BOUT_OMP_PERF(for nowait) - for (int ind = 0; ind < nsys; ind++) { - // ind = (iy - ys) * nmode + kz - int iy = ys + ind / nmode; - int kz = ind % nmode; - - // wave number is 1/[rad]; DST has extra 2. - BoutReal kwave = kz * 2.0 * PI / (2. * zlen); - - tridagMatrix(&a3D(ind, 0), &b3D(ind, 0), &c3D(ind, 0), &bcmplx3D(ind, 0), iy, - kz, // wave number index - kwave, // kwave (inverse wave length) - &Acoef, &C1coef, &C2coef, &Dcoef, - false, // Don't include guard cells in arrays - false); // Z domain not periodic - } - } + auto matrices = transform.forward(*this, rhs, x0, Acoef, C1coef, C2coef, Dcoef); // Solve tridiagonal systems - cr->setCoefs(a3D, b3D, c3D); - cr->solve(bcmplx3D, xcmplx3D); - - // FFT back to real space - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - // ZFFT routine expects input of length LocalNz - auto k1d = Array(localmesh->LocalNz); - - BOUT_OMP_PERF(for nowait) - for (int ind = 0; ind < nxny; ++ind) { // Loop over X and Y - // ind = (ix - xs)*(ye - ys + 1) + (iy - ys) - int ix = xs + ind / ny; - int iy = ys + ind % ny; - - for (int kz = 0; kz < nmode; kz++) { - k1d[kz] = xcmplx3D((iy - ys) * nmode + kz, ix - xs); - } - - for (int kz = nmode; kz < localmesh->LocalNz; kz++) { - k1d[kz] = 0.0; // Filtering out all higher harmonics - } - - DST_rev(std::begin(k1d), localmesh->LocalNz - 2, &x(ix, iy, 1)); - - x(ix, iy, 0) = -x(ix, iy, 2); - x(ix, iy, localmesh->LocalNz - 1) = -x(ix, iy, localmesh->LocalNz - 3); - } - } - } else { - const BoutReal zlength = getUniform(coords->zlength()); - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - // ZFFT routine expects input of this length - auto k1d = Array(localmesh->LocalNz / 2 + 1); - - // Loop over X and Y indices, including boundaries but not guard cells - // (unless periodic in x) - - BOUT_OMP_PERF(for) - for (int ind = 0; ind < nxny; ++ind) { - // ind = (ix - xs)*(ye - ys + 1) + (iy - ys) - int ix = xs + ind / ny; - int iy = ys + ind % ny; - - // Take FFT in Z direction, apply shift, and put result in k1d - - if (((ix < inbndry) && isInnerBoundaryFlagSetOnFirstX(INVERT_SET)) - || ((localmesh->LocalNx - ix - 1 < outbndry) - && isOuterBoundaryFlagSetOnLastX(INVERT_SET))) { - // Use the values in x0 in the boundary - rfft(x0(ix, iy), localmesh->LocalNz, std::begin(k1d)); - } else { - rfft(rhs(ix, iy), localmesh->LocalNz, std::begin(k1d)); - } - - // Copy into array, transposing so kz is first index - for (int kz = 0; kz < nmode; kz++) { - bcmplx3D((iy - ys) * nmode + kz, ix - xs) = k1d[kz]; - } - } + cr->setCoefs(matrices.a, matrices.b, matrices.c); + cr->solve(matrices.bcmplx, xcmplx3D); - // Get elements of the tridiagonal matrix - // including boundary conditions - BOUT_OMP_PERF(for nowait) - for (int ind = 0; ind < nsys; ind++) { - // ind = (iy - ys) * nmode + kz - int iy = ys + ind / nmode; - int kz = ind % nmode; - - BoutReal kwave = kz * 2.0 * PI / zlength; // wave number is 1/[rad] - tridagMatrix(&a3D(ind, 0), &b3D(ind, 0), &c3D(ind, 0), &bcmplx3D(ind, 0), iy, - kz, // True for the component constant (DC) in Z - kwave, // Z wave number - &Acoef, &C1coef, &C2coef, &Dcoef, - false); // Don't include guard cells in arrays - } - } - - // Solve tridiagonal systems - cr->setCoefs(a3D, b3D, c3D); - cr->solve(bcmplx3D, xcmplx3D); - - if (localmesh->periodicX) { - // Subtract X average of kz=0 mode - std::vector local(ny + 1, 0.0); - for (int y = 0; y < ny; y++) { - for (int ix = xs; ix <= xe; ix++) { - local[y] += xcmplx3D(y * nmode, ix - xs).real(); - } - } - local[ny] = static_cast(xe - xs + 1); - - // Global reduce - std::vector global(ny + 1, 0.0); - MPI_Allreduce(local.data(), global.data(), ny + 1, MPI_DOUBLE, MPI_SUM, - localmesh->getXcomm()); - // Subtract average from kz=0 modes - for (int y = 0; y < ny; y++) { - BoutReal avg = global[y] / global[ny]; - for (int ix = xs; ix <= xe; ix++) { - xcmplx3D(y * nmode, ix - xs) -= avg; - } - } - } - - // FFT back to real space - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array((localmesh->LocalNz) / 2 - + 1); // ZFFT routine expects input of this length - - const bool zero_DC = isGlobalFlagSet(INVERT_ZERO_DC); - - BOUT_OMP_PERF(for nowait) - for (int ind = 0; ind < nxny; ++ind) { // Loop over X and Y - // ind = (ix - xs)*(ye - ys + 1) + (iy - ys) - int ix = xs + ind / ny; - int iy = ys + ind % ny; - - if (zero_DC) { - k1d[0] = 0.; - } + return transform.backward(rhs, xcmplx3D); + } + const FFTTransform transform( + *localmesh, nmode, xs, xe, ys, ye, localmesh->zstart, localmesh->zend, inbndry, + outbndry, isInnerBoundaryFlagSetOnFirstX(INVERT_SET), + isOuterBoundaryFlagSetOnLastX(INVERT_SET), isGlobalFlagSet(INVERT_ZERO_DC)); - for (int kz = static_cast(zero_DC); kz < nmode; kz++) { - k1d[kz] = xcmplx3D((iy - ys) * nmode + kz, ix - xs); - } + auto matrices = transform.forward(*this, rhs, x0, Acoef, C1coef, C2coef, Dcoef); - for (int kz = nmode; kz < localmesh->LocalNz / 2 + 1; kz++) { - k1d[kz] = 0.0; // Filtering out all higher harmonics - } + // Solve tridiagonal systems + cr->setCoefs(matrices.a, matrices.b, matrices.c); + cr->solve(matrices.bcmplx, xcmplx3D); - irfft(std::begin(k1d), localmesh->LocalNz, x(ix, iy)); + if (localmesh->periodicX) { + // Subtract X average of kz=0 mode + std::vector local(ny + 1, 0.0); + for (int y = 0; y < ny; y++) { + for (int ix = xs; ix <= xe; ix++) { + local[y] += xcmplx3D(y * nmode, ix - xs).real(); + } + } + local[ny] = static_cast(xe - xs + 1); + + // Global reduce + std::vector global(ny + 1, 0.0); + MPI_Allreduce(local.data(), global.data(), ny + 1, MPI_DOUBLE, MPI_SUM, + localmesh->getXcomm()); + // Subtract average from kz=0 modes + for (int y = 0; y < ny; y++) { + BoutReal avg = global[y] / global[ny]; + for (int ix = xs; ix <= xe; ix++) { + xcmplx3D(y * nmode, ix - xs) -= avg; } } } - checkData(x); - - return x; + return transform.backward(rhs, xcmplx3D); } void LaplaceCyclic ::verify_solution(const Matrix& a_ver, @@ -591,21 +321,21 @@ void LaplaceCyclic ::verify_solution(const Matrix& a_ver, } if (xproc > 0) { - MPI_Irecv(&rbufdown[0], nsys, MPI_DOUBLE_COMPLEX, myrank - 1, 901, MPI_COMM_WORLD, + MPI_Irecv(&rbufdown[0], nsys, MPI_DOUBLE_COMPLEX, myrank - 1, 901, BoutComm::get(), &request[1]); for (int kz = 0; kz < nsys; kz++) { sbufdown[kz] = x_ver(kz, 1); } - MPI_Isend(&sbufdown[0], nsys, MPI_DOUBLE_COMPLEX, myrank - 1, 900, MPI_COMM_WORLD, + MPI_Isend(&sbufdown[0], nsys, MPI_DOUBLE_COMPLEX, myrank - 1, 900, BoutComm::get(), &request[0]); } if (xproc < nprocs - 1) { - MPI_Irecv(&rbufup[0], nsys, MPI_DOUBLE_COMPLEX, myrank + 1, 900, MPI_COMM_WORLD, + MPI_Irecv(&rbufup[0], nsys, MPI_DOUBLE_COMPLEX, myrank + 1, 900, BoutComm::get(), &request[3]); for (int kz = 0; kz < nsys; kz++) { sbufup[kz] = x_ver(kz, nx); } - MPI_Isend(&sbufup[0], nsys, MPI_DOUBLE_COMPLEX, myrank + 1, 901, MPI_COMM_WORLD, + MPI_Isend(&sbufup[0], nsys, MPI_DOUBLE_COMPLEX, myrank + 1, 901, BoutComm::get(), &request[2]); } diff --git a/src/invert/laplace/impls/iterative_parallel_tri/iterative_parallel_tri.cxx b/src/invert/laplace/impls/iterative_parallel_tri/iterative_parallel_tri.cxx index 5255ea939d..e9219091e2 100644 --- a/src/invert/laplace/impls/iterative_parallel_tri/iterative_parallel_tri.cxx +++ b/src/invert/laplace/impls/iterative_parallel_tri/iterative_parallel_tri.cxx @@ -67,12 +67,14 @@ LaplaceIPT::LaplaceIPT(Options* opt, CELL_LOC loc, Mesh* mesh_in, Solver* UNUSED au(ny, nmode), bu(ny, nmode), rl(nmode), ru(nmode), r1(ny, nmode), r2(ny, nmode), first_call(ny), x0saved(ny, 4, nmode), converged(nmode), fine_error(4, nmode) { + bout::fft::assertZSerial(*localmesh, "`ipt` inversion"); + A.setLocation(location); C.setLocation(location); D.setLocation(location); // Number of procs must be a factor of 2 - const int n = localmesh->NXPE; + const int n = localmesh->getNXPE(); if (!is_pow2(n)) { throw BoutException("LaplaceIPT error: NXPE must be a power of 2"); } @@ -234,7 +236,6 @@ FieldPerp LaplaceIPT::solve(const FieldPerp& b, const FieldPerp& x0) { ASSERT1(localmesh == b.getMesh() && localmesh == x0.getMesh()); ASSERT1(b.getLocation() == location); ASSERT1(x0.getLocation() == location); - TRACE("LaplaceIPT::solve(const, const)"); FieldPerp x{emptyFrom(b)}; diff --git a/src/invert/laplace/impls/multigrid/multigrid_laplace.cxx b/src/invert/laplace/impls/multigrid/multigrid_laplace.cxx index 04c8c7e0c9..c06a83585b 100644 --- a/src/invert/laplace/impls/multigrid/multigrid_laplace.cxx +++ b/src/invert/laplace/impls/multigrid/multigrid_laplace.cxx @@ -1,5 +1,5 @@ /************************************************************************** - * Perpendicular Laplacian inversion. + * Perpendicular Laplacian inversion. * Using Geometrical Multigrid Solver * * Equation solved is: @@ -9,7 +9,7 @@ * Copyright 2016 K.S. Kang * * Contact: Ben Dudson, bd512@york.ac.uk - * + * * This file is part of BOUT++. * * BOUT++ is free software: you can redistribute it and/or modify @@ -32,9 +32,11 @@ #if not BOUT_USE_METRIC_3D +#include +#include #include -#include #include +#include #if BOUT_USE_OPENMP #include @@ -46,8 +48,6 @@ LaplaceMultigrid::LaplaceMultigrid(Options* opt, const CELL_LOC loc, Mesh* mesh_ Solver* UNUSED(solver)) : Laplacian(opt, loc, mesh_in), A(0.0), C1(1.0), C2(1.0), D(1.0) { - TRACE("LaplaceMultigrid::LaplaceMultigrid(Options *opt)"); - // periodic x-direction not handled: see MultigridAlg::communications ASSERT1(!localmesh->periodicX); @@ -110,11 +110,7 @@ LaplaceMultigrid::LaplaceMultigrid(Options* opt, const CELL_LOC loc, Mesh* mesh_ } Nz_global = localmesh->GlobalNz; Nz_local = Nz_global; // No parallelization in z-direction (for now) - // - //else { - // Nz_local = localmesh->zend - localmesh->zstart + 1; // excluding guard cells - // Nz_global = localmesh->GlobalNz - 2*localmesh->zstart; // excluding guard cells - // } + if (mgcount == 0) { output << "Nz=" << Nz_global << "(" << Nz_local << ")" << endl; } @@ -217,14 +213,14 @@ LaplaceMultigrid::LaplaceMultigrid(Options* opt, const CELL_LOC loc, Mesh* mesh_ } BOUT_OMP_SAFE(parallel) BOUT_OMP_SAFE(master) - { output << "Num threads = " << omp_get_num_threads() << endl; } + { + output << "Num threads = " << omp_get_num_threads() << endl; + } } } FieldPerp LaplaceMultigrid::solve(const FieldPerp& b_in, const FieldPerp& x0) { - TRACE("LaplaceMultigrid::solve(const FieldPerp, const FieldPerp)"); - ASSERT1(localmesh == b_in.getMesh() && localmesh == x0.getMesh()); ASSERT1(b_in.getLocation() == location); ASSERT1(x0.getLocation() == location); @@ -389,11 +385,9 @@ FieldPerp LaplaceMultigrid::solve(const FieldPerp& b_in, const FieldPerp& x0) { } if ((pcheck == 3) && (mgcount == 0)) { - FILE* outf; - char outfile[256]; - sprintf(outfile, "test_matF_%d.mat", kMG->rProcI); + std::string outfile = fmt::format("test_matF_{:d}.mat", kMG->rProcI); output << "Out file= " << outfile << endl; - outf = fopen(outfile, "w"); + FILE* outf = fopen(outfile.c_str(), "w"); int dim = (lxx + 2) * (lzz + 2); fprintf(outf, "dim = %d (%d, %d)\n", dim, lxx, lzz); @@ -416,11 +410,9 @@ FieldPerp LaplaceMultigrid::solve(const FieldPerp& b_in, const FieldPerp& x0) { output << i << "dimension= " << kMG->lnx[i - 1] << "(" << kMG->gnx[i - 1] << ")," << kMG->lnz[i - 1] << endl; - FILE* outf; - char outfile[256]; - sprintf(outfile, "test_matC%1d_%d.mat", i, kMG->rProcI); + std::string outfile = fmt::format("test_matC{:1d}_{:d}.mat", i, kMG->rProcI); output << "Out file= " << outfile << endl; - outf = fopen(outfile, "w"); + FILE* outf = fopen(outfile.c_str(), "w"); int dim = (kMG->lnx[i - 1] + 2) * (kMG->lnz[i - 1] + 2); fprintf(outf, "dim = %d (%d,%d)\n", dim, kMG->lnx[i - 1], kMG->lnz[i - 1]); @@ -578,7 +570,6 @@ FieldPerp LaplaceMultigrid::solve(const FieldPerp& b_in, const FieldPerp& x0) { } void LaplaceMultigrid::generateMatrixF(int level) { - TRACE("LaplaceMultigrid::generateMatrixF(int)"); // Set (fine-level) matrix entries diff --git a/src/invert/laplace/impls/multigrid/multigrid_solver.cxx b/src/invert/laplace/impls/multigrid/multigrid_solver.cxx index 5ff616aa54..074e482320 100644 --- a/src/invert/laplace/impls/multigrid/multigrid_solver.cxx +++ b/src/invert/laplace/impls/multigrid/multigrid_solver.cxx @@ -1,5 +1,5 @@ /************************************************************************** - * Perpendicular Laplacian inversion. + * Perpendicular Laplacian inversion. * Using Geometrical Multigrid Solver * * Equation solved is: @@ -9,7 +9,7 @@ * Copyright 2015 K.S. Kang * * Contact: Ben Dudson, bd512@york.ac.uk - * + * * This file is part of BOUT++. * * BOUT++ is free software: you can redistribute it and/or modify @@ -35,6 +35,9 @@ #include "bout/unused.hxx" #include +#include +#include + Multigrid1DP::Multigrid1DP(int level, int lx, int lz, int gx, int dl, int merge, MPI_Comm comm, int check) : MultigridAlg(level, lx, lz, gx, lz, comm, check) { @@ -195,11 +198,9 @@ void Multigrid1DP::setMultigridC(int UNUSED(plag)) { if (pcheck == 2) { for (int i = level; i >= 0; i--) { - FILE* outf; - char outfile[256]; - sprintf(outfile, "2DP_matC%1d_%d.mat", i, rMG->rProcI); + std::string outfile = fmt::format("2DP_matC{:1d}_{:d}.mat", i, rMG->rProcI); output << "Out file= " << outfile << endl; - outf = fopen(outfile, "w"); + FILE* outf = fopen(outfile.c_str(), "w"); int dim = (rMG->lnx[i] + 2) * (rMG->lnz[i] + 2); fprintf(outf, "dim = %d (%d, %d)\n", dim, rMG->lnx[i], rMG->lnz[i]); @@ -222,11 +223,9 @@ void Multigrid1DP::setMultigridC(int UNUSED(plag)) { } if (pcheck == 3) { for (int i = level; i >= 0; i--) { - FILE* outf; - char outfile[256]; - sprintf(outfile, "S1D_matC%1d_%d.mat", i, sMG->rProcI); + std::string outfile = fmt::format("S1D_matC{:1d}_{:d}.mat", i, sMG->rProcI); output << "Out file= " << outfile << endl; - outf = fopen(outfile, "w"); + FILE* outf = fopen(outfile.c_str(), "w"); int dim = (sMG->lnx[i] + 2) * (sMG->lnz[i] + 2); fprintf(outf, "dim = %d\n", dim); @@ -456,11 +455,9 @@ void Multigrid1DP::convertMatrixF2D(int level) { } } if (pcheck == 3) { - FILE* outf; - char outfile[256]; - sprintf(outfile, "2DP_CP_%d.mat", rProcI); + std::string outfile = fmt::format("2DP_CP_{}.mat", rProcI); output << "Out file= " << outfile << endl; - outf = fopen(outfile, "w"); + FILE* outf = fopen(outfile.c_str(), "w"); fprintf(outf, "dim = (%d, %d)\n", ggx, gnz[0]); for (int ii = 0; ii < dim; ii++) { @@ -476,11 +473,10 @@ void Multigrid1DP::convertMatrixF2D(int level) { MPI_SUM, comm2D); if (pcheck == 3) { - FILE* outf; - char outfile[256]; - sprintf(outfile, "2DP_Conv_%d.mat", rProcI); + + std::string outfile = fmt::format("2DP_Conv_{}.mat", rProcI); output << "Out file= " << outfile << endl; - outf = fopen(outfile, "w"); + FILE* outf = fopen(outfile.c_str(), "w"); fprintf(outf, "dim = (%d, %d)\n", ggx, gnz[0]); for (int ii = 0; ii < dim; ii++) { @@ -625,10 +621,8 @@ void Multigrid2DPf1D::setMultigridC(int UNUSED(plag)) { } if (pcheck == 2) { for (int i = level; i >= 0; i--) { - FILE* outf; - char outfile[256]; - sprintf(outfile, "S2D_matC%1d_%d.mat", i, sMG->rProcI); - outf = fopen(outfile, "w"); + std::string outfile = fmt::format("S2D_matC{:1d}_{:d}.mat", i, sMG->rProcI); + FILE* outf = fopen(outfile.c_str(), "w"); output << "Out file= " << outfile << endl; int dim = (sMG->lnx[i] + 2) * (sMG->lnz[i] + 2); fprintf(outf, "dim = %d\n", dim); diff --git a/src/invert/laplace/impls/naulin/naulin_laplace.cxx b/src/invert/laplace/impls/naulin/naulin_laplace.cxx index e6f68d850d..bf8711245e 100644 --- a/src/invert/laplace/impls/naulin/naulin_laplace.cxx +++ b/src/invert/laplace/impls/naulin/naulin_laplace.cxx @@ -138,11 +138,16 @@ */ // clang-format on +#include "bout/build_defines.hxx" + +#if not BOUT_USE_METRIC_3D + #include #include #include #include #include +#include #include #include @@ -203,7 +208,7 @@ LaplaceNaulin::LaplaceNaulin(Options* opt, const CELL_LOC loc, Mesh* mesh_in, ASSERT0(underrelax_recovery >= 1.); delp2solver = create(opt->getSection("delp2solver"), location, localmesh); std::string delp2type; - opt->getSection("delp2solver")->get("type", delp2type, "cyclic"); + opt->getSection("delp2solver")->get("type", delp2type, LaplaceFactory::default_type); // Check delp2solver is using an FFT scheme, otherwise it will not exactly // invert Delp2 and we will not converge ASSERT0(delp2type == "cyclic" || delp2type == "spt" || delp2type == "tri"); @@ -402,7 +407,7 @@ void LaplaceNaulin::copy_x_boundaries(Field3D& x, const Field3D& x0, Mesh* local if (localmesh->firstX()) { for (int i = localmesh->xstart - 1; i >= 0; --i) { for (int j = localmesh->ystart; j <= localmesh->yend; ++j) { - for (int k = 0; k < localmesh->LocalNz; ++k) { + for (int k = localmesh->zstart; k <= localmesh->zend; ++k) { x(i, j, k) = x0(i, j, k); } } @@ -411,7 +416,7 @@ void LaplaceNaulin::copy_x_boundaries(Field3D& x, const Field3D& x0, Mesh* local if (localmesh->lastX()) { for (int i = localmesh->xend + 1; i < localmesh->LocalNx; ++i) { for (int j = localmesh->ystart; j <= localmesh->yend; ++j) { - for (int k = 0; k < localmesh->LocalNz; ++k) { + for (int k = localmesh->zstart; k <= localmesh->zend; ++k) { x(i, j, k) = x0(i, j, k); } } @@ -426,3 +431,5 @@ void LaplaceNaulin::outputVars(Options& output_options, output_options[fmt::format("{}_mean_underrelax_counts", getPerformanceName())] .assignRepeat(naulinsolver_mean_underrelax_counts, time_dimension); } + +#endif diff --git a/src/invert/laplace/impls/naulin/naulin_laplace.hxx b/src/invert/laplace/impls/naulin/naulin_laplace.hxx index 1998a046d7..c4e7d41d9e 100644 --- a/src/invert/laplace/impls/naulin/naulin_laplace.hxx +++ b/src/invert/laplace/impls/naulin/naulin_laplace.hxx @@ -28,11 +28,22 @@ class LaplaceNaulin; #ifndef BOUT_LAP_NAULIN_H #define BOUT_LAP_NAULIN_H +#include #include + +#if BOUT_USE_METRIC_3D + +namespace { +const RegisterUnavailableLaplace + registerlaplacenaulin(LAPLACE_NAULIN, "BOUT++ was configured with 3D metrics"); +} + +#else + #include namespace { -RegisterLaplace registerlaplacenaulin(LAPLACE_NAULIN); +const RegisterLaplace registerlaplacenaulin(LAPLACE_NAULIN); } /// Solves the 2D Laplacian equation @@ -198,4 +209,6 @@ private: void copy_x_boundaries(Field3D& x, const Field3D& x0, Mesh* mesh); }; +#endif // BOUT_USE_METRIC_3D + #endif // BOUT_LAP_NAULIN_H diff --git a/src/invert/laplace/impls/pcr/pcr.cxx b/src/invert/laplace/impls/pcr/pcr.cxx index 48bbdbac4b..325242a3b2 100644 --- a/src/invert/laplace/impls/pcr/pcr.cxx +++ b/src/invert/laplace/impls/pcr/pcr.cxx @@ -36,23 +36,31 @@ * **************************************************************************/ +#include "bout/build_defines.hxx" + +#if not BOUT_USE_METRIC_3D + #include "pcr.hxx" -#include "bout/globals.hxx" +#include "../../common_transform.hxx" + +#include +#include +#include +#include #include #include +#include #include +#include +#include #include #include #include -#include -#include -#include - -#include "bout/boutcomm.hxx" #include - #include +#include +#include #include #include @@ -67,6 +75,8 @@ LaplacePCR::LaplacePCR(Options* opt, CELL_LOC loc, Mesh* mesh_in, Solver* UNUSED ncx(localmesh->LocalNx), ny(localmesh->LocalNy), avec(ny, nmode, ncx), bvec(ny, nmode, ncx), cvec(ny, nmode, ncx) { + bout::fft::assertZSerial(*localmesh, "`pcr` inversion"); + Acoef.setLocation(location); C1coef.setLocation(location); C2coef.setLocation(location); @@ -117,8 +127,8 @@ LaplacePCR::LaplacePCR(Options* opt, CELL_LOC loc, Mesh* mesh_in, Solver* UNUSED // (unless periodic in x) xe = localmesh->LocalNx - 1; } - int n = xe - xs + 1; // Number of X points on this processor, - // including boundaries but not guard cells + const int n = xe - xs + 1; // Number of X points on this processor, + // including boundaries but not guard cells a.reallocate(nmode, n); b.reallocate(nmode, n); @@ -141,7 +151,7 @@ FieldPerp LaplacePCR::solve(const FieldPerp& rhs, const FieldPerp& x0) { FieldPerp x{emptyFrom(rhs)}; // Result - int jy = rhs.getIndex(); // Get the Y index + const int jy = rhs.getIndex(); // Get the Y index x.setIndex(jy); // Get the width of the boundary @@ -160,155 +170,32 @@ FieldPerp LaplacePCR::solve(const FieldPerp& rhs, const FieldPerp& x0) { } if (dst) { - const BoutReal zlen = getUniform(coords->dz) * (localmesh->LocalNz - 3); - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array( - localmesh->LocalNz); // ZFFT routine expects input of this length - - // Loop over X indices, including boundaries but not guard cells. (unless periodic - // in x) - BOUT_OMP_PERF(for) - for (int ix = xs; ix <= xe; ix++) { - // Take DST in Z direction and put result in k1d - - if (((ix < inbndry) && isInnerBoundaryFlagSetOnFirstX(INVERT_SET)) - || ((localmesh->LocalNx - ix - 1 < outbndry) - && isOuterBoundaryFlagSetOnLastX(INVERT_SET))) { - // Use the values in x0 in the boundary - DST(x0[ix] + 1, localmesh->LocalNz - 2, std::begin(k1d)); - } else { - DST(rhs[ix] + 1, localmesh->LocalNz - 2, std::begin(k1d)); - } - - // Copy into array, transposing so kz is first index - for (int kz = 0; kz < nmode; kz++) { - bcmplx(kz, ix - xs) = k1d[kz]; - } - } + const DSTTransform transform( + *localmesh, nmode, xs, xe, 0, 0, localmesh->zstart, localmesh->zend, inbndry, + outbndry, isInnerBoundaryFlagSetOnFirstX(INVERT_SET), + isOuterBoundaryFlagSetOnLastX(INVERT_SET), isGlobalFlagSet(INVERT_ZERO_DC)); - // Get elements of the tridiagonal matrix - // including boundary conditions - BOUT_OMP_PERF(for nowait) - for (int kz = 0; kz < nmode; kz++) { - BoutReal kwave = - kz * 2.0 * PI / (2. * zlen); // wave number is 1/[rad]; DST has extra 2. - - tridagMatrix(&a(kz, 0), &b(kz, 0), &c(kz, 0), &bcmplx(kz, 0), jy, - kz, // wave number index - kwave, // kwave (inverse wave length) - &Acoef, &C1coef, &C2coef, &Dcoef, - false); // Don't include guard cells in arrays - } - } // BOUT_OMP_PERF(parallel) + auto matrices = transform.forward(*this, rhs, x0, Acoef, C1coef, C2coef, Dcoef); // Solve tridiagonal systems - cr_pcr_solver(a, b, c, bcmplx, xcmplx); - - // FFT back to real space - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array( - localmesh->LocalNz); // ZFFT routine expects input of this length - - BOUT_OMP_PERF(for nowait) - for (int ix = xs; ix <= xe; ix++) { - for (int kz = 0; kz < nmode; kz++) { - k1d[kz] = xcmplx(kz, ix - xs); - } + cr_pcr_solver(matrices.a, matrices.b, matrices.c, matrices.bcmplx, xcmplx); - for (int kz = nmode; kz < (localmesh->LocalNz); kz++) { - k1d[kz] = 0.0; // Filtering out all higher harmonics - } - - DST_rev(std::begin(k1d), localmesh->LocalNz - 2, x[ix] + 1); - - x(ix, 0) = -x(ix, 2); - x(ix, localmesh->LocalNz - 1) = -x(ix, localmesh->LocalNz - 3); - } - } - } else { - const BoutReal zlength = getUniform(coords->zlength()); - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array((localmesh->LocalNz) / 2 - + 1); // ZFFT routine expects input of this length - - // Loop over X indices, including boundaries but not guard cells (unless periodic in - // x) - BOUT_OMP_PERF(for) - for (int ix = xs; ix <= xe; ix++) { - // Take FFT in Z direction, apply shift, and put result in k1d - - if (((ix < inbndry) && isInnerBoundaryFlagSetOnFirstX(INVERT_SET)) - || ((localmesh->LocalNx - ix - 1 < outbndry) - && isOuterBoundaryFlagSetOnLastX(INVERT_SET))) { - // Use the values in x0 in the boundary - rfft(x0[ix], localmesh->LocalNz, std::begin(k1d)); - } else { - rfft(rhs[ix], localmesh->LocalNz, std::begin(k1d)); - } - - // Copy into array, transposing so kz is first index - for (int kz = 0; kz < nmode; kz++) { - bcmplx(kz, ix - xs) = k1d[kz]; - } - } - - // Get elements of the tridiagonal matrix - // including boundary conditions - BOUT_OMP_PERF(for nowait) - for (int kz = 0; kz < nmode; kz++) { - BoutReal kwave = kz * 2.0 * PI / zlength; // wave number is 1/[rad] - tridagMatrix(&a(kz, 0), &b(kz, 0), &c(kz, 0), &bcmplx(kz, 0), jy, - kz, // True for the component constant (DC) in Z - kwave, // Z wave number - &Acoef, &C1coef, &C2coef, &Dcoef, - false); // Don't include guard cells in arrays - } - } // BOUT_OMP_PERF(parallel) - - // Solve tridiagonal systems - cr_pcr_solver(a, b, c, bcmplx, xcmplx); - - // FFT back to real space - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array((localmesh->LocalNz) / 2 - + 1); // ZFFT routine expects input of this length - - const bool zero_DC = isGlobalFlagSet(INVERT_ZERO_DC); - - BOUT_OMP_PERF(for nowait) - for (int ix = xs; ix <= xe; ix++) { - if (zero_DC) { - k1d[0] = 0.; - } - - for (int kz = static_cast(zero_DC); kz < nmode; kz++) { - k1d[kz] = xcmplx(kz, ix - xs); - } - - for (int kz = nmode; kz < (localmesh->LocalNz) / 2 + 1; kz++) { - k1d[kz] = 0.0; // Filtering out all higher harmonics - } - - irfft(std::begin(k1d), localmesh->LocalNz, x[ix]); - } - } + return transform.backward(rhs, xcmplx); } + const FFTTransform transform( + *localmesh, nmode, xs, xe, 0, 0, localmesh->zstart, localmesh->zend, inbndry, + outbndry, isInnerBoundaryFlagSetOnFirstX(INVERT_SET), + isOuterBoundaryFlagSetOnLastX(INVERT_SET), isGlobalFlagSet(INVERT_ZERO_DC)); + + auto matrices = transform.forward(*this, rhs, x0, Acoef, C1coef, C2coef, Dcoef); - checkData(x); + // Solve tridiagonal systems + cr_pcr_solver(matrices.a, matrices.b, matrices.c, matrices.bcmplx, xcmplx); - return x; + return transform.backward(rhs, xcmplx); } Field3D LaplacePCR::solve(const Field3D& rhs, const Field3D& x0) { - TRACE("LaplacePCR::solve(Field3D, Field3D)"); ASSERT1(rhs.getLocation() == location); ASSERT1(x0.getLocation() == location); @@ -316,8 +203,6 @@ Field3D LaplacePCR::solve(const Field3D& rhs, const Field3D& x0) { Timer timer("invert"); - Field3D x{emptyFrom(rhs)}; // Result - // Get the width of the boundary // If the flags to assign that only one guard cell should be used is set @@ -333,7 +218,7 @@ Field3D LaplacePCR::solve(const Field3D& rhs, const Field3D& x0) { outbndry = 1; } - int nx = xe - xs + 1; // Number of X points on this processor + const int nx = xe - xs + 1; // Number of X points on this processor // Get range of Y indices int ys = localmesh->ystart; @@ -356,185 +241,33 @@ Field3D LaplacePCR::solve(const Field3D& rhs, const Field3D& x0) { const int ny = (ye - ys + 1); // Number of Y points nsys = nmode * ny; // Number of systems of equations to solve - const int nxny = nx * ny; // Number of points in X-Y - - auto a3D = Matrix(nsys, nx); - auto b3D = Matrix(nsys, nx); - auto c3D = Matrix(nsys, nx); auto xcmplx3D = Matrix(nsys, nx); - auto bcmplx3D = Matrix(nsys, nx); if (dst) { - const BoutReal zlen = getUniform(coords->dz) * (localmesh->LocalNz - 3); - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array( - localmesh->LocalNz); // ZFFT routine expects input of this length - - // Loop over X and Y indices, including boundaries but not guard cells. - // (unless periodic in x) - BOUT_OMP_PERF(for) - for (int ind = 0; ind < nxny; ++ind) { - // ind = (ix - xs)*(ye - ys + 1) + (iy - ys) - int ix = xs + ind / ny; - int iy = ys + ind % ny; - - // Take DST in Z direction and put result in k1d - - if (((ix < inbndry) && isInnerBoundaryFlagSetOnFirstX(INVERT_SET)) - || ((localmesh->LocalNx - ix - 1 < outbndry) - && isOuterBoundaryFlagSetOnLastX(INVERT_SET))) { - // Use the values in x0 in the boundary - DST(x0(ix, iy) + 1, localmesh->LocalNz - 2, std::begin(k1d)); - } else { - DST(rhs(ix, iy) + 1, localmesh->LocalNz - 2, std::begin(k1d)); - } + const DSTTransform transform( + *localmesh, nmode, xs, xe, ys, ye, localmesh->zstart, localmesh->zend, inbndry, + outbndry, isInnerBoundaryFlagSetOnFirstX(INVERT_SET), + isOuterBoundaryFlagSetOnLastX(INVERT_SET), isGlobalFlagSet(INVERT_ZERO_DC)); - // Copy into array, transposing so kz is first index - for (int kz = 0; kz < nmode; kz++) { - bcmplx3D((iy - ys) * nmode + kz, ix - xs) = k1d[kz]; - } - } - - // Get elements of the tridiagonal matrix - // including boundary conditions - BOUT_OMP_PERF(for nowait) - for (int ind = 0; ind < nsys; ind++) { - // ind = (iy - ys) * nmode + kz - int iy = ys + ind / nmode; - int kz = ind % nmode; - - BoutReal kwave = - kz * 2.0 * PI / (2. * zlen); // wave number is 1/[rad]; DST has extra 2. - - tridagMatrix(&a3D(ind, 0), &b3D(ind, 0), &c3D(ind, 0), &bcmplx3D(ind, 0), iy, - kz, // wave number index - kwave, // kwave (inverse wave length) - &Acoef, &C1coef, &C2coef, &Dcoef, - false); // Don't include guard cells in arrays - } - } // BOUT_OMP_PERF(parallel) + auto matrices = transform.forward(*this, rhs, x0, Acoef, C1coef, C2coef, Dcoef); // Solve tridiagonal systems - cr_pcr_solver(a3D, b3D, c3D, bcmplx3D, xcmplx3D); - - // FFT back to real space - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array( - localmesh->LocalNz); // ZFFT routine expects input of this length - - BOUT_OMP_PERF(for nowait) - for (int ind = 0; ind < nxny; ++ind) { // Loop over X and Y - // ind = (ix - xs)*(ye - ys + 1) + (iy - ys) - int ix = xs + ind / ny; - int iy = ys + ind % ny; - - for (int kz = 0; kz < nmode; kz++) { - k1d[kz] = xcmplx3D((iy - ys) * nmode + kz, ix - xs); - } - - for (int kz = nmode; kz < localmesh->LocalNz; kz++) { - k1d[kz] = 0.0; // Filtering out all higher harmonics - } - - DST_rev(std::begin(k1d), localmesh->LocalNz - 2, &x(ix, iy, 1)); + cr_pcr_solver(matrices.a, matrices.b, matrices.c, matrices.bcmplx, xcmplx3D); - x(ix, iy, 0) = -x(ix, iy, 2); - x(ix, iy, localmesh->LocalNz - 1) = -x(ix, iy, localmesh->LocalNz - 3); - } - } - } else { - const BoutReal zlength = getUniform(coords->zlength()); - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array(localmesh->LocalNz / 2 - + 1); // ZFFT routine expects input of this length - - // Loop over X and Y indices, including boundaries but not guard cells - // (unless periodic in x) - - BOUT_OMP_PERF(for) - for (int ind = 0; ind < nxny; ++ind) { - // ind = (ix - xs)*(ye - ys + 1) + (iy - ys) - int ix = xs + ind / ny; - int iy = ys + ind % ny; - - // Take FFT in Z direction, apply shift, and put result in k1d - - if (((ix < inbndry) && isInnerBoundaryFlagSetOnFirstX(INVERT_SET)) - || ((localmesh->LocalNx - ix - 1 < outbndry) - && isOuterBoundaryFlagSetOnLastX(INVERT_SET))) { - // Use the values in x0 in the boundary - rfft(x0(ix, iy), localmesh->LocalNz, std::begin(k1d)); - } else { - rfft(rhs(ix, iy), localmesh->LocalNz, std::begin(k1d)); - } - - // Copy into array, transposing so kz is first index - for (int kz = 0; kz < nmode; kz++) { - bcmplx3D((iy - ys) * nmode + kz, ix - xs) = k1d[kz]; - } - } - - // Get elements of the tridiagonal matrix - // including boundary conditions - BOUT_OMP_PERF(for nowait) - for (int ind = 0; ind < nsys; ind++) { - // ind = (iy - ys) * nmode + kz - int iy = ys + ind / nmode; - int kz = ind % nmode; - - BoutReal kwave = kz * 2.0 * PI / zlength; // wave number is 1/[rad] - tridagMatrix(&a3D(ind, 0), &b3D(ind, 0), &c3D(ind, 0), &bcmplx3D(ind, 0), iy, - kz, // True for the component constant (DC) in Z - kwave, // Z wave number - &Acoef, &C1coef, &C2coef, &Dcoef, - false); // Don't include guard cells in arrays - } - } // BOUT_OMP_PERF(parallel) - - // Solve tridiagonal systems - cr_pcr_solver(a3D, b3D, c3D, bcmplx3D, xcmplx3D); - - // FFT back to real space - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array((localmesh->LocalNz) / 2 - + 1); // ZFFT routine expects input of this length - - const bool zero_DC = isGlobalFlagSet(INVERT_ZERO_DC); - - BOUT_OMP_PERF(for nowait) - for (int ind = 0; ind < nxny; ++ind) { // Loop over X and Y - int ix = xs + ind / ny; - int iy = ys + ind % ny; - - if (zero_DC) { - k1d[0] = 0.; - } - - for (int kz = static_cast(zero_DC); kz < nmode; kz++) { - k1d[kz] = xcmplx3D((iy - ys) * nmode + kz, ix - xs); - } - - for (int kz = nmode; kz < localmesh->LocalNz / 2 + 1; kz++) { - k1d[kz] = 0.0; // Filtering out all higher harmonics - } - - irfft(std::begin(k1d), localmesh->LocalNz, x(ix, iy)); - } - } + return transform.backward(rhs, xcmplx3D); } + const FFTTransform transform( + *localmesh, nmode, xs, xe, ys, ye, localmesh->zstart, localmesh->zend, inbndry, + outbndry, isInnerBoundaryFlagSetOnFirstX(INVERT_SET), + isOuterBoundaryFlagSetOnLastX(INVERT_SET), isGlobalFlagSet(INVERT_ZERO_DC)); + + auto matrices = transform.forward(*this, rhs, x0, Acoef, C1coef, C2coef, Dcoef); - checkData(x); + // Solve tridiagonal systems + cr_pcr_solver(matrices.a, matrices.b, matrices.c, matrices.bcmplx, xcmplx3D); - return x; + return transform.backward(rhs, xcmplx3D); } /** @@ -555,6 +288,8 @@ void LaplacePCR ::cr_pcr_solver(Matrix& a_mpi, Matrix& b_mpi const int xend = localmesh->xend; const int nx = xend - xstart + 1; // number of interior points + const int nsys = std::get<0>(a_mpi.shape()); + // Handle boundary points so that the PCR algorithm works with arrays of // the same size on each rank. // Note that this modifies the coefficients of b and r in the first and last @@ -581,7 +316,7 @@ void LaplacePCR ::cr_pcr_solver(Matrix& a_mpi, Matrix& b_mpi // don't want to copy them. // xs = xstart if a proc has no boundary points // xs = 0 if a proc has boundary points - int offset = localmesh->xstart - xs; + const int offset = localmesh->xstart - xs; aa(kz, ix + 1) = a_mpi(kz, ix + offset); bb(kz, ix + 1) = b_mpi(kz, ix + offset); cc(kz, ix + 1) = c_mpi(kz, ix + offset); @@ -621,6 +356,8 @@ void LaplacePCR ::cr_pcr_solver(Matrix& a_mpi, Matrix& b_mpi void LaplacePCR ::eliminate_boundary_rows(Matrix& a, Matrix& b, Matrix& c, Matrix& r) { + const int nsys = std::get<0>(a.shape()); + if (localmesh->firstX()) { for (int kz = 0; kz < nsys; kz++) { // Do forward elimination on *all* boundary rows up to xstart @@ -656,6 +393,8 @@ void LaplacePCR ::apply_boundary_conditions(const Matrix& a, const Matrix& r, Matrix& x) { + const int nsys = std::get<0>(a.shape()); + if (localmesh->firstX()) { for (int kz = 0; kz < nsys; kz++) { for (int ix = localmesh->xstart - 1; ix >= 0; ix--) { @@ -664,7 +403,7 @@ void LaplacePCR ::apply_boundary_conditions(const Matrix& a, } } if (localmesh->lastX()) { - int n = xe - xs + 1; // actual length of array + const int n = xe - xs + 1; // actual length of array for (int kz = 0; kz < nsys; kz++) { for (int ix = n - localmesh->xstart; ix < n; ix++) { x(kz, ix) = (r(kz, ix) - a(kz, ix) * x(kz, ix - 1)) / b(kz, ix); @@ -681,7 +420,9 @@ void LaplacePCR ::apply_boundary_conditions(const Matrix& a, void LaplacePCR ::cr_forward_multiple_row(Matrix& a, Matrix& b, Matrix& c, Matrix& r) const { - MPI_Comm comm = BoutComm::get(); + const int nsys = std::get<0>(a.shape()); + + const MPI_Comm comm = BoutComm::get(); Array alpha(nsys); Array gamma(nsys); Array sbuf(4 * nsys); @@ -754,7 +495,9 @@ void LaplacePCR ::cr_forward_multiple_row(Matrix& a, Matrix& void LaplacePCR ::cr_backward_multiple_row(Matrix& a, Matrix& b, Matrix& c, Matrix& r, Matrix& x) const { - MPI_Comm comm = BoutComm::get(); + const int nsys = std::get<0>(a.shape()); + + const MPI_Comm comm = BoutComm::get(); MPI_Status status; MPI_Request request[2]; @@ -793,7 +536,7 @@ void LaplacePCR ::cr_backward_multiple_row(Matrix& a, Matrix dist_row = dist_row / 2; } if (xproc < nprocs - 1) { - MPI_Wait(request + 1, &status); + MPI_Wait(&request[1], &status); } } @@ -805,6 +548,8 @@ void LaplacePCR ::pcr_forward_single_row(Matrix& a, Matrix& Matrix& c, Matrix& r, Matrix& x) const { + const int nsys = std::get<0>(a.shape()); + Array alpha(nsys); Array gamma(nsys); Array sbuf(4 * nsys); @@ -813,12 +558,11 @@ void LaplacePCR ::pcr_forward_single_row(Matrix& a, Matrix& MPI_Status status; Array request(4); - MPI_Comm comm = BoutComm::get(); + const MPI_Comm comm = BoutComm::get(); const int nlevel = log2(nprocs); const int nhprocs = nprocs / 2; int dist_rank = 1; - int dist2_rank = 2; /// Parallel cyclic reduction continues until 2x2 matrix are made between a pair of /// rank, (myrank, myrank+nhprocs). @@ -936,7 +680,6 @@ void LaplacePCR ::pcr_forward_single_row(Matrix& a, Matrix& } dist_rank *= 2; - dist2_rank *= 2; } /// Solving 2x2 matrix. All pair of ranks, myrank and myrank+nhprocs, solves it @@ -1041,21 +784,21 @@ void LaplacePCR ::verify_solution(const Matrix& a_ver, } if (xproc > 0) { - MPI_Irecv(&rbufdown[0], nsys, MPI_DOUBLE_COMPLEX, myrank - 1, 901, MPI_COMM_WORLD, + MPI_Irecv(&rbufdown[0], nsys, MPI_DOUBLE_COMPLEX, myrank - 1, 901, BoutComm::get(), &request[1]); for (int kz = 0; kz < nsys; kz++) { sbufdown[kz] = x_ver(kz, 1); } - MPI_Isend(&sbufdown[0], nsys, MPI_DOUBLE_COMPLEX, myrank - 1, 900, MPI_COMM_WORLD, + MPI_Isend(&sbufdown[0], nsys, MPI_DOUBLE_COMPLEX, myrank - 1, 900, BoutComm::get(), &request[0]); } if (xproc < nprocs - 1) { - MPI_Irecv(&rbufup[0], nsys, MPI_DOUBLE_COMPLEX, myrank + 1, 900, MPI_COMM_WORLD, + MPI_Irecv(&rbufup[0], nsys, MPI_DOUBLE_COMPLEX, myrank + 1, 900, BoutComm::get(), &request[3]); for (int kz = 0; kz < nsys; kz++) { sbufup[kz] = x_ver(kz, nx); } - MPI_Isend(&sbufup[0], nsys, MPI_DOUBLE_COMPLEX, myrank + 1, 901, MPI_COMM_WORLD, + MPI_Isend(&sbufup[0], nsys, MPI_DOUBLE_COMPLEX, myrank + 1, 901, BoutComm::get(), &request[2]); } @@ -1107,3 +850,5 @@ void LaplacePCR ::verify_solution(const Matrix& a_ver, output.write("max abs error {}\n", max_error); output.write("max abs error location {} {}\n", max_loc_x, max_loc_z); } + +#endif // BOUT_USE_METRIC_3D diff --git a/src/invert/laplace/impls/pcr/pcr.hxx b/src/invert/laplace/impls/pcr/pcr.hxx index ec4637f56c..2b90e17466 100644 --- a/src/invert/laplace/impls/pcr/pcr.hxx +++ b/src/invert/laplace/impls/pcr/pcr.hxx @@ -29,13 +29,24 @@ class LaplacePCR; #ifndef BOUT_PCR_H #define BOUT_PCR_H -#include +#include #include + +#if BOUT_USE_METRIC_3D + +namespace { +const RegisterUnavailableLaplace + registerlaplacepcr(LAPLACE_PCR, "BOUT++ was configured with 3D metrics"); +} + +#else + +#include #include #include namespace { -RegisterLaplace registerlaplacepcr(LAPLACE_PCR); +const RegisterLaplace registerlaplacepcr(LAPLACE_PCR); } class LaplacePCR : public Laplacian { @@ -175,4 +186,6 @@ private: bool dst{false}; }; +#endif // BOUT_USE_METRIC_3D + #endif // BOUT_PCR_H diff --git a/src/invert/laplace/impls/pcr_thomas/pcr_thomas.cxx b/src/invert/laplace/impls/pcr_thomas/pcr_thomas.cxx index 61c8f58694..84ff871ed0 100644 --- a/src/invert/laplace/impls/pcr_thomas/pcr_thomas.cxx +++ b/src/invert/laplace/impls/pcr_thomas/pcr_thomas.cxx @@ -36,10 +36,18 @@ * **************************************************************************/ +#include "bout/build_defines.hxx" + +#if not BOUT_USE_METRIC_3D + #include "pcr_thomas.hxx" -#include "bout/globals.hxx" +#include "../../common_transform.hxx" + +#include "bout/array.hxx" #include "bout/boutcomm.hxx" +#include "bout/dcomplex.hxx" +#include "bout/globals.hxx" #include #include #include @@ -65,6 +73,8 @@ LaplacePCR_THOMAS::LaplacePCR_THOMAS(Options* opt, CELL_LOC loc, Mesh* mesh_in, ncx(localmesh->LocalNx), ny(localmesh->LocalNy), avec(ny, nmode, ncx), bvec(ny, nmode, ncx), cvec(ny, nmode, ncx) { + bout::fft::assertZSerial(*localmesh, "`pcr_thomas` inversion"); + Acoef.setLocation(location); C1coef.setLocation(location); C2coef.setLocation(location); @@ -135,11 +145,6 @@ FieldPerp LaplacePCR_THOMAS::solve(const FieldPerp& rhs, const FieldPerp& x0) { ASSERT1(rhs.getLocation() == location); ASSERT1(x0.getLocation() == location); - FieldPerp x{emptyFrom(rhs)}; // Result - - int jy = rhs.getIndex(); // Get the Y index - x.setIndex(jy); - // Get the width of the boundary // If the flags to assign that only one guard cell should be used is set @@ -156,155 +161,33 @@ FieldPerp LaplacePCR_THOMAS::solve(const FieldPerp& rhs, const FieldPerp& x0) { } if (dst) { - const BoutReal zlength = getUniform(coords->dz) * (localmesh->LocalNz - 3); - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array( - localmesh->LocalNz); // ZFFT routine expects input of this length - - // Loop over X indices, including boundaries but not guard cells. (unless periodic - // in x) - BOUT_OMP_PERF(for) - for (int ix = xs; ix <= xe; ix++) { - // Take DST in Z direction and put result in k1d - - if (((ix < inbndry) && isInnerBoundaryFlagSetOnFirstX(INVERT_SET)) - || ((localmesh->LocalNx - ix - 1 < outbndry) - && isOuterBoundaryFlagSetOnLastX(INVERT_SET))) { - // Use the values in x0 in the boundary - DST(x0[ix] + 1, localmesh->LocalNz - 2, std::begin(k1d)); - } else { - DST(rhs[ix] + 1, localmesh->LocalNz - 2, std::begin(k1d)); - } - - // Copy into array, transposing so kz is first index - for (int kz = 0; kz < nmode; kz++) { - bcmplx(kz, ix - xs) = k1d[kz]; - } - } + const DSTTransform transform( + *localmesh, nmode, xs, xe, 0, 0, localmesh->zstart, localmesh->zend, inbndry, + outbndry, isInnerBoundaryFlagSetOnFirstX(INVERT_SET), + isOuterBoundaryFlagSetOnLastX(INVERT_SET), isGlobalFlagSet(INVERT_ZERO_DC)); - // Get elements of the tridiagonal matrix - // including boundary conditions - BOUT_OMP_PERF(for nowait) - for (int kz = 0; kz < nmode; kz++) { - // wave number is 1/[rad]; DST has extra 2. - const BoutReal kwave = kz * 2.0 * PI / (2. * zlength); - - tridagMatrix(&a(kz, 0), &b(kz, 0), &c(kz, 0), &bcmplx(kz, 0), jy, - kz, // wave number index - kwave, // kwave (inverse wave length) - &Acoef, &C1coef, &C2coef, &Dcoef, - false); // Don't include guard cells in arrays - } - } + auto matrices = transform.forward(*this, rhs, x0, Acoef, C1coef, C2coef, Dcoef); // Solve tridiagonal systems - pcr_thomas_solver(a, b, c, bcmplx, xcmplx); - - // FFT back to real space - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array( - localmesh->LocalNz); // ZFFT routine expects input of this length - - BOUT_OMP_PERF(for nowait) - for (int ix = xs; ix <= xe; ix++) { - for (int kz = 0; kz < nmode; kz++) { - k1d[kz] = xcmplx(kz, ix - xs); - } - - for (int kz = nmode; kz < (localmesh->LocalNz); kz++) { - k1d[kz] = 0.0; // Filtering out all higher harmonics - } + pcr_thomas_solver(matrices.a, matrices.b, matrices.c, matrices.bcmplx, xcmplx); - DST_rev(std::begin(k1d), localmesh->LocalNz - 2, x[ix] + 1); - - x(ix, 0) = -x(ix, 2); - x(ix, localmesh->LocalNz - 1) = -x(ix, localmesh->LocalNz - 3); - } - } - } else { - const BoutReal zlength = getUniform(coords->zlength()); - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array((localmesh->LocalNz) / 2 - + 1); // ZFFT routine expects input of this length - - // Loop over X indices, including boundaries but not guard cells (unless periodic in - // x) - BOUT_OMP_PERF(for) - for (int ix = xs; ix <= xe; ix++) { - // Take FFT in Z direction, apply shift, and put result in k1d - - if (((ix < inbndry) && isInnerBoundaryFlagSetOnFirstX(INVERT_SET)) - || ((localmesh->LocalNx - ix - 1 < outbndry) - && isOuterBoundaryFlagSetOnLastX(INVERT_SET))) { - // Use the values in x0 in the boundary - rfft(x0[ix], localmesh->LocalNz, std::begin(k1d)); - } else { - rfft(rhs[ix], localmesh->LocalNz, std::begin(k1d)); - } - - // Copy into array, transposing so kz is first index - for (int kz = 0; kz < nmode; kz++) { - bcmplx(kz, ix - xs) = k1d[kz]; - } - } - - // Get elements of the tridiagonal matrix - // including boundary conditions - BOUT_OMP_PERF(for nowait) - for (int kz = 0; kz < nmode; kz++) { - const BoutReal kwave = kz * 2.0 * PI / zlength; // wave number is 1/[rad] - tridagMatrix(&a(kz, 0), &b(kz, 0), &c(kz, 0), &bcmplx(kz, 0), jy, - kz, // True for the component constant (DC) in Z - kwave, // Z wave number - &Acoef, &C1coef, &C2coef, &Dcoef, - false); // Don't include guard cells in arrays - } - } - - // Solve tridiagonal systems - pcr_thomas_solver(a, b, c, bcmplx, xcmplx); - - // FFT back to real space - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array((localmesh->LocalNz) / 2 - + 1); // ZFFT routine expects input of this length - - const bool zero_DC = isGlobalFlagSet(INVERT_ZERO_DC); - - BOUT_OMP_PERF(for nowait) - for (int ix = xs; ix <= xe; ix++) { - if (zero_DC) { - k1d[0] = 0.; - } - - for (int kz = static_cast(zero_DC); kz < nmode; kz++) { - k1d[kz] = xcmplx(kz, ix - xs); - } + return transform.backward(rhs, xcmplx); + } - for (int kz = nmode; kz < (localmesh->LocalNz) / 2 + 1; kz++) { - k1d[kz] = 0.0; // Filtering out all higher harmonics - } + const FFTTransform transform( + *localmesh, nmode, xs, xe, 0, 0, localmesh->zstart, localmesh->zend, inbndry, + outbndry, isInnerBoundaryFlagSetOnFirstX(INVERT_SET), + isOuterBoundaryFlagSetOnLastX(INVERT_SET), isGlobalFlagSet(INVERT_ZERO_DC)); - irfft(std::begin(k1d), localmesh->LocalNz, x[ix]); - } - } - } + auto matrices = transform.forward(*this, rhs, x0, Acoef, C1coef, C2coef, Dcoef); - checkData(x); + // Solve tridiagonal systems + pcr_thomas_solver(matrices.a, matrices.b, matrices.c, matrices.bcmplx, xcmplx); - return x; + return transform.backward(rhs, xcmplx); } Field3D LaplacePCR_THOMAS::solve(const Field3D& rhs, const Field3D& x0) { - TRACE("LaplacePCR_THOMAS::solve(Field3D, Field3D)"); ASSERT1(rhs.getLocation() == location); ASSERT1(x0.getLocation() == location); @@ -312,8 +195,6 @@ Field3D LaplacePCR_THOMAS::solve(const Field3D& rhs, const Field3D& x0) { Timer timer("invert"); - Field3D x{emptyFrom(rhs)}; // Result - // Get the width of the boundary // If the flags to assign that only one guard cell should be used is set @@ -329,7 +210,7 @@ Field3D LaplacePCR_THOMAS::solve(const Field3D& rhs, const Field3D& x0) { outbndry = 1; } - int nx = xe - xs + 1; // Number of X points on this processor + const int nx = xe - xs + 1; // Number of X points on this processor // Get range of Y indices int ys = localmesh->ystart; @@ -352,186 +233,33 @@ Field3D LaplacePCR_THOMAS::solve(const Field3D& rhs, const Field3D& x0) { const int ny = (ye - ys + 1); // Number of Y points nsys = nmode * ny; // Number of systems of equations to solve - const int nxny = nx * ny; // Number of points in X-Y - - auto a3D = Matrix(nsys, nx); - auto b3D = Matrix(nsys, nx); - auto c3D = Matrix(nsys, nx); - auto xcmplx3D = Matrix(nsys, nx); - auto bcmplx3D = Matrix(nsys, nx); if (dst) { - const BoutReal zlength = getUniform(coords->dz) * (localmesh->LocalNz - 3); - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array( - localmesh->LocalNz); // ZFFT routine expects input of this length - - // Loop over X and Y indices, including boundaries but not guard cells. - // (unless periodic in x) - BOUT_OMP_PERF(for) - for (int ind = 0; ind < nxny; ++ind) { - // ind = (ix - xs)*(ye - ys + 1) + (iy - ys) - int ix = xs + ind / ny; - int iy = ys + ind % ny; - - // Take DST in Z direction and put result in k1d - - if (((ix < inbndry) && isInnerBoundaryFlagSetOnFirstX(INVERT_SET)) - || ((localmesh->LocalNx - ix - 1 < outbndry) - && isOuterBoundaryFlagSetOnLastX(INVERT_SET))) { - // Use the values in x0 in the boundary - DST(x0(ix, iy) + 1, localmesh->LocalNz - 2, std::begin(k1d)); - } else { - DST(rhs(ix, iy) + 1, localmesh->LocalNz - 2, std::begin(k1d)); - } - - // Copy into array, transposing so kz is first index - for (int kz = 0; kz < nmode; kz++) { - bcmplx3D((iy - ys) * nmode + kz, ix - xs) = k1d[kz]; - } - } - - // Get elements of the tridiagonal matrix - // including boundary conditions - BOUT_OMP_PERF(for nowait) - for (int ind = 0; ind < nsys; ind++) { - // ind = (iy - ys) * nmode + kz - int iy = ys + ind / nmode; - int kz = ind % nmode; - - // wave number is 1/[rad]; DST has extra 2. - BoutReal kwave = kz * 2.0 * PI / (2. * zlength); - - tridagMatrix(&a3D(ind, 0), &b3D(ind, 0), &c3D(ind, 0), &bcmplx3D(ind, 0), iy, - kz, // wave number index - kwave, // kwave (inverse wave length) - &Acoef, &C1coef, &C2coef, &Dcoef, - false); // Don't include guard cells in arrays - } - } - - // Solve tridiagonal systems - pcr_thomas_solver(a3D, b3D, c3D, bcmplx3D, xcmplx3D); - - // FFT back to real space - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array( - localmesh->LocalNz); // ZFFT routine expects input of this length - - BOUT_OMP_PERF(for nowait) - for (int ind = 0; ind < nxny; ++ind) { // Loop over X and Y - // ind = (ix - xs)*(ye - ys + 1) + (iy - ys) - int ix = xs + ind / ny; - int iy = ys + ind % ny; - - for (int kz = 0; kz < nmode; kz++) { - k1d[kz] = xcmplx3D((iy - ys) * nmode + kz, ix - xs); - } + const DSTTransform transform( + *localmesh, nmode, xs, xe, ys, ye, localmesh->zstart, localmesh->zend, inbndry, + outbndry, isInnerBoundaryFlagSetOnFirstX(INVERT_SET), + isOuterBoundaryFlagSetOnLastX(INVERT_SET), isGlobalFlagSet(INVERT_ZERO_DC)); - for (int kz = nmode; kz < localmesh->LocalNz; kz++) { - k1d[kz] = 0.0; // Filtering out all higher harmonics - } - - DST_rev(std::begin(k1d), localmesh->LocalNz - 2, &x(ix, iy, 1)); - - x(ix, iy, 0) = -x(ix, iy, 2); - x(ix, iy, localmesh->LocalNz - 1) = -x(ix, iy, localmesh->LocalNz - 3); - } - } - } else { - const BoutReal zlength = getUniform(coords->zlength()); - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array(localmesh->LocalNz / 2 - + 1); // ZFFT routine expects input of this length - - // Loop over X and Y indices, including boundaries but not guard cells - // (unless periodic in x) - - BOUT_OMP_PERF(for) - for (int ind = 0; ind < nxny; ++ind) { - // ind = (ix - xs)*(ye - ys + 1) + (iy - ys) - int ix = xs + ind / ny; - int iy = ys + ind % ny; - - // Take FFT in Z direction, apply shift, and put result in k1d - - if (((ix < inbndry) && isInnerBoundaryFlagSetOnFirstX(INVERT_SET)) - || ((localmesh->LocalNx - ix - 1 < outbndry) - && isOuterBoundaryFlagSetOnLastX(INVERT_SET))) { - // Use the values in x0 in the boundary - rfft(x0(ix, iy), localmesh->LocalNz, std::begin(k1d)); - } else { - rfft(rhs(ix, iy), localmesh->LocalNz, std::begin(k1d)); - } - - // Copy into array, transposing so kz is first index - for (int kz = 0; kz < nmode; kz++) { - bcmplx3D((iy - ys) * nmode + kz, ix - xs) = k1d[kz]; - } - } - - // Get elements of the tridiagonal matrix - // including boundary conditions - BOUT_OMP_PERF(for nowait) - for (int ind = 0; ind < nsys; ind++) { - // ind = (iy - ys) * nmode + kz - int iy = ys + ind / nmode; - int kz = ind % nmode; - - // wave number is 1/[rad] - BoutReal kwave = kz * 2.0 * PI / zlength; - tridagMatrix(&a3D(ind, 0), &b3D(ind, 0), &c3D(ind, 0), &bcmplx3D(ind, 0), iy, - kz, // True for the component constant (DC) in Z - kwave, // Z wave number - &Acoef, &C1coef, &C2coef, &Dcoef, - false); // Don't include guard cells in arrays - } - } + auto matrices = transform.forward(*this, rhs, x0, Acoef, C1coef, C2coef, Dcoef); // Solve tridiagonal systems - pcr_thomas_solver(a3D, b3D, c3D, bcmplx3D, xcmplx3D); + pcr_thomas_solver(matrices.a, matrices.b, matrices.c, matrices.bcmplx, xcmplx3D); - // FFT back to real space - BOUT_OMP_PERF(parallel) - { - /// Create a local thread-scope working array - auto k1d = Array((localmesh->LocalNz) / 2 - + 1); // ZFFT routine expects input of this length - - const bool zero_DC = isGlobalFlagSet(INVERT_ZERO_DC); - - BOUT_OMP_PERF(for nowait) - for (int ind = 0; ind < nxny; ++ind) { // Loop over X and Y - int ix = xs + ind / ny; - int iy = ys + ind % ny; + return transform.backward(rhs, xcmplx3D); + } - if (zero_DC) { - k1d[0] = 0.; - } + const FFTTransform transform( + *localmesh, nmode, xs, xe, ys, ye, localmesh->zstart, localmesh->zend, inbndry, + outbndry, isInnerBoundaryFlagSetOnFirstX(INVERT_SET), + isOuterBoundaryFlagSetOnLastX(INVERT_SET), isGlobalFlagSet(INVERT_ZERO_DC)); - for (int kz = static_cast(zero_DC); kz < nmode; kz++) { - k1d[kz] = xcmplx3D((iy - ys) * nmode + kz, ix - xs); - } + auto matrices = transform.forward(*this, rhs, x0, Acoef, C1coef, C2coef, Dcoef); - for (int kz = nmode; kz < localmesh->LocalNz / 2 + 1; kz++) { - k1d[kz] = 0.0; // Filtering out all higher harmonics - } + // Solve tridiagonal systems + pcr_thomas_solver(matrices.a, matrices.b, matrices.c, matrices.bcmplx, xcmplx3D); - irfft(std::begin(k1d), localmesh->LocalNz, x(ix, iy)); - } - } - } - - checkData(x); - - return x; + return transform.backward(rhs, xcmplx3D); } /** @@ -555,6 +283,8 @@ void LaplacePCR_THOMAS ::pcr_thomas_solver(Matrix& a_mpi, const int xend = localmesh->xend; const int nx = xend - xstart + 1; // number of interior points + const int nsys = std::get<0>(a_mpi.shape()); + // Handle boundary points so that the PCR algorithm works with arrays of // the same size on each rank. // Note that this modifies the coefficients of b and r in the first and last @@ -621,6 +351,8 @@ void LaplacePCR_THOMAS ::eliminate_boundary_rows(const Matrix& a, const Matrix& c, Matrix& r) { + const int nsys = std::get<0>(a.shape()); + if (localmesh->firstX()) { // x index is first interior row const int xstart = localmesh->xstart; @@ -655,6 +387,8 @@ void LaplacePCR_THOMAS ::apply_boundary_conditions(const Matrix& a, const Matrix& r, Matrix& x) { + const int nsys = std::get<0>(a.shape()); + if (localmesh->firstX()) { for (int kz = 0; kz < nsys; kz++) { for (int ix = localmesh->xstart - 1; ix >= 0; ix--) { @@ -680,6 +414,8 @@ void LaplacePCR_THOMAS ::apply_boundary_conditions(const Matrix& a, void LaplacePCR_THOMAS ::cr_forward_multiple_row(Matrix& a, Matrix& b, Matrix& c, Matrix& r) const { + const int nsys = std::get<0>(a.shape()); + MPI_Comm comm = BoutComm::get(); Array alpha(nsys); Array gamma(nsys); @@ -755,6 +491,8 @@ void LaplacePCR_THOMAS ::cr_backward_multiple_row(Matrix& a, Matrix& c, Matrix& r, Matrix& x) const { + const int nsys = std::get<0>(a.shape()); + MPI_Comm comm = BoutComm::get(); MPI_Status status; @@ -806,6 +544,8 @@ void LaplacePCR_THOMAS ::pcr_forward_single_row(Matrix& a, Matrix& c, Matrix& r, Matrix& x) const { + const int nsys = std::get<0>(a.shape()); + Array alpha(nsys); Array gamma(nsys); Array sbuf(4 * nsys); @@ -983,6 +723,8 @@ void LaplacePCR_THOMAS ::pThomas_forward_multiple_row(Matrix& a, Matrix& b, Matrix& c, Matrix& r) const { + const int nsys = std::get<0>(a.shape()); + for (int kz = 0; kz < nsys; kz++) { for (int i = 3; i <= n_mpi; i++) { const dcomplex alpha = -a(kz, i) / b(kz, i - 1); @@ -1014,6 +756,8 @@ void LaplacePCR_THOMAS ::pcr_double_row_substitution(Matrix& a, Matrix& c, Matrix& r, Matrix& x) { + const int nsys = std::get<0>(a.shape()); + Array alpha(nsys); Array gamma(nsys); Array sbuf(4 * nsys); @@ -1140,21 +884,21 @@ void LaplacePCR_THOMAS ::verify_solution(const Matrix& a_ver, } if (xproc > 0) { - MPI_Irecv(&rbufdown[0], nsys, MPI_DOUBLE_COMPLEX, myrank - 1, 901, MPI_COMM_WORLD, + MPI_Irecv(&rbufdown[0], nsys, MPI_DOUBLE_COMPLEX, myrank - 1, 901, BoutComm::get(), &request[1]); for (int kz = 0; kz < nsys; kz++) { sbufdown[kz] = x_ver(kz, 1); } - MPI_Isend(&sbufdown[0], nsys, MPI_DOUBLE_COMPLEX, myrank - 1, 900, MPI_COMM_WORLD, + MPI_Isend(&sbufdown[0], nsys, MPI_DOUBLE_COMPLEX, myrank - 1, 900, BoutComm::get(), &request[0]); } if (xproc < nprocs - 1) { - MPI_Irecv(&rbufup[0], nsys, MPI_DOUBLE_COMPLEX, myrank + 1, 900, MPI_COMM_WORLD, + MPI_Irecv(&rbufup[0], nsys, MPI_DOUBLE_COMPLEX, myrank + 1, 900, BoutComm::get(), &request[3]); for (int kz = 0; kz < nsys; kz++) { sbufup[kz] = x_ver(kz, nx); } - MPI_Isend(&sbufup[0], nsys, MPI_DOUBLE_COMPLEX, myrank + 1, 901, MPI_COMM_WORLD, + MPI_Isend(&sbufup[0], nsys, MPI_DOUBLE_COMPLEX, myrank + 1, 901, BoutComm::get(), &request[2]); } @@ -1189,3 +933,5 @@ void LaplacePCR_THOMAS ::verify_solution(const Matrix& a_ver, } output.write("max abs error {}\n", max_error); } + +#endif // BOUT_USE_METRIC_3D diff --git a/src/invert/laplace/impls/pcr_thomas/pcr_thomas.hxx b/src/invert/laplace/impls/pcr_thomas/pcr_thomas.hxx index e12a647789..1b5c6b97b9 100644 --- a/src/invert/laplace/impls/pcr_thomas/pcr_thomas.hxx +++ b/src/invert/laplace/impls/pcr_thomas/pcr_thomas.hxx @@ -29,8 +29,19 @@ class LaplacePCR_THOMAS; #ifndef BOUT_PCR_THOMAS_H #define BOUT_PCR_THOMAS_H +#include "bout/build_defines.hxx" +#include "bout/invert_laplace.hxx" + +#if BOUT_USE_METRIC_3D + +namespace { +const RegisterUnavailableLaplace + registerlaplacepcrthomas(LAPLACE_PCR_THOMAS, "BOUT++ was configured with 3D metrics"); +} + +#else + #include -#include #include #include @@ -178,4 +189,6 @@ private: bool dst{false}; }; +#endif // BOUT_USE_METRIC_3D + #endif // BOUT_PCR_THOMAS_H diff --git a/src/invert/laplace/impls/petsc/petsc_laplace.cxx b/src/invert/laplace/impls/petsc/petsc_laplace.cxx index 5f417d4faa..89ba25405b 100644 --- a/src/invert/laplace/impls/petsc/petsc_laplace.cxx +++ b/src/invert/laplace/impls/petsc/petsc_laplace.cxx @@ -31,41 +31,140 @@ #include "petsc_laplace.hxx" #include +#include #include +#include +#include +#include +#include #include +#include #include +#include #include +#include #include #include -#define KSP_RICHARDSON "richardson" -#define KSP_CHEBYSHEV "chebyshev" -#define KSP_CG "cg" -#define KSP_GMRES "gmres" -#define KSP_TCQMR "tcqmr" -#define KSP_BCGS "bcgs" -#define KSP_CGS "cgs" -#define KSP_TFQMR "tfqmr" -#define KSP_CR "cr" -#define KSP_LSQR "lsqr" -#define KSP_BICG "bicg" -#define KSP_PREONLY "preonly" - -static PetscErrorCode laplacePCapply(PC pc, Vec x, Vec y) { +#include +#include +#include +#include + +namespace { +PetscErrorCode laplacePCapply(PC pc, Vec x, Vec y) { PetscFunctionBegin; // NOLINT LaplacePetsc* laplace = nullptr; - const int ierr = PCShellGetContext(pc, reinterpret_cast(&laplace)); // NOLINT - CHKERRQ(ierr); + CHKERRQ(PCShellGetContext(pc, reinterpret_cast(&laplace))); // NOLINT PetscFunctionReturn(laplace->precon(x, y)); // NOLINT } +auto set_stencil(const Mesh& localmesh, bool fourth_order) { + OperatorStencil stencil; + IndexOffset zero; + // Start with a square stencil 1-point wide + std::set offsets = { + // clang-format off + zero.xm().zp(), zero.zp(), zero.xp().zp(), + zero.xm(), zero, zero.xp(), + zero.xm().zm(), zero.zm(), zero.xp().zm(), + // clang-format on + }; + + if (fourth_order) { + // Add a square stencil 2-points wide + offsets.insert({ + // clang-format off + zero.xm(2).zp(2), zero.xm().zp(2), zero.zp(2), zero.xp().zp(2), zero.xp(2).zp(2), + zero.xm(2).zp(), zero.xp(2).zp(), + zero.xm(2), zero.xp(2), + zero.xm(2).zm(), zero.xp(2).zm(), + zero.xm(2).zm(2), zero.xm().zm(2), zero.zm(2), zero.xp().zm(2), zero.xp(2).zm(2), + // clang-format on + }); + } + + const std::vector offsetsVec(offsets.begin(), offsets.end()); + stencil.add( + [&localmesh](IndPerp ind) -> bool { + return (localmesh.xstart <= ind.x() && ind.x() <= localmesh.xend + and (localmesh.zstart <= ind.z() && ind.z() <= localmesh.zend)); + }, + offsetsVec); + + // Add inner X boundary + if (localmesh.firstX()) { + const auto first_boundary = localmesh.xstart - 1; + const auto second_boundary = localmesh.xstart - 2; + + if (fourth_order) { + stencil.add( + [first_boundary, second_boundary](IndPerp ind) -> bool { + const auto x = ind.x(); + return x == first_boundary or x == second_boundary; + }, + {zero, zero.xp(1), zero.xp(2), zero.xp(3), zero.xp(4)}); + } else { + stencil.add( + [first_boundary](IndPerp ind) -> bool { return ind.x() == first_boundary; }, + {zero, zero.xp()}); + } + } + // Add outer X boundary + if (localmesh.lastX()) { + const auto first_boundary = localmesh.xend + 1; + const auto second_boundary = localmesh.xend + 2; + + if (fourth_order) { + stencil.add( + [first_boundary, second_boundary](IndPerp ind) -> bool { + const auto x = ind.x(); + return x == first_boundary or x == second_boundary; + }, + {zero, zero.xm(1), zero.xm(2), zero.xm(3), zero.xm(4)}); + } else { + stencil.add( + [first_boundary](IndPerp ind) -> bool { return ind.x() == first_boundary; }, + {zero, zero.xm()}); + } + } + + stencil.add([]([[maybe_unused]] IndPerp ind) -> bool { return true; }, {zero}); + return stencil; +} +} // namespace + LaplacePetsc::LaplacePetsc(Options* opt, const CELL_LOC loc, Mesh* mesh_in, - Solver* UNUSED(solver)) - : Laplacian(opt, loc, mesh_in), A(0.0), C1(1.0), C2(1.0), D(1.0), Ex(0.0), Ez(0.0), - issetD(false), issetC(false), issetE(false), - lib(opt == nullptr ? &(Options::root()["laplace"]) : opt) { + [[maybe_unused]] Solver* solver) + : Laplacian(opt, loc, mesh_in), A(0.0, mesh_in), C1(1.0, mesh_in), C2(1.0, mesh_in), + D(1.0, mesh_in), Ex(0.0, mesh_in), Ez(0.0, mesh_in), issetD(false), issetC(false), + issetE(false), comm(localmesh->getXZcomm()), + opts(opt == nullptr ? &(Options::root()["laplace"]) : opt), + // WARNING: only a few of these options actually make sense: see the + // PETSc documentation to work out which they are (possibly + // pbjacobi, sor might be useful choices?) + ksptype((*opts)["ksptype"].doc("KSP solver type").withDefault(KSPGMRES)), + pctype((*opts)["pctype"] + .doc("Preconditioner type. See the PETSc documentation for options") + .withDefault("none")), + richardson_damping_factor((*opts)["richardson_damping_factor"].withDefault(1.0)), + chebyshev_max((*opts)["chebyshev_max"].withDefault(100)), + chebyshev_min((*opts)["chebyshev_min"].withDefault(0.01)), + gmres_max_steps((*opts)["gmres_max_steps"].withDefault(30)), + rtol((*opts)["rtol"].doc("Relative tolerance for KSP solver").withDefault(1e-5)), + atol((*opts)["atol"].doc("Absolute tolerance for KSP solver").withDefault(1e-50)), + dtol((*opts)["dtol"].doc("Divergence tolerance for KSP solver").withDefault(1e5)), + maxits( + (*opts)["maxits"].doc("Maximum number of KSP iterations").withDefault(100000)), + direct((*opts)["direct"].doc("Use direct (LU) solver?").withDefault(false)), + fourth_order( + (*opts)["fourth_order"].doc("Use fourth order stencil").withDefault(false)), + indexer(std::make_shared>( + localmesh, set_stencil(*localmesh, fourth_order))), + operator2D(indexer), lib(opts) { + A.setLocation(location); C1.setLocation(location); C2.setLocation(location); @@ -73,13 +172,6 @@ LaplacePetsc::LaplacePetsc(Options* opt, const CELL_LOC loc, Mesh* mesh_in, Ex.setLocation(location); Ez.setLocation(location); - // Get Options in Laplace Section - if (!opt) { - opts = Options::getRoot()->getSection("laplace"); - } else { - opts = opt; - } - #if CHECK > 0 // Checking flags are set to something which is not implemented checkFlags(); @@ -91,229 +183,27 @@ LaplacePetsc::LaplacePetsc(Options* opt, const CELL_LOC loc, Mesh* mesh_in, } #endif - // Get communicator for group of processors in X - all points in z-x plane for fixed y. - comm = localmesh->getXcomm(); - - // Need to determine local size to use based on prior parallelisation - // Coefficient values are stored only on local processors. - localN = (localmesh->xend - localmesh->xstart + 1) * (localmesh->LocalNz); - if (localmesh->firstX()) { - localN += - localmesh->xstart - * (localmesh->LocalNz); // If on first processor add on width of boundary region - } - if (localmesh->lastX()) { - localN += - localmesh->xstart - * (localmesh->LocalNz); // If on last processor add on width of boundary region - } - - // Calculate 'size' (the total number of points in physical grid) - if (bout::globals::mpi->MPI_Allreduce(&localN, &size, 1, MPI_INT, MPI_SUM, comm) - != MPI_SUCCESS) { - throw BoutException("Error in MPI_Allreduce during LaplacePetsc initialisation"); - } - - // Calculate total (physical) grid dimensions - meshz = localmesh->LocalNz; - meshx = size / meshz; - - // Create PETSc type of vectors for the solution and the RHS vector - VecCreate(comm, &xs); - VecSetSizes(xs, localN, size); - VecSetFromOptions(xs); - VecDuplicate(xs, &bs); - - // Get 4th order solver switch - opts->get("fourth_order", fourth_order, false); - - // Set size of (the PETSc) Matrix on each processor to localN x localN - MatCreate(comm, &MatA); - MatSetSizes(MatA, localN, localN, size, size); - MatSetFromOptions(MatA); - - /* Pre allocate memory - * nnz denotes an array containing the number of non-zeros in the various rows - * for - * d_nnz - The diagonal terms in the matrix - * o_nnz - The off-diagonal terms in the matrix (needed when running in - * parallel) - */ - PetscInt *d_nnz, *o_nnz; - PetscMalloc((localN) * sizeof(PetscInt), &d_nnz); - PetscMalloc((localN) * sizeof(PetscInt), &o_nnz); - if (fourth_order) { - // first and last 2*localmesh-LocalNz entries are the edge x-values that (may) have 'off-diagonal' components (i.e. on another processor) - if (localmesh->firstX() && localmesh->lastX()) { - for (int i = 0; i < localmesh->LocalNz; i++) { - d_nnz[i] = 15; - d_nnz[localN - 1 - i] = 15; - o_nnz[i] = 0; - o_nnz[localN - 1 - i] = 0; - } - for (int i = (localmesh->LocalNz); i < 2 * (localmesh->LocalNz); i++) { - d_nnz[i] = 20; - d_nnz[localN - 1 - i] = 20; - o_nnz[i] = 0; - o_nnz[localN - 1 - i] = 0; - } - } else if (localmesh->firstX()) { - for (int i = 0; i < localmesh->LocalNz; i++) { - d_nnz[i] = 15; - d_nnz[localN - 1 - i] = 15; - o_nnz[i] = 0; - o_nnz[localN - 1 - i] = 10; - } - for (int i = (localmesh->LocalNz); i < 2 * (localmesh->LocalNz); i++) { - d_nnz[i] = 20; - d_nnz[localN - 1 - i] = 20; - o_nnz[i] = 0; - o_nnz[localN - 1 - i] = 5; - } - } else if (localmesh->lastX()) { - for (int i = 0; i < localmesh->LocalNz; i++) { - d_nnz[i] = 15; - d_nnz[localN - 1 - i] = 15; - o_nnz[i] = 10; - o_nnz[localN - 1 - i] = 0; - } - for (int i = (localmesh->LocalNz); i < 2 * (localmesh->LocalNz); i++) { - d_nnz[i] = 20; - d_nnz[localN - 1 - i] = 20; - o_nnz[i] = 5; - o_nnz[localN - 1 - i] = 0; - } - } else { - for (int i = 0; i < localmesh->LocalNz; i++) { - d_nnz[i] = 15; - d_nnz[localN - 1 - i] = 15; - o_nnz[i] = 10; - o_nnz[localN - 1 - i] = 10; - } - for (int i = (localmesh->LocalNz); i < 2 * (localmesh->LocalNz); i++) { - d_nnz[i] = 20; - d_nnz[localN - 1 - i] = 20; - o_nnz[i] = 5; - o_nnz[localN - 1 - i] = 5; - } - } - - for (int i = 2 * (localmesh->LocalNz); i < localN - 2 * ((localmesh->LocalNz)); i++) { - d_nnz[i] = 25; - d_nnz[localN - 1 - i] = 25; - o_nnz[i] = 0; - o_nnz[localN - 1 - i] = 0; - } - - // Use d_nnz and o_nnz for preallocating the matrix - if (localmesh->firstX() && localmesh->lastX()) { - // Only one processor in X - MatSeqAIJSetPreallocation(MatA, 0, d_nnz); - } else { - MatMPIAIJSetPreallocation(MatA, 0, d_nnz, 0, o_nnz); - } - } else { - // first and last localmesh->LocalNz entries are the edge x-values that (may) have 'off-diagonal' components (i.e. on another processor) - if (localmesh->firstX() && localmesh->lastX()) { - for (int i = 0; i < localmesh->LocalNz; i++) { - d_nnz[i] = 6; - d_nnz[localN - 1 - i] = 6; - o_nnz[i] = 0; - o_nnz[localN - 1 - i] = 0; - } - } else if (localmesh->firstX()) { - for (int i = 0; i < localmesh->LocalNz; i++) { - d_nnz[i] = 6; - d_nnz[localN - 1 - i] = 6; - o_nnz[i] = 0; - o_nnz[localN - 1 - i] = 3; - } - } else if (localmesh->lastX()) { - for (int i = 0; i < localmesh->LocalNz; i++) { - d_nnz[i] = 6; - d_nnz[localN - 1 - i] = 6; - o_nnz[i] = 3; - o_nnz[localN - 1 - i] = 0; - } - } else { - for (int i = 0; i < localmesh->LocalNz; i++) { - d_nnz[i] = 6; - d_nnz[localN - 1 - i] = 6; - o_nnz[i] = 3; - o_nnz[localN - 1 - i] = 3; - } - } - - for (int i = localmesh->LocalNz; i < localN - (localmesh->LocalNz); i++) { - d_nnz[i] = 9; - d_nnz[localN - 1 - i] = 9; - o_nnz[i] = 0; - o_nnz[localN - 1 - i] = 0; - } - - // Use d_nnz and o_nnz for preallocating the matrix - if (localmesh->firstX() && localmesh->lastX()) { - MatSeqAIJSetPreallocation(MatA, 0, d_nnz); - } else { - MatMPIAIJSetPreallocation(MatA, 0, d_nnz, 0, o_nnz); - } - } - // Free the d_nnz and o_nnz arrays, as these are will not be used anymore - PetscFree(d_nnz); - PetscFree(o_nnz); - // Sets up the internal matrix data structures for the later use. - MatSetUp(MatA); - - // Declare KSP Context (abstract PETSc object that manages all Krylov methods) - KSPCreate(comm, &ksp); - - // Get KSP Solver Type (Generalizes Minimal RESidual is the default) - ksptype = (*opts)["ksptype"].doc("KSP solver type").withDefault(KSP_GMRES); - - // Get preconditioner type - // WARNING: only a few of these options actually make sense: see the - // PETSc documentation to work out which they are (possibly - // pbjacobi, sor might be useful choices?) - pctype = (*opts)["pctype"] - .doc("Preconditioner type. See the PETSc documentation for options") - .withDefault("none"); - // Let "user" be a synonym for "shell" if (pctype == "user") { pctype = PCSHELL; } - // Get Options specific to particular solver types - opts->get("richardson_damping_factor", richardson_damping_factor, 1.0, true); - opts->get("chebyshev_max", chebyshev_max, 100, true); - opts->get("chebyshev_min", chebyshev_min, 0.01, true); - opts->get("gmres_max_steps", gmres_max_steps, 30, true); - - // Get Tolerances for KSP solver - rtol = (*opts)["rtol"].doc("Relative tolerance for KSP solver").withDefault(1e-5); - atol = (*opts)["atol"].doc("Absolute tolerance for KSP solver").withDefault(1e-50); - dtol = (*opts)["dtol"].doc("Divergence tolerance for KSP solver").withDefault(1e5); - maxits = (*opts)["maxits"].doc("Maximum number of KSP iterations").withDefault(100000); - - // Get direct solver switch - direct = (*opts)["direct"].doc("Use direct (LU) solver?").withDefault(false); if (direct) { - output << endl - << "Using LU decompostion for direct solution of system" << endl - << endl; + output.write("\nUsing LU decompostion for direct solution of system\n"); } if (pctype == PCSHELL) { - rightprec = (*opts)["rightprec"].doc("Right preconditioning?").withDefault(true); // Options for preconditioner are in a subsection pcsolve = Laplacian::create(opts->getSection("precon")); } +} - // Ensure that the matrix is constructed first time - // coefchanged = true; - // lastflag = -1; +LaplacePetsc::~LaplacePetsc() { + if (ksp_initialised) { + KSPDestroy(&ksp); + } } FieldPerp LaplacePetsc::solve(const FieldPerp& b) { return solve(b, b); } @@ -336,7 +226,6 @@ FieldPerp LaplacePetsc::solve(const FieldPerp& b) { return solve(b, b); } * \returns sol The solution x of the problem Ax=b. */ FieldPerp LaplacePetsc::solve(const FieldPerp& b, const FieldPerp& x0) { - TRACE("LaplacePetsc::solve"); ASSERT1(localmesh == b.getMesh() && localmesh == x0.getMesh()); ASSERT1(b.getLocation() == location); @@ -346,406 +235,38 @@ FieldPerp LaplacePetsc::solve(const FieldPerp& b, const FieldPerp& x0) { checkFlags(); #endif - int y = b.getIndex(); // Get the Y index - sol.setIndex(y); // Initialize the solution field. - sol = 0.; - - // Determine which row/columns of the matrix are locally owned - MatGetOwnershipRange(MatA, &Istart, &Iend); - - int i = Istart; // The row in the PETSc matrix + // Set member variable so that we can pass through to shell preconditioner if + // required + yindex = b.getIndex(); { - Timer timer("petscsetup"); + const Timer timer("petscsetup"); - // if ((fourth_order) && !(lastflag&INVERT_4TH_ORDER)) throw BoutException("Should not change INVERT_4TH_ORDER flag in LaplacePetsc: 2nd order and 4th order require different pre-allocation to optimize PETSc solver"); + const bool inner_X_neumann = isInnerBoundaryFlagSet(INVERT_AC_GRAD); + const bool outer_X_neumann = isOuterBoundaryFlagSet(INVERT_AC_GRAD); - /* Set Matrix Elements - * - * Loop over locally owned rows of matrix A - * i labels NODE POINT from - * bottom left = (0,0) = 0 - * to - * top right = (meshx-1,meshz-1) = meshx*meshz-1 - * - * i increments by 1 for an increase of 1 in Z - * i increments by meshz for an increase of 1 in X. - * - * In other word the indexing is done in a row-major order, but starting at - * bottom left rather than top left - */ - // X=0 to localmesh->xstart-1 defines the boundary region of the domain. - // Set the values for the inner boundary region - if (localmesh->firstX()) { - for (int x = 0; x < localmesh->xstart; x++) { - for (int z = 0; z < localmesh->LocalNz; z++) { - PetscScalar val; // Value of element to be set in the matrix - // If Neumann Boundary Conditions are set. - if (isInnerBoundaryFlagSet(INVERT_AC_GRAD)) { - // Set values corresponding to nodes adjacent in x - if (fourth_order) { - // Fourth Order Accuracy on Boundary - Element(i, x, z, 0, 0, - -25.0 / (12.0 * coords->dx(x, y, z)) / sqrt(coords->g_11(x, y, z)), - MatA); - Element(i, x, z, 1, 0, - 4.0 / coords->dx(x, y, z) / sqrt(coords->g_11(x, y, z)), MatA); - Element(i, x, z, 2, 0, - -3.0 / coords->dx(x, y, z) / sqrt(coords->g_11(x, y, z)), MatA); - Element(i, x, z, 3, 0, - 4.0 / (3.0 * coords->dx(x, y, z)) / sqrt(coords->g_11(x, y, z)), - MatA); - Element(i, x, z, 4, 0, - -1.0 / (4.0 * coords->dx(x, y, z)) / sqrt(coords->g_11(x, y, z)), - MatA); - } else { - // Second Order Accuracy on Boundary - // Element(i,x,z, 0, 0, -3.0 / (2.0*coords->dx(x,y)), MatA ); - // Element(i,x,z, 1, 0, 2.0 / coords->dx(x,y), MatA ); - // Element(i,x,z, 2, 0, -1.0 / (2.0*coords->dx(x,y)), MatA ); - // Element(i,x,z, 3, 0, 0.0, MatA ); // Reset these elements to 0 - // in case 4th order flag was used previously: not allowed now - // Element(i,x,z, 4, 0, 0.0, MatA ); - // Second Order Accuracy on Boundary, set half-way between grid points - Element(i, x, z, 0, 0, - -1.0 / coords->dx(x, y, z) / sqrt(coords->g_11(x, y, z)), MatA); - Element(i, x, z, 1, 0, - 1.0 / coords->dx(x, y, z) / sqrt(coords->g_11(x, y, z)), MatA); - Element(i, x, z, 2, 0, 0.0, MatA); - // Element(i,x,z, 3, 0, 0.0, MatA ); // Reset - // these elements to 0 in case 4th order flag was - // used previously: not allowed now - // Element(i,x,z, 4, 0, 0.0, MatA ); - } - } else { - if (fourth_order) { - // Set Diagonal Values to 1 - Element(i, x, z, 0, 0, 1., MatA); - - // Set off diagonal elements to zero - Element(i, x, z, 1, 0, 0.0, MatA); - Element(i, x, z, 2, 0, 0.0, MatA); - Element(i, x, z, 3, 0, 0.0, MatA); - Element(i, x, z, 4, 0, 0.0, MatA); - } else { - Element(i, x, z, 0, 0, 0.5, MatA); - Element(i, x, z, 1, 0, 0.5, MatA); - Element(i, x, z, 2, 0, 0., MatA); - } - } - - val = 0; // Initialize val - - // Set Components of RHS - // If the inner boundary value should be set by b or x0 - if (isInnerBoundaryFlagSet(INVERT_RHS)) { - val = b[x][z]; - } else if (isInnerBoundaryFlagSet(INVERT_SET)) { - val = x0[x][z]; - } - - // Set components of the RHS (the PETSc vector bs) - // 1 element is being set in row i to val - // INSERT_VALUES replaces existing entries with new values - VecSetValues(bs, 1, &i, &val, INSERT_VALUES); - - // Set components of the and trial solution (the PETSc vector xs) - // 1 element is being set in row i to val - // INSERT_VALUES replaces existing entries with new values - val = x0[x][z]; - VecSetValues(xs, 1, &i, &val, INSERT_VALUES); - - i++; // Increment row in Petsc matrix - } - } - } - - // Set the values for the main domain - for (int x = localmesh->xstart; x <= localmesh->xend; x++) { - for (int z = 0; z < localmesh->LocalNz; z++) { - // NOTE: Only A0 is the A from setCoefA () - BoutReal A0, A1, A2, A3, A4, A5; - A0 = A(x, y, z); - - ASSERT3(std::isfinite(A0)); - - // Set the matrix coefficients - Coeffs(x, y, z, A1, A2, A3, A4, A5); - - BoutReal dx = coords->dx(x, y, z); - BoutReal dx2 = SQ(dx); - BoutReal dz = coords->dz(x, y, z); - BoutReal dz2 = SQ(dz); - BoutReal dxdz = dx * dz; - - ASSERT3(std::isfinite(A1)); - ASSERT3(std::isfinite(A2)); - ASSERT3(std::isfinite(A3)); - ASSERT3(std::isfinite(A4)); - ASSERT3(std::isfinite(A5)); - - // Set Matrix Elements - PetscScalar val = 0.; - if (fourth_order) { - // f(i,j) = f(x,z) - val = A0 - (5.0 / 2.0) * ((A1 / dx2) + (A2 / dz2)); - Element(i, x, z, 0, 0, val, MatA); - - // f(i-2,j-2) - val = A3 / (144.0 * dxdz); - Element(i, x, z, -2, -2, val, MatA); - - // f(i-2,j-1) - val = -1.0 * A3 / (18.0 * dxdz); - Element(i, x, z, -2, -1, val, MatA); - - // f(i-2,j) - val = (1.0 / 12.0) * ((-1.0 * A1 / dx2) + (A4 / dx)); - Element(i, x, z, -2, 0, val, MatA); - - // f(i-2,j+1) - val = A3 / (18.0 * dxdz); - Element(i, x, z, -2, 1, val, MatA); - - // f(i-2,j+2) - val = -1.0 * A3 / (144.0 * dxdz); - Element(i, x, z, -2, 2, val, MatA); - - // f(i-1,j-2) - val = -1.0 * A3 / (18.0 * dxdz); - Element(i, x, z, -1, -2, val, MatA); - - // f(i-1,j-1) - val = 4.0 * A3 / (9.0 * dxdz); - Element(i, x, z, -1, -1, val, MatA); - - // f(i-1,j) - val = (4.0 * A1 / (3.0 * dx2)) - (2.0 * A4 / (3.0 * dx)); - Element(i, x, z, -1, 0, val, MatA); - - // f(i-1,j+1) - val = -4.0 * A3 / (9.0 * dxdz); - Element(i, x, z, -1, 1, val, MatA); - - // f(i-1,j+2) - val = A3 / (18.0 * dxdz); - Element(i, x, z, -1, 2, val, MatA); - - // f(i,j-2) - val = (1.0 / 12.0) * ((-1.0 * A2 / dz2) + (A5 / dz)); - Element(i, x, z, 0, -2, val, MatA); - - // f(i,j-1) - val = (4.0 * A2 / (3.0 * dz2)) - (2.0 * A5 / (3.0 * dz)); - Element(i, x, z, 0, -1, val, MatA); - - // f(i,j+1) - val = (4.0 * A2 / (3.0 * dz2)) + (2.0 * A5 / (3.0 * dz)); - Element(i, x, z, 0, 1, val, MatA); - - // f(i,j+2) - val = (-1.0 / 12.0) * ((A2 / dz2) + (A5 / dz)); - Element(i, x, z, 0, 2, val, MatA); - - // f(i+1,j-2) - val = A3 / (18.0 * dxdz); - Element(i, x, z, 1, -2, val, MatA); - - // f(i+1,j-1) - val = -4.0 * A3 / (9.0 * dxdz); - Element(i, x, z, 1, -1, val, MatA); - - // f(i+1,j) - val = (4.0 * A1 / (3.0 * dx2)) + (2.0 * A4 / (3.0 * dx)); - Element(i, x, z, 1, 0, val, MatA); - - // f(i+1,j+1) - val = 4.0 * A3 / (9.0 * dxdz); - Element(i, x, z, 1, 1, val, MatA); - - // f(i+1,j+2) - val = -1.0 * A3 / (18.0 * dxdz); - Element(i, x, z, 1, 2, val, MatA); - - // f(i+2,j-2) - val = -1.0 * A3 / (144.0 * dxdz); - Element(i, x, z, 2, -2, val, MatA); - - // f(i+2,j-1) - val = A3 / (18.0 * dxdz); - Element(i, x, z, 2, -1, val, MatA); - - // f(i+2,j) - val = (-1.0 / 12.0) * ((A1 / dx2) + (A4 / dx)); - Element(i, x, z, 2, 0, val, MatA); - - // f(i+2,j+1) - val = -1.0 * A3 / (18.0 * dxdz); - Element(i, x, z, 2, 1, val, MatA); - - // f(i+2,j+2) - val = A3 / (144.0 * dxdz); - Element(i, x, z, 2, 2, val, MatA); - } else { - // Second order - - // f(i,j) = f(x,z) - val = A0 - 2.0 * ((A1 / dx2) + (A2 / dz2)); - Element(i, x, z, 0, 0, val, MatA); - - // f(i-1,j-1) - val = A3 / (4.0 * dxdz); - Element(i, x, z, -1, -1, val, MatA); - - // f(i-1,j) - val = (A1 / dx2) - A4 / (2.0 * dx); - Element(i, x, z, -1, 0, val, MatA); - - // f(i-1,j+1) - val = -1.0 * A3 / (4.0 * dxdz); - Element(i, x, z, -1, 1, val, MatA); - - // f(i,j-1) - val = (A2 / dz2) - (A5 / (2.0 * dz)); - Element(i, x, z, 0, -1, val, MatA); - - // f(i,j+1) - val = (A2 / dz2) + (A5 / (2.0 * dz)); - Element(i, x, z, 0, 1, val, MatA); - - // f(i+1,j-1) - val = -1.0 * A3 / (4.0 * dxdz); - Element(i, x, z, 1, -1, val, MatA); - - // f(i+1,j) - val = (A1 / dx2) + (A4 / (2.0 * dx)); - Element(i, x, z, 1, 0, val, MatA); - - // f(i+1,j+1) - val = A3 / (4.0 * dxdz); - Element(i, x, z, 1, 1, val, MatA); - } - // Set Components of RHS Vector - val = b[x][z]; - VecSetValues(bs, 1, &i, &val, INSERT_VALUES); - - // Set Components of Trial Solution Vector - val = x0[x][z]; - VecSetValues(xs, 1, &i, &val, INSERT_VALUES); - i++; - } + // Set the operator matrix + if (fourth_order) { + setFourthOrderMatrix(yindex, inner_X_neumann, outer_X_neumann); + } else { + setSecondOrderMatrix(yindex, inner_X_neumann, outer_X_neumann); } - // X=localmesh->xend+1 to localmesh->LocalNx-1 defines the upper boundary region of the domain. - // Set the values for the outer boundary region - if (localmesh->lastX()) { - for (int x = localmesh->xend + 1; x < localmesh->LocalNx; x++) { - for (int z = 0; z < localmesh->LocalNz; z++) { - // Set Diagonal Values to 1 - PetscScalar val = 1; - Element(i, x, z, 0, 0, val, MatA); - - // If Neumann Boundary Conditions are set. - if (isOuterBoundaryFlagSet(INVERT_AC_GRAD)) { - // Set values corresponding to nodes adjacent in x - if (fourth_order) { - // Fourth Order Accuracy on Boundary - Element(i, x, z, 0, 0, - 25.0 / (12.0 * coords->dx(x, y, z)) / sqrt(coords->g_11(x, y, z)), - MatA); - Element(i, x, z, -1, 0, - -4.0 / coords->dx(x, y, z) / sqrt(coords->g_11(x, y, z)), MatA); - Element(i, x, z, -2, 0, - 3.0 / coords->dx(x, y, z) / sqrt(coords->g_11(x, y, z)), MatA); - Element(i, x, z, -3, 0, - -4.0 / (3.0 * coords->dx(x, y, z)) / sqrt(coords->g_11(x, y, z)), - MatA); - Element(i, x, z, -4, 0, - 1.0 / (4.0 * coords->dx(x, y, z)) / sqrt(coords->g_11(x, y, z)), - MatA); - } else { - // // Second Order Accuracy on Boundary - // Element(i,x,z, 0, 0, 3.0 / (2.0*coords->dx(x,y)), MatA ); - // Element(i,x,z, -1, 0, -2.0 / coords->dx(x,y), MatA ); - // Element(i,x,z, -2, 0, 1.0 / (2.0*coords->dx(x,y)), MatA ); - // Element(i,x,z, -3, 0, 0.0, MatA ); // Reset these elements to 0 - // in case 4th order flag was used previously: not allowed now - // Element(i,x,z, -4, 0, 0.0, MatA ); - // Second Order Accuracy on Boundary, set half-way between grid - // points - Element(i, x, z, 0, 0, - 1.0 / coords->dx(x, y, z) / sqrt(coords->g_11(x, y, z)), MatA); - Element(i, x, z, -1, 0, - -1.0 / coords->dx(x, y, z) / sqrt(coords->g_11(x, y, z)), MatA); - Element(i, x, z, -2, 0, 0.0, MatA); - // Element(i,x,z, -3, 0, 0.0, MatA ); // Reset these elements to 0 - // in case 4th order flag was used previously: not allowed now - // Element(i,x,z, -4, 0, 0.0, MatA ); - } - } else { - if (fourth_order) { - // Set off diagonal elements to zero - Element(i, x, z, -1, 0, 0.0, MatA); - Element(i, x, z, -2, 0, 0.0, MatA); - Element(i, x, z, -3, 0, 0.0, MatA); - Element(i, x, z, -4, 0, 0.0, MatA); - } else { - Element(i, x, z, 0, 0, 0.5, MatA); - Element(i, x, z, -1, 0, 0.5, MatA); - Element(i, x, z, -2, 0, 0., MatA); - } - } - - // Set Components of RHS - // If the inner boundary value should be set by b or x0 - val = 0; - if (isOuterBoundaryFlagSet(INVERT_RHS)) { - val = b[x][z]; - } else if (isOuterBoundaryFlagSet(INVERT_SET)) { - val = x0[x][z]; - } - - // Set components of the RHS (the PETSc vector bs) - // 1 element is being set in row i to val - // INSERT_VALUES replaces existing entries with new values - VecSetValues(bs, 1, &i, &val, INSERT_VALUES); - - // Set components of the and trial solution (the PETSc vector xs) - // 1 element is being set in row i to val - // INSERT_VALUES replaces existing entries with new values - val = x0[x][z]; - VecSetValues(xs, 1, &i, &val, INSERT_VALUES); - - i++; // Increment row in Petsc matrix - } - } - } + operator2D.assemble(); + MatSetBlockSize(*operator2D.get(), 1); - if (i != Iend) { - throw BoutException("Petsc index sanity check failed"); + // Declare KSP Context (abstract PETSc object that manages all Krylov methods) + if (ksp_initialised) { + KSPDestroy(&ksp); } - - // Assemble Matrix - MatAssemblyBegin(MatA, MAT_FINAL_ASSEMBLY); - MatAssemblyEnd(MatA, MAT_FINAL_ASSEMBLY); - - // // Record which flags were used for this matrix - // lastflag = flags; - - // Assemble RHS Vector - VecAssemblyBegin(bs); - VecAssemblyEnd(bs); - - // Assemble Trial Solution Vector - VecAssemblyBegin(xs); - VecAssemblyEnd(xs); + KSPCreate(comm, &ksp); // Configure Linear Solver #if PETSC_VERSION_GE(3, 5, 0) - KSPSetOperators(ksp, MatA, MatA); + KSPSetOperators(ksp, *operator2D.get(), *operator2D.get()); #else - KSPSetOperators(ksp, MatA, MatA, DIFFERENT_NONZERO_PATTERN); + KSPSetOperators(ksp, *operator2D.get(), *operator2D.get(), DIFFERENT_NONZERO_PATTERN); #endif - PC pc; // The preconditioner option + PC pc = nullptr; // The preconditioner option if (direct) { // If a direct solver has been chosen // Get the preconditioner @@ -798,22 +319,37 @@ FieldPerp LaplacePetsc::solve(const FieldPerp& b, const FieldPerp& x0) { } else { KSPSetPCSide(ksp, PC_LEFT); // Left preconditioning } - //ierr = PCShellSetApply(pc,laplacePCapply);CHKERRQ(ierr); - //ierr = PCShellSetContext(pc,this);CHKERRQ(ierr); - //ierr = KSPSetPCSide(ksp, PC_RIGHT);CHKERRQ(ierr); } lib.setOptionsFromInputFile(ksp); } } + PetscVector rhs(b, indexer); + PetscVector guess(x0, indexer); + + // Set boundary conditions + if (!isInnerBoundaryFlagSet(INVERT_RHS)) { + BOUT_FOR_SERIAL(index, indexer->getRegionInnerX()) { + rhs(index) = isInnerBoundaryFlagSet(INVERT_SET) ? x0[index] : 0.0; + } + } + if (!isOuterBoundaryFlagSet(INVERT_RHS)) { + BOUT_FOR_SERIAL(index, indexer->getRegionOuterX()) { + rhs(index) = isInnerBoundaryFlagSet(INVERT_SET) ? x0[index] : 0.0; + } + } + + rhs.assemble(); + guess.assemble(); + // Call the actual solver { - Timer timer("petscsolve"); - KSPSolve(ksp, bs, xs); // Call the solver to solve the system + const Timer timer("petscsolve"); + KSPSolve(ksp, *rhs.get(), *guess.get()); } - KSPConvergedReason reason; + KSPConvergedReason reason = KSP_CONVERGED_ITERATING; KSPGetConvergedReason(ksp, &reason); if (reason == -3) { // Too many iterations, might be fixed by taking smaller timestep throw BoutIterationFail("petsc_laplace: too many iterations"); @@ -824,114 +360,14 @@ FieldPerp LaplacePetsc::solve(const FieldPerp& b, const FieldPerp& x0) { KSPConvergedReasons[reason], static_cast(reason)); } - // Add data to FieldPerp Object - i = Istart; - // Set the inner boundary values - if (localmesh->firstX()) { - for (int x = 0; x < localmesh->xstart; x++) { - for (int z = 0; z < localmesh->LocalNz; z++) { - PetscScalar val = 0; - VecGetValues(xs, 1, &i, &val); - sol[x][z] = val; - i++; // Increment row in Petsc matrix - } - } - } - - // Set the main domain values - for (int x = localmesh->xstart; x <= localmesh->xend; x++) { - for (int z = 0; z < localmesh->LocalNz; z++) { - PetscScalar val = 0; - VecGetValues(xs, 1, &i, &val); - sol[x][z] = val; - i++; // Increment row in Petsc matrix - } - } - - // Set the outer boundary values - if (localmesh->lastX()) { - for (int x = localmesh->xend + 1; x < localmesh->LocalNx; x++) { - for (int z = 0; z < localmesh->LocalNz; z++) { - PetscScalar val = 0; - VecGetValues(xs, 1, &i, &val); - sol[x][z] = val; - i++; // Increment row in Petsc matrix - } - } - } - - if (i != Iend) { - throw BoutException("Petsc index sanity check 2 failed"); - } - + auto sol = guess.toField(); + sol.setIndex(yindex); checkData(sol); // Return the solution return sol; } -/*! - * Sets the elements of the matrix A, which is used to solve the problem Ax=b. - * - * \param[in] - * i - * The row of the PETSc matrix - * \param[in] x Local x index of the mesh - * \param[in] z Local z index of the mesh - * \param[in] xshift The shift in rows from the index x - * \param[in] zshift The shift in columns from the index z - * \param[in] ele Value of the element - * \param[in] MatA The matrix A used in the inversion - * - * \param[out] MatA The matrix A used in the inversion - */ -void LaplacePetsc::Element(int i, int x, int z, int xshift, int zshift, PetscScalar ele, - Mat& MatA) { - - // Need to convert LOCAL x to GLOBAL x in order to correctly calculate - // PETSC Matrix Index. - int xoffset = Istart / meshz; - if (Istart % meshz != 0) { - throw BoutException("Petsc index sanity check 3 failed"); - } - - // Calculate the row to be set - int row_new = x + xshift; // should never be out of range. - if (!localmesh->firstX()) { - row_new += (xoffset - localmesh->xstart); - } - - // Calculate the column to be set - int col_new = z + zshift; - if (col_new < 0) { - col_new += meshz; - } else if (col_new > meshz - 1) { - col_new -= meshz; - } - - // Convert to global indices - int index = (row_new * meshz) + col_new; - -#if CHECK > 2 - if (!std::isfinite(ele)) { - throw BoutException("Non-finite element at x={:d}, z={:d}, row={:d}, col={:d}\n", x, - z, i, index); - } -#endif - - /* Inserts or adds a block of values into a matrix - * Input: - * MatA - The matrix to set the values in - * 1 - The number of rows to be set - * &i - The global index of the row - * 1 - The number of columns to be set - * &index - The global index of the column - * &ele - The vlaue to be set - * INSERT_VALUES replaces existing entries with new values - */ - MatSetValues(MatA, 1, &i, 1, &index, &ele, INSERT_VALUES); -} - /*! * Set the matrix components of A in Ax=b, solving * D*Laplace_perp(x) + (1/C1)Grad_perp(C2)*Grad_perp(x) + Ax = B @@ -964,19 +400,19 @@ void LaplacePetsc::Element(int i, int x, int z, int xshift, int zshift, PetscSca * \param[out] coef5 Convenient variable used to set matrix * (see manual for details) */ -void LaplacePetsc::Coeffs(int x, int y, int z, BoutReal& coef1, BoutReal& coef2, - BoutReal& coef3, BoutReal& coef4, BoutReal& coef5) { +LaplacePetsc::CoeffsA LaplacePetsc::Coeffs(Ind3D i) { + const auto x = i.x(); - coef1 = coords->g11(x, y, z); // X 2nd derivative coefficient - coef2 = coords->g33(x, y, z); // Z 2nd derivative coefficient - coef3 = 2. * coords->g13(x, y, z); // X-Z mixed derivative coefficient + BoutReal coef1 = coords->g11[i]; // X 2nd derivative coefficient + BoutReal coef2 = coords->g33[i]; // Z 2nd derivative coefficient + BoutReal coef3 = 2. * coords->g13[i]; // X-Z mixed derivative coefficient - coef4 = 0.0; - coef5 = 0.0; + BoutReal coef4 = 0.0; + BoutReal coef5 = 0.0; // If global flag all_terms are set (true by default) if (all_terms) { - coef4 = coords->G1(x, y, z); // X 1st derivative - coef5 = coords->G3(x, y, z); // Z 1st derivative + coef4 = coords->G1[i]; // X 1st derivative + coef5 = coords->G3[i]; // Z 1st derivative ASSERT3(std::isfinite(coef4)); ASSERT3(std::isfinite(coef5)); @@ -985,71 +421,48 @@ void LaplacePetsc::Coeffs(int x, int y, int z, BoutReal& coef1, BoutReal& coef2, if (nonuniform) { // non-uniform mesh correction if ((x != 0) && (x != (localmesh->LocalNx - 1))) { - coef4 -= 0.5 - * ((coords->dx(x + 1, y, z) - coords->dx(x - 1, y, z)) - / SQ(coords->dx(x, y, z))) + coef4 -= 0.5 * ((coords->dx[i.xp()] - coords->dx[i.xm()]) / SQ(coords->dx[i])) * coef1; // BOUT-06 term } } if (localmesh->IncIntShear) { // d2dz2 term - coef2 += coords->g11(x, y, z) * coords->IntShiftTorsion(x, y, z) - * coords->IntShiftTorsion(x, y, z); + coef2 += coords->g11[i] * coords->IntShiftTorsion[i] * coords->IntShiftTorsion[i]; // Mixed derivative coef3 = 0.0; // This cancels out } if (issetD) { - coef1 *= D(x, y, z); - coef2 *= D(x, y, z); - coef3 *= D(x, y, z); - coef4 *= D(x, y, z); - coef5 *= D(x, y, z); + coef1 *= D[i]; + coef2 *= D[i]; + coef3 *= D[i]; + coef4 *= D[i]; + coef5 *= D[i]; } // A second/fourth order derivative term if (issetC) { - // if( (x > 0) && (x < (localmesh->LocalNx-1)) ) //Valid if doing second order derivative, not if fourth: should only be called for xstart<=x<=xend anyway if ((x > 1) && (x < (localmesh->LocalNx - 2))) { - int zp = z + 1; // z plus 1 - if (zp > meshz - 1) { - zp -= meshz; - } - int zm = z - 1; // z minus 1 - if (zm < 0) { - zm += meshz; - } - BoutReal ddx_C; - BoutReal ddz_C; + BoutReal ddx_C = BoutNaN; + BoutReal ddz_C = BoutNaN; if (fourth_order) { - int zpp = z + 2; // z plus 1 plus 1 - if (zpp > meshz - 1) { - zpp -= meshz; - } - int zmm = z - 2; // z minus 1 minus 1 - if (zmm < 0) { - zmm += meshz; - } // Fourth order discretization of C in x - ddx_C = (-C2(x + 2, y, z) + 8. * C2(x + 1, y, z) - 8. * C2(x - 1, y, z) - + C2(x - 2, y, z)) - / (12. * coords->dx(x, y, z) * (C1(x, y, z))); + ddx_C = (-C2[i.xpp()] + (8. * C2[i.xp()]) - (8. * C2[i.xm()]) + C2[i.xmm()]) + / (12. * coords->dx[i] * (C1[i])); // Fourth order discretization of C in z - ddz_C = (-C2(x, y, zpp) + 8. * C2(x, y, zp) - 8. * C2(x, y, zm) + C2(x, y, zmm)) - / (12. * coords->dz(x, y, z) * (C1(x, y, z))); + ddz_C = (-C2[i.zpp()] + (8. * C2[i.zp()]) - (8. * C2[i.zm()]) + C2[i.zmm()]) + / (12. * coords->dz[i] * (C1[i])); } else { // Second order discretization of C in x - ddx_C = (C2(x + 1, y, z) - C2(x - 1, y, z)) - / (2. * coords->dx(x, y, z) * (C1(x, y, z))); + ddx_C = (C2[i.xp()] - C2[i.xm()]) / (2. * coords->dx[i] * (C1[i])); // Second order discretization of C in z - ddz_C = - (C2(x, y, zp) - C2(x, y, zm)) / (2. * coords->dz(x, y, z) * (C1(x, y, z))); + ddz_C = (C2[i.zp()] - C2[i.zm()]) / (2. * coords->dz[i] * (C1[i])); } - coef4 += coords->g11(x, y, z) * ddx_C + coords->g13(x, y, z) * ddz_C; - coef5 += coords->g13(x, y, z) * ddx_C + coords->g33(x, y, z) * ddz_C; + coef4 += (coords->g11[i] * ddx_C) + (coords->g13[i] * ddz_C); + coef5 += (coords->g13[i] * ddx_C) + (coords->g33[i] * ddz_C); } } @@ -1063,99 +476,199 @@ void LaplacePetsc::Coeffs(int x, int y, int z, BoutReal& coef1, BoutReal& coef2, */ if (issetE) { // These coefficients are 0 by default - coef4 += Ex(x, y, z); - coef5 += Ez(x, y, z); + coef4 += Ex[i]; + coef5 += Ez[i]; } -} - -void LaplacePetsc::vecToField(Vec xs, FieldPerp& f) { - ASSERT1(localmesh == f.getMesh()); + return {coef1, coef2, coef3, coef4, coef5}; +} - f.allocate(); - int i = Istart; - if (localmesh->firstX()) { - for (int x = 0; x < localmesh->xstart; x++) { - for (int z = 0; z < localmesh->LocalNz; z++) { - PetscScalar val; - VecGetValues(xs, 1, &i, &val); - f[x][z] = val; - i++; // Increment row in Petsc matrix - } +void LaplacePetsc::setSecondOrderMatrix(int y, bool inner_X_neumann, + bool outer_X_neumann) { + // Set the boundaries + if (inner_X_neumann) { + const auto dx = sliceXZ(coords->dx, y); + const auto g11 = sliceXZ(coords->g11, y); + + BOUT_FOR_SERIAL(i, indexer->getRegionInnerX()) { + const auto factor = 1. / dx[i] / std::sqrt(g11[i]); + operator2D(i, i) = -factor; + operator2D(i, i.xp()) = factor; } - } - - for (int x = localmesh->xstart; x <= localmesh->xend; x++) { - for (int z = 0; z < localmesh->LocalNz; z++) { - PetscScalar val; - VecGetValues(xs, 1, &i, &val); - f[x][z] = val; - i++; // Increment row in Petsc matrix + } else { + BOUT_FOR_SERIAL(i, indexer->getRegionInnerX()) { + operator2D(i, i) = 0.5; + operator2D(i, i.xp()) = 0.5; } } + if (outer_X_neumann) { + const auto dx = sliceXZ(coords->dx, y); + const auto g11 = sliceXZ(coords->g11, y); - if (localmesh->lastX()) { - for (int x = localmesh->xend + 1; x < localmesh->LocalNx; x++) { - for (int z = 0; z < localmesh->LocalNz; z++) { - PetscScalar val; - VecGetValues(xs, 1, &i, &val); - f[x][z] = val; - i++; // Increment row in Petsc matrix - } + BOUT_FOR_SERIAL(i, indexer->getRegionOuterX()) { + const auto factor = 1. / dx[i] / std::sqrt(g11[i]); + operator2D(i, i) = factor; + operator2D(i, i.xm()) = -factor; + } + } else { + BOUT_FOR_SERIAL(i, indexer->getRegionOuterX()) { + operator2D(i, i) = 0.5; + operator2D(i, i.xm()) = 0.5; } } - ASSERT1(i == Iend); -} -void LaplacePetsc::fieldToVec(const FieldPerp& f, Vec bs) { - ASSERT1(localmesh == f.getMesh()); + // Set the interior region + BOUT_FOR_SERIAL(l, indexer->getRegionNobndry()) { + const auto i = localmesh->indPerpto3D(l, y); - int i = Istart; - if (localmesh->firstX()) { - for (int x = 0; x < localmesh->xstart; x++) { - for (int z = 0; z < localmesh->LocalNz; z++) { - PetscScalar val = f[x][z]; - VecSetValues(bs, 1, &i, &val, INSERT_VALUES); - i++; // Increment row in Petsc matrix - } - } + // NOTE: Only A0 is the A from setCoefA () + const BoutReal A0 = A[i]; + + ASSERT3(std::isfinite(A0)); + + // Set the matrix coefficients + const auto [A1, A2, A3, A4, A5] = Coeffs(i); + + ASSERT3(std::isfinite(A1)); + ASSERT3(std::isfinite(A2)); + ASSERT3(std::isfinite(A3)); + ASSERT3(std::isfinite(A4)); + ASSERT3(std::isfinite(A5)); + + const BoutReal dx = coords->dx[i]; + const BoutReal dx2 = SQ(dx); + const BoutReal dz = coords->dz[i]; + const BoutReal dz2 = SQ(dz); + const BoutReal dxdz = dx * dz; + operator2D(l, l) = A0 - (2.0 * ((A1 / dx2) + (A2 / dz2))); + operator2D(l, l.xm().zm()) = A3 / (4.0 * dxdz); + operator2D(l, l.xm()) = (A1 / dx2) - (A4 / (2.0 * dx)); + operator2D(l, l.xm().zp()) = -1.0 * A3 / (4.0 * dxdz); + operator2D(l, l.zm()) = (A2 / dz2) - (A5 / (2.0 * dz)); + operator2D(l, l.zp()) = (A2 / dz2) + (A5 / (2.0 * dz)); + operator2D(l, l.xp().zm()) = -1.0 * A3 / (4.0 * dxdz); + operator2D(l, l.xp()) = (A1 / dx2) + (A4 / (2.0 * dx)); + operator2D(l, l.xp().zp()) = A3 / (4.0 * dxdz); } +} - for (int x = localmesh->xstart; x <= localmesh->xend; x++) { - for (int z = 0; z < localmesh->LocalNz; z++) { - PetscScalar val = f[x][z]; - VecSetValues(bs, 1, &i, &val, INSERT_VALUES); - i++; // Increment row in Petsc matrix +void LaplacePetsc::setFourthOrderMatrix(int y, bool inner_X_neumann, + bool outer_X_neumann) { + + // Set boundaries + if (inner_X_neumann) { + const auto dx = sliceXZ(coords->dx, y); + const auto g11 = sliceXZ(coords->g11, y); + + BOUT_FOR_SERIAL(i, indexer->getRegionInnerX()) { + const auto factor = 1. / dx[i] / std::sqrt(g11[i]); + operator2D(i, i) = (-25.0 / 12.0) * factor; + operator2D(i, i.xp(1)) = 4.0 * factor; + operator2D(i, i.xp(2)) = -3.0 * factor; + operator2D(i, i.xp(3)) = (4.0 / 3.0) * factor; + operator2D(i, i.xp(4)) = (-1.0 / 4.0) * factor; + } + } else { + BOUT_FOR_SERIAL(i, indexer->getRegionInnerX()) { + operator2D(i, i) = 1.0; + operator2D(i, i.xp(1)) = 0.0; + operator2D(i, i.xp(2)) = 0.0; + operator2D(i, i.xp(3)) = 0.0; + operator2D(i, i.xp(4)) = 0.0; } } - if (localmesh->lastX()) { - for (int x = localmesh->xend + 1; x < localmesh->LocalNx; x++) { - for (int z = 0; z < localmesh->LocalNz; z++) { - PetscScalar val = f[x][z]; - VecSetValues(bs, 1, &i, &val, INSERT_VALUES); - i++; // Increment row in Petsc matrix - } + if (outer_X_neumann) { + const auto dx = sliceXZ(coords->dx, y); + const auto g11 = sliceXZ(coords->g11, y); + + BOUT_FOR_SERIAL(i, indexer->getRegionOuterX()) { + const auto factor = 1. / dx[i] / std::sqrt(g11[i]); + operator2D(i, i) = (25.0 / 12.0) * factor; + operator2D(i, i.xm(1)) = -4.0 * factor; + operator2D(i, i.xm(2)) = 3.0 * factor; + operator2D(i, i.xm(3)) = (-4.0 / 3.0) * factor; + operator2D(i, i.xm(4)) = (1.0 / 4.0) * factor; + } + } else { + BOUT_FOR_SERIAL(i, indexer->getRegionOuterX()) { + operator2D(i, i) = 1.0; + operator2D(i, i.xm(1)) = 0.0; + operator2D(i, i.xm(2)) = 0.0; + operator2D(i, i.xm(3)) = 0.0; + operator2D(i, i.xm(4)) = 0.0; } } - ASSERT1(i == Iend); - VecAssemblyBegin(bs); - VecAssemblyEnd(bs); + // Set interior region + BOUT_FOR_SERIAL(l, indexer->getRegionNobndry()) { + const auto i = localmesh->indPerpto3D(l, y); + + // NOTE: Only A0 is the A from setCoefA () + const BoutReal A0 = A[i]; + + ASSERT3(std::isfinite(A0)); + + // Set the matrix coefficients + const auto [A1, A2, A3, A4, A5] = Coeffs(i); + + ASSERT3(std::isfinite(A1)); + ASSERT3(std::isfinite(A2)); + ASSERT3(std::isfinite(A3)); + ASSERT3(std::isfinite(A4)); + ASSERT3(std::isfinite(A5)); + + const BoutReal dx = coords->dx[i]; + const BoutReal dx2 = SQ(dx); + const BoutReal dz = coords->dz[i]; + const BoutReal dz2 = SQ(dz); + const BoutReal dxdz = dx * dz; + + operator2D(l, l) = A0 - ((5.0 / 2.0) * ((A1 / dx2) + (A2 / dz2))); + operator2D(l, l.xmm().zmm()) = A3 / (144.0 * dxdz); + operator2D(l, l.xmm().zm()) = -1.0 * A3 / (18.0 * dxdz); + operator2D(l, l.xmm()) = (1.0 / 12.0) * ((-1.0 * A1 / dx2) + (A4 / dx)); + operator2D(l, l.xmm().zp()) = A3 / (18.0 * dxdz); + operator2D(l, l.xmm().zpp()) = -1.0 * A3 / (144.0 * dxdz); + operator2D(l, l.xm().zmm()) = -1.0 * A3 / (18.0 * dxdz); + operator2D(l, l.xm().zm()) = 4.0 * A3 / (9.0 * dxdz); + operator2D(l, l.xm()) = (4.0 * A1 / (3.0 * dx2)) - (2.0 * A4 / (3.0 * dx)); + operator2D(l, l.xm().zp()) = -4.0 * A3 / (9.0 * dxdz); + operator2D(l, l.xm().zpp()) = A3 / (18.0 * dxdz); + operator2D(l, l.zmm()) = (1.0 / 12.0) * ((-1.0 * A2 / dz2) + (A5 / dz)); + operator2D(l, l.zm()) = (4.0 * A2 / (3.0 * dz2)) - (2.0 * A5 / (3.0 * dz)); + operator2D(l, l.zp()) = (4.0 * A2 / (3.0 * dz2)) + (2.0 * A5 / (3.0 * dz)); + operator2D(l, l.zpp()) = (-1.0 / 12.0) * ((A2 / dz2) + (A5 / dz)); + operator2D(l, l.xp().zmm()) = A3 / (18.0 * dxdz); + operator2D(l, l.xp().zm()) = -4.0 * A3 / (9.0 * dxdz); + operator2D(l, l.xp()) = (4.0 * A1 / (3.0 * dx2)) + (2.0 * A4 / (3.0 * dx)); + operator2D(l, l.xp().zp()) = 4.0 * A3 / (9.0 * dxdz); + operator2D(l, l.xp().zpp()) = -1.0 * A3 / (18.0 * dxdz); + operator2D(l, l.xpp().zmm()) = -1.0 * A3 / (144.0 * dxdz); + operator2D(l, l.xpp().zm()) = A3 / (18.0 * dxdz); + operator2D(l, l.xpp()) = (-1.0 / 12.0) * ((A1 / dx2) + (A4 / dx)); + operator2D(l, l.xpp().zp()) = -1.0 * A3 / (18.0 * dxdz); + operator2D(l, l.xpp().zpp()) = A3 / (144.0 * dxdz); + } } /// Preconditioner function int LaplacePetsc::precon(Vec x, Vec y) { - // Get field to be preconditioned - FieldPerp xfield; - vecToField(x, xfield); - xfield.setIndex(sol.getIndex()); // y index stored in sol variable + FieldPerp xfield(indexer->getMesh(), location, yindex); + xfield = 0.0; + + BOUT_FOR_SERIAL(i, indexer->getRegionAll()) { + const auto ind = indexer->getGlobal(i); + PetscScalar val = BoutNaN; + VecGetValues(x, 1, &ind, &val); + xfield[i] = val; + } // Call the preconditioner solver - FieldPerp yfield = pcsolve->solve(xfield); + const FieldPerp yfield = pcsolve->solve(xfield); + + VecCopy(*PetscVector{yfield, indexer}.get(), y); - // Put result into y - fieldToVec(yfield, y); return 0; } diff --git a/src/invert/laplace/impls/petsc/petsc_laplace.hxx b/src/invert/laplace/impls/petsc/petsc_laplace.hxx index 1d56abd00b..abef05c0ae 100644 --- a/src/invert/laplace/impls/petsc/petsc_laplace.hxx +++ b/src/invert/laplace/impls/petsc/petsc_laplace.hxx @@ -41,11 +41,15 @@ RegisterUnavailableLaplace registerlaplacepetsc(LAPLACE_PETSC, #else +#include #include +#include #include #include #include +#include #include +#include #include @@ -59,12 +63,7 @@ class LaplacePetsc : public Laplacian { public: LaplacePetsc(Options* opt = nullptr, const CELL_LOC loc = CELL_CENTRE, Mesh* mesh_in = nullptr, Solver* solver = nullptr); - ~LaplacePetsc() { - KSPDestroy(&ksp); - VecDestroy(&xs); - VecDestroy(&bs); - MatDestroy(&MatA); - } + ~LaplacePetsc() override; using Laplacian::setCoefA; using Laplacian::setCoefC; @@ -199,9 +198,21 @@ public: int precon(Vec x, Vec y); ///< Preconditioner function private: - void Element(int i, int x, int z, int xshift, int zshift, PetscScalar ele, Mat& MatA); - void Coeffs(int x, int y, int z, BoutReal& A1, BoutReal& A2, BoutReal& A3, BoutReal& A4, - BoutReal& A5); + struct CoeffsA { + BoutReal A1; + BoutReal A2; + BoutReal A3; + BoutReal A4; + BoutReal A5; + }; + + /// Calculate the coefficients ``A1-5`` + CoeffsA Coeffs(Ind3D i); + + /// Set `operator2D` for the second order scheme + void setSecondOrderMatrix(int y, bool inner_X_neumann, bool outer_X_neumann); + /// Set `operator2D` for the fourth order scheme + void setFourthOrderMatrix(int y, bool inner_X_neumann, bool outer_X_neumann); /* Ex and Ez * Additional 1st derivative terms to allow for solution field to be @@ -218,19 +229,14 @@ private: bool issetC; bool issetE; - FieldPerp sol; // solution Field + /// Y-index of solution field. + int yindex = -1; - // Istart is the first row of MatA owned by the process, Iend is 1 greater than the last row. - int Istart, Iend; - - int meshx, meshz, size, - localN; // Mesh sizes, total size, no of points on this processor MPI_Comm comm; - Mat MatA; - Vec xs, bs; // Solution and RHS vectors - KSP ksp; + KSP ksp = nullptr; ///< PETSc solver + bool ksp_initialised = false; - Options* opts; // Laplace Section Options Object + Options* opts; ///< Laplace Section Options Object std::string ksptype; ///< KSP solver type std::string pctype; ///< Preconditioner type @@ -246,14 +252,13 @@ private: bool direct; //Use direct LU solver if true. bool fourth_order; + IndexerPtr indexer; + PetscMatrix operator2D; PetscLib lib; bool rightprec; // Right preconditioning std::unique_ptr pcsolve; // Laplacian solver for preconditioning - void vecToField(Vec x, FieldPerp& f); // Copy a vector into a fieldperp - void fieldToVec(const FieldPerp& f, Vec x); // Copy a fieldperp into a vector - static constexpr int implemented_flags = INVERT_START_NEW; static constexpr int implemented_boundary_flags = INVERT_AC_GRAD | INVERT_SET | INVERT_RHS; diff --git a/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx b/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx index efca1d70be..3be5e43e63 100644 --- a/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx +++ b/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx @@ -174,7 +174,6 @@ void setBC(PetscVector& rhs, const Field3D& b_in, } Field3D LaplacePetsc3dAmg::solve(const Field3D& b_in, const Field3D& x0) { - AUTO_TRACE(); // Timing reported in the log files. Includes any matrix construction. // The timing for just the solve phase can be retrieved from the "petscsolve" diff --git a/src/invert/laplace/impls/serial_band/serial_band.cxx b/src/invert/laplace/impls/serial_band/serial_band.cxx index d7b4ac5d3b..0cf8d7259d 100644 --- a/src/invert/laplace/impls/serial_band/serial_band.cxx +++ b/src/invert/laplace/impls/serial_band/serial_band.cxx @@ -45,6 +45,9 @@ LaplaceSerialBand::LaplaceSerialBand(Options* opt, const CELL_LOC loc, Mesh* mesh_in, Solver* UNUSED(solver)) : Laplacian(opt, loc, mesh_in), Acoef(0.0), Ccoef(1.0), Dcoef(1.0) { + + bout::fft::assertZSerial(*localmesh, "`band` inversion"); + Acoef.setLocation(location); Ccoef.setLocation(location); Dcoef.setLocation(location); diff --git a/src/invert/laplace/impls/serial_tri/serial_tri.cxx b/src/invert/laplace/impls/serial_tri/serial_tri.cxx index 0fb9294d76..d051ce0c1e 100644 --- a/src/invert/laplace/impls/serial_tri/serial_tri.cxx +++ b/src/invert/laplace/impls/serial_tri/serial_tri.cxx @@ -24,6 +24,10 @@ * **************************************************************************/ +#include "bout/build_defines.hxx" + +#if not BOUT_USE_METRIC_3D + #include "serial_tri.hxx" #include "bout/globals.hxx" @@ -33,14 +37,15 @@ #include #include #include -#include -#include - #include +#include LaplaceSerialTri::LaplaceSerialTri(Options* opt, CELL_LOC loc, Mesh* mesh_in, Solver* UNUSED(solver)) : Laplacian(opt, loc, mesh_in), A(0.0), C(1.0), D(1.0) { + + bout::fft::assertZSerial(*localmesh, "`tri` inversion"); + A.setLocation(location); C.setLocation(location); D.setLocation(location); @@ -246,3 +251,5 @@ FieldPerp LaplaceSerialTri::solve(const FieldPerp& b, const FieldPerp& x0) { return x; // Result of the inversion } + +#endif // BOUT_USE_METRIC_3D diff --git a/src/invert/laplace/impls/serial_tri/serial_tri.hxx b/src/invert/laplace/impls/serial_tri/serial_tri.hxx index 5b0419fa27..332c272f3f 100644 --- a/src/invert/laplace/impls/serial_tri/serial_tri.hxx +++ b/src/invert/laplace/impls/serial_tri/serial_tri.hxx @@ -29,8 +29,19 @@ class LaplaceSerialTri; #ifndef BOUT_SERIAL_TRI_H #define BOUT_SERIAL_TRI_H +#include "bout/build_defines.hxx" +#include "bout/invert_laplace.hxx" + +#if BOUT_USE_METRIC_3D + +namespace { +const RegisterUnavailableLaplace + registerlaplacetri(LAPLACE_TRI, "BOUT++ was configured with 3D metrics"); +} + +#else + #include -#include #include namespace { @@ -41,7 +52,7 @@ class LaplaceSerialTri : public Laplacian { public: LaplaceSerialTri(Options* opt = nullptr, const CELL_LOC loc = CELL_CENTRE, Mesh* mesh_in = nullptr, Solver* solver = nullptr); - ~LaplaceSerialTri(){}; + ~LaplaceSerialTri() {}; using Laplacian::setCoefA; void setCoefA(const Field2D& val) override { @@ -80,4 +91,6 @@ private: Field2D A, C, D; }; +#endif // BOUT_USE_METRIC_3D + #endif // BOUT_SERIAL_TRI_H diff --git a/src/invert/laplace/impls/spt/spt.cxx b/src/invert/laplace/impls/spt/spt.cxx index 2e4c844c94..34a1a64cae 100644 --- a/src/invert/laplace/impls/spt/spt.cxx +++ b/src/invert/laplace/impls/spt/spt.cxx @@ -31,6 +31,10 @@ * */ +#include "bout/build_defines.hxx" + +#if not BOUT_USE_METRIC_3D + #include #include #include @@ -45,6 +49,9 @@ LaplaceSPT::LaplaceSPT(Options* opt, const CELL_LOC loc, Mesh* mesh_in, Solver* UNUSED(solver)) : Laplacian(opt, loc, mesh_in), Acoef(0.0), Ccoef(1.0), Dcoef(1.0) { + + bout::fft::assertZSerial(*localmesh, "`spt` inversion"); + Acoef.setLocation(location); Ccoef.setLocation(location); Dcoef.setLocation(location); @@ -95,7 +102,7 @@ FieldPerp LaplaceSPT::solve(const FieldPerp& b, const FieldPerp& x0) { if (isInnerBoundaryFlagSetOnFirstX(INVERT_SET)) { // Copy x0 inner boundary into bs for (int ix = 0; ix < xbndry; ix++) { - for (int iz = 0; iz < localmesh->LocalNz; iz++) { + for (int iz = localmesh->zstart; iz <= localmesh->zend; iz++) { bs[ix][iz] = x0[ix][iz]; } } @@ -103,7 +110,7 @@ FieldPerp LaplaceSPT::solve(const FieldPerp& b, const FieldPerp& x0) { if (isOuterBoundaryFlagSetOnLastX(INVERT_SET)) { // Copy x0 outer boundary into bs for (int ix = localmesh->LocalNx - 1; ix >= localmesh->LocalNx - xbndry; ix--) { - for (int iz = 0; iz < localmesh->LocalNz; iz++) { + for (int iz = localmesh->zstart; iz <= localmesh->zend; iz++) { bs[ix][iz] = x0[ix][iz]; } } @@ -181,7 +188,7 @@ Field3D LaplaceSPT::solve(const Field3D& b, const Field3D& x0) { // Copy x0 inner boundary into bs for (int ix = 0; ix < xbndry; ix++) { for (int iy = 0; iy < localmesh->LocalNy; iy++) { - for (int iz = 0; iz < localmesh->LocalNz; iz++) { + for (int iz = localmesh->zstart; iz <= localmesh->zend; iz++) { bs(ix, iy, iz) = x0(ix, iy, iz); } } @@ -191,7 +198,7 @@ Field3D LaplaceSPT::solve(const Field3D& b, const Field3D& x0) { // Copy x0 outer boundary into bs for (int ix = localmesh->LocalNx - 1; ix >= localmesh->LocalNx - xbndry; ix--) { for (int iy = 0; iy < localmesh->LocalNy; iy++) { - for (int iz = 0; iz < localmesh->LocalNz; iz++) { + for (int iz = localmesh->zstart; iz <= localmesh->zend; iz++) { bs(ix, iy, iz) = x0(ix, iy, iz); } } @@ -341,14 +348,14 @@ int LaplaceSPT::start(const FieldPerp& b, SPT_data& data) { // Send data localmesh->sendXOut(std::begin(data.buffer), 4 * (maxmode + 1), data.comm_tag); - } else if (localmesh->PE_XIND == 1) { + } else if (localmesh->getXProcIndex() == 1) { // Post a receive data.recv_handle = localmesh->irecvXIn(std::begin(data.buffer), 4 * (maxmode + 1), data.comm_tag); } data.proc++; // Now moved onto the next processor - if (localmesh->NXPE == 2) { + if (localmesh->getNXPE() == 2) { data.dir = -1; // Special case. Otherwise reversal handled in spt_continue } @@ -366,7 +373,7 @@ int LaplaceSPT::next(SPT_data& data) { return 1; } - if (localmesh->PE_XIND == data.proc) { + if (localmesh->getXProcIndex() == data.proc) { /// This processor's turn to do inversion // Wait for data to arrive @@ -450,7 +457,7 @@ int LaplaceSPT::next(SPT_data& data) { } } - if (localmesh->PE_XIND != 0) { // If not finished yet + if (localmesh->getXProcIndex() != 0) { // If not finished yet /// Send data if (data.dir > 0) { @@ -460,7 +467,7 @@ int LaplaceSPT::next(SPT_data& data) { } } - } else if (localmesh->PE_XIND == data.proc + data.dir) { + } else if (localmesh->getXProcIndex() == data.proc + data.dir) { // This processor is next, post receive if (data.dir > 0) { @@ -474,7 +481,7 @@ int LaplaceSPT::next(SPT_data& data) { data.proc += data.dir; - if (data.proc == localmesh->NXPE - 1) { + if (data.proc == localmesh->getNXPE() - 1) { data.dir = -1; // Reverses direction at the end } @@ -519,7 +526,7 @@ void LaplaceSPT::finish(SPT_data& data, FieldPerp& x) { if (!localmesh->firstX()) { // Set left boundary to zero (Prevent unassigned values in corners) for (int ix = 0; ix < localmesh->xstart; ix++) { - for (int kz = 0; kz < localmesh->LocalNz; kz++) { + for (int kz = localmesh->zstart; kz <= localmesh->zend; kz++) { x(ix, kz) = 0.0; } } @@ -527,7 +534,7 @@ void LaplaceSPT::finish(SPT_data& data, FieldPerp& x) { if (!localmesh->lastX()) { // Same for right boundary for (int ix = localmesh->xend + 1; ix < localmesh->LocalNx; ix++) { - for (int kz = 0; kz < localmesh->LocalNz; kz++) { + for (int kz = localmesh->zstart; kz <= localmesh->zend; kz++) { x(ix, kz) = 0.0; } } @@ -550,3 +557,5 @@ void LaplaceSPT::SPT_data::allocate(int mm, int nx) { buffer.reallocate(4 * mm); } + +#endif // BOUT_USE_METRIC_3D diff --git a/src/invert/laplace/impls/spt/spt.hxx b/src/invert/laplace/impls/spt/spt.hxx index a9d5b2583f..e951f431e7 100644 --- a/src/invert/laplace/impls/spt/spt.hxx +++ b/src/invert/laplace/impls/spt/spt.hxx @@ -1,16 +1,16 @@ /************************************************************************** - * Perpendicular Laplacian inversion. + * Perpendicular Laplacian inversion. * PARALLEL CODE - SIMPLE ALGORITHM - * + * * I'm just calling this Simple Parallel Tridag. Naive parallelisation of * the serial code. For use as a reference case. - * + * * Overlap calculation / communication of poloidal slices to achieve some * parallelism. * * Changelog * --------- - * + * * 2014-06 Ben Dudson * * Removed static variables in functions, changing to class members. * @@ -18,7 +18,7 @@ * Copyright 2010 B.D.Dudson, S.Farley, M.V.Umansky, X.Q.Xu * * Contact: Ben Dudson, bd512@york.ac.uk - * + * * This file is part of BOUT++. * * BOUT++ is free software: you can redistribute it and/or modify @@ -39,10 +39,20 @@ class LaplaceSPT; #ifndef BOUT_SPT_H +#include "bout/build_defines.hxx" +#include "bout/invert_laplace.hxx" + +#if BOUT_USE_METRIC_3D + +namespace { +const RegisterUnavailableLaplace + registerlaplacespt(LAPLACE_SPT, "BOUT++ was configured with 3D metrics"); +} + +#else #define BOUT_SPT_H #include -#include #include #include #include @@ -132,8 +142,8 @@ private: Array buffer; }; - int ys, ye; // Range of Y indices - SPT_data slicedata; // Used to solve for a single FieldPerp + int ys, ye; // Range of Y indices + SPT_data slicedata; // Used to solve for a single FieldPerp Array alldata; // Used to solve a Field3D Array dc1d; ///< 1D in Z for taking FFTs @@ -153,7 +163,9 @@ private: namespace { // Note: After class definition so compiler knows that // registered class is derived from Laplacian -RegisterLaplace registerlaplacespt(LAPLACE_SPT); +const RegisterLaplace registerlaplacespt(LAPLACE_SPT); } // namespace +#endif // BOUT_USE_METRIC_3D + #endif // BOUT_SPT_H diff --git a/src/invert/laplace/invert_laplace.cxx b/src/invert/laplace/invert_laplace.cxx index 4032499781..e9182042de 100644 --- a/src/invert/laplace/invert_laplace.cxx +++ b/src/invert/laplace/invert_laplace.cxx @@ -33,17 +33,18 @@ #include #include +#include #include #include #include #include -#include #include #include #include #include #include #include +#include #include // Implementations: @@ -90,19 +91,12 @@ Laplacian::Laplacian(Options* options, const CELL_LOC loc, Mesh* mesh_in, const BoutReal filter = (*options)["filter"] .doc("Fraction of Z modes to filter out. Between 0 and 1") .withDefault(0.0); - const int ncz = localmesh->LocalNz; + const int ncz = localmesh->zend - localmesh->zstart + 1; // convert filtering into an integer number of modes maxmode = (*options)["maxmode"] .doc("The maximum Z mode to solve for") - .withDefault(ROUND((1.0 - filter) * static_cast(ncz / 2))); - - // Clamp maxmode between 0 and nz/2 - if (maxmode < 0) { - maxmode = 0; - } - if (maxmode > ncz / 2) { - maxmode = ncz / 2; - } + .withDefault(ROUND((1.0 - filter) * static_cast(ncz) / 2.0)); + maxmode = std::clamp(maxmode, 0, ncz / 2); low_mem = (*options)["low_mem"] .doc("If true, reduce the amount of memory used") @@ -155,7 +149,6 @@ void Laplacian::cleanup() { instance.reset(); } **********************************************************************************/ Field3D Laplacian::solve(const Field3D& b) { - TRACE("Laplacian::solve(Field3D)"); ASSERT1(b.getLocation() == location); ASSERT1(localmesh == b.getMesh()); @@ -216,7 +209,6 @@ Field2D Laplacian::solve(const Field2D& b) { * \returns x All the y-slices of x_slice in the equation A*x_slice = b_slice */ Field3D Laplacian::solve(const Field3D& b, const Field3D& x0) { - TRACE("Laplacian::solve(Field3D, Field3D)"); ASSERT1(b.getLocation() == location); ASSERT1(x0.getLocation() == location); @@ -261,7 +253,7 @@ Field2D Laplacian::solve(const Field2D& b, const Field2D& x0) { **********************************************************************************/ void Laplacian::tridagCoefs(int jx, int jy, int jz, dcomplex& a, dcomplex& b, dcomplex& c, - const Field2D* ccoef, const Field2D* d, CELL_LOC loc) { + const Field2D* ccoef, const Field2D* d, CELL_LOC loc) const { if (loc == CELL_DEFAULT) { loc = location; @@ -278,13 +270,13 @@ void Laplacian::tridagCoefs(int jx, int jy, int jz, dcomplex& a, dcomplex& b, dc void Laplacian::tridagCoefs(int /* jx */, int /* jy */, BoutReal /* kwave */, dcomplex& /* a */, dcomplex& /* b */, dcomplex& /* c */, const Field2D* /* c1coef */, const Field2D* /* c2coef */, - const Field2D* /* d */, CELL_LOC /* loc */) { + const Field2D* /* d */, CELL_LOC /* loc */) const { throw BoutException("Laplacian::tridagCoefs() does not support 3d metrics."); } #else void Laplacian::tridagCoefs(int jx, int jy, BoutReal kwave, dcomplex& a, dcomplex& b, dcomplex& c, const Field2D* c1coef, const Field2D* c2coef, - const Field2D* d, CELL_LOC loc) { + const Field2D* d, CELL_LOC loc) const { /* Function: Laplacian::tridagCoef * Purpose: - Set the matrix components of A in Ax=b, solving * @@ -426,14 +418,14 @@ void Laplacian::tridagMatrix(dcomplex* /*avec*/, dcomplex* /*bvec*/, dcomplex* / dcomplex* /*bk*/, int /*jy*/, int /*kz*/, BoutReal /*kwave*/, const Field2D* /*a*/, const Field2D* /*c1coef*/, const Field2D* /*c2coef*/, const Field2D* /*d*/, - bool /*includeguards*/, bool /*zperiodic*/) { + bool /*includeguards*/, bool /*zperiodic*/) const { throw BoutException("Error: tridagMatrix does not yet work with 3D metric."); } #else void Laplacian::tridagMatrix(dcomplex* avec, dcomplex* bvec, dcomplex* cvec, dcomplex* bk, int jy, int kz, BoutReal kwave, const Field2D* a, const Field2D* c1coef, const Field2D* c2coef, - const Field2D* d, bool includeguards, bool zperiodic) { + const Field2D* d, bool includeguards, bool zperiodic) const { ASSERT1(a->getLocation() == location); ASSERT1(c1coef->getLocation() == location); ASSERT1(c2coef->getLocation() == location); @@ -462,7 +454,8 @@ void Laplacian::tridagMatrix(dcomplex* avec, dcomplex* bvec, dcomplex* cvec, dco // Setting the width of the boundary. // NOTE: The default is a width of (localmesh->xstart) guard cells - int inbndry = localmesh->xstart, outbndry = localmesh->xstart; + int inbndry = localmesh->xstart; + int outbndry = localmesh->xstart; // If the flags to assign that only one guard cell should be used is set if (isGlobalFlagSet(INVERT_BOTH_BNDRY_ONE) || (localmesh->xstart < 2)) { diff --git a/src/invert/laplacexy/impls/hypre/laplacexy-hypre.cxx b/src/invert/laplacexy/impls/hypre/laplacexy-hypre.cxx index 9323513f21..61632a332d 100644 --- a/src/invert/laplacexy/impls/hypre/laplacexy-hypre.cxx +++ b/src/invert/laplacexy/impls/hypre/laplacexy-hypre.cxx @@ -1,6 +1,6 @@ #include "bout/build_defines.hxx" -#if BOUT_HAS_HYPRE +#if BOUT_HAS_HYPRE and not BOUT_USE_METRIC_3D #include "laplacexy-hypre.hxx" @@ -23,8 +23,10 @@ #include #if BOUT_HAS_CUDA && defined(__CUDACC__) -#define gpuErrchk(ans) \ - { gpuAssert((ans), __FILE__, __LINE__); } +#define gpuErrchk(ans) \ + { \ + gpuAssert((ans), __FILE__, __LINE__); \ + } inline void gpuAssert(cudaError_t code, const char* file, int line, bool abort = true) { if (code != cudaSuccess) { fprintf(stderr, "GPUassert: %s %s %d\n", cudaGetErrorString(code), file, line); diff --git a/src/invert/laplacexy/impls/hypre/laplacexy-hypre.hxx b/src/invert/laplacexy/impls/hypre/laplacexy-hypre.hxx index f415f9ebf4..f9b1d90fb3 100644 --- a/src/invert/laplacexy/impls/hypre/laplacexy-hypre.hxx +++ b/src/invert/laplacexy/impls/hypre/laplacexy-hypre.hxx @@ -43,6 +43,12 @@ RegisterUnavailableLaplaceXY registerlaplacexyhypre("hypre", "BOUT++ was not configured with HYPRE"); } +#elif BOUT_USE_METRIC_3D +namespace { +RegisterUnavailableLaplaceXY + registerlaplacexyhypre("hypre", "BOUT++ was configured with 3D metrics"); +} + #else // BOUT_HAS_HYPRE class Mesh; diff --git a/src/invert/laplacexz/impls/cyclic/laplacexz-cyclic.cxx b/src/invert/laplacexz/impls/cyclic/laplacexz-cyclic.cxx index a242bf3432..b3e619df0c 100644 --- a/src/invert/laplacexz/impls/cyclic/laplacexz-cyclic.cxx +++ b/src/invert/laplacexz/impls/cyclic/laplacexz-cyclic.cxx @@ -5,7 +5,6 @@ #include #include -#include #include #include @@ -14,6 +13,7 @@ LaplaceXZcyclic::LaplaceXZcyclic(Mesh* m, Options* options, const CELL_LOC loc) : LaplaceXZ(m, options, loc) { // Note: `m` may be nullptr, but localmesh is set in LaplaceXZ base constructor + bout::fft::assertZSerial(*localmesh, "`cyclic` X-Z inversion"); // Number of Z Fourier modes, including DC nmode = (localmesh->LocalNz) / 2 + 1; @@ -61,7 +61,7 @@ LaplaceXZcyclic::LaplaceXZcyclic(Mesh* m, Options* options, const CELL_LOC loc) } void LaplaceXZcyclic::setCoefs(const Field2D& A2D, const Field2D& B2D) { - TRACE("LaplaceXZcyclic::setCoefs"); + Timer timer("invert"); ASSERT1(A2D.getMesh() == localmesh); diff --git a/src/invert/laplacexz/impls/petsc/laplacexz-petsc.cxx b/src/invert/laplacexz/impls/petsc/laplacexz-petsc.cxx index 769e797352..3e55262b6d 100644 --- a/src/invert/laplacexz/impls/petsc/laplacexz-petsc.cxx +++ b/src/invert/laplacexz/impls/petsc/laplacexz-petsc.cxx @@ -15,10 +15,8 @@ #if BOUT_HAS_PETSC // Requires PETSc #include -#include - -#include #include +#include LaplaceXZpetsc::LaplaceXZpetsc(Mesh* m, Options* opt, const CELL_LOC loc) : LaplaceXZ(m, opt, loc), lib(opt == nullptr ? &(Options::root()["laplacexz"]) : opt), @@ -97,8 +95,6 @@ LaplaceXZpetsc::LaplaceXZpetsc(Mesh* m, Options* opt, const CELL_LOC loc) * given by b */ - TRACE("LaplaceXZpetsc::LaplaceXZpetsc"); - if (opt == nullptr) { // If no options supplied, use default opt = &(Options::root())["laplacexz"]; @@ -159,7 +155,7 @@ LaplaceXZpetsc::LaplaceXZpetsc(Mesh* m, Options* opt, const CELL_LOC loc) .withDefault("petsc"); // Get MPI communicator - MPI_Comm comm = localmesh->getXcomm(); + MPI_Comm comm = localmesh->getXZcomm(); // Local size int localN = (localmesh->xend - localmesh->xstart + 1) * (localmesh->LocalNz); @@ -202,25 +198,25 @@ LaplaceXZpetsc::LaplaceXZpetsc(Mesh* m, Options* opt, const CELL_LOC loc) // X boundaries if (localmesh->firstX()) { - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { d_nnz[z] = 2; } } else { // One point on another processor - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { d_nnz[z] -= 1; o_nnz[z] += 1; } } if (localmesh->lastX()) { - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { int ind = localN - (localmesh->LocalNz) + z; d_nnz[ind] = 2; } } else { // One point on another processor - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { int ind = localN - (localmesh->LocalNz) + z; d_nnz[ind] -= 1; o_nnz[ind] += 1; @@ -260,8 +256,6 @@ LaplaceXZpetsc::LaplaceXZpetsc(Mesh* m, Options* opt, const CELL_LOC loc) LaplaceXZpetsc::~LaplaceXZpetsc() { - TRACE("LaplaceXZpetsc::~LaplaceXZpetsc"); - PetscBool petsc_is_finalised; PetscFinalized(&petsc_is_finalised); @@ -293,8 +287,6 @@ void LaplaceXZpetsc::setCoefs(const Field3D& Ain, const Field3D& Bin) { * Bin - The B coefficient in div(A grad_perp(B)) + Bf = b */ - TRACE("LaplaceXZpetsc::setCoefs"); - ASSERT1(Ain.getMesh() == localmesh); ASSERT1(Bin.getMesh() == localmesh); ASSERT1(Ain.getLocation() == location); @@ -335,7 +327,7 @@ void LaplaceXZpetsc::setCoefs(const Field3D& Ain, const Field3D& Bin) { /* NOTE: Sign of the elements are opposite of what one might expect, * see note about BC in LaplaceXZ constructor for more details */ - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { PetscScalar val = 1.0; MatSetValues(it.MatA, 1, &row, 1, &row, &val, INSERT_VALUES); @@ -347,7 +339,7 @@ void LaplaceXZpetsc::setCoefs(const Field3D& Ain, const Field3D& Bin) { } } else if (inner_boundary_flags & INVERT_SET) { // Setting BC from x0 - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { PetscScalar val = 1.0; MatSetValues(it.MatA, 1, &row, 1, &row, &val, INSERT_VALUES); @@ -359,7 +351,7 @@ void LaplaceXZpetsc::setCoefs(const Field3D& Ain, const Field3D& Bin) { } } else if (inner_boundary_flags & INVERT_RHS) { // Setting BC from b - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { PetscScalar val = 1.0; MatSetValues(it.MatA, 1, &row, 1, &row, &val, INSERT_VALUES); @@ -374,7 +366,7 @@ void LaplaceXZpetsc::setCoefs(const Field3D& Ain, const Field3D& Bin) { /* NOTE: Sign of the elements are opposite of what one might expect, * see note about BC in LaplaceXZ constructor for more details */ - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { PetscScalar val = 0.5; MatSetValues(it.MatA, 1, &row, 1, &row, &val, INSERT_VALUES); @@ -393,7 +385,7 @@ void LaplaceXZpetsc::setCoefs(const Field3D& Ain, const Field3D& Bin) { Coordinates* coords = localmesh->getCoordinates(location); for (int x = localmesh->xstart; x <= localmesh->xend; x++) { - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { // stencil entries PetscScalar c, xm, xp, zm, zp; // Diagonal entries @@ -655,7 +647,7 @@ void LaplaceXZpetsc::setCoefs(const Field3D& Ain, const Field3D& Bin) { if (localmesh->lastX()) { if (outer_boundary_flags & INVERT_AC_GRAD) { // Neumann 0 - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { PetscScalar val = 1.0; MatSetValues(it.MatA, 1, &row, 1, &row, &val, INSERT_VALUES); @@ -667,7 +659,7 @@ void LaplaceXZpetsc::setCoefs(const Field3D& Ain, const Field3D& Bin) { } } else if (outer_boundary_flags & INVERT_SET) { // Setting BC from x0 - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { PetscScalar val = 1.0; MatSetValues(it.MatA, 1, &row, 1, &row, &val, INSERT_VALUES); @@ -679,7 +671,7 @@ void LaplaceXZpetsc::setCoefs(const Field3D& Ain, const Field3D& Bin) { } } else if (outer_boundary_flags & INVERT_RHS) { // Setting BC from b - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { PetscScalar val = 1.0; MatSetValues(it.MatA, 1, &row, 1, &row, &val, INSERT_VALUES); @@ -693,7 +685,7 @@ void LaplaceXZpetsc::setCoefs(const Field3D& Ain, const Field3D& Bin) { //Default: Dirichlet on outer X boundary PetscScalar val = 0.5; - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { MatSetValues(it.MatA, 1, &row, 1, &row, &val, INSERT_VALUES); int col = row - (localmesh->LocalNz); // -1 in X @@ -767,8 +759,6 @@ Field3D LaplaceXZpetsc::solve(const Field3D& bin, const Field3D& x0in) { * result - The solved x (returned as a Field3D) in the matrix problem Ax=b */ - TRACE("LaplaceXZpetsc::solve"); - ASSERT1(bin.getMesh() == localmesh); ASSERT1(x0in.getMesh() == localmesh); ASSERT1(bin.getLocation() == location); @@ -804,7 +794,7 @@ Field3D LaplaceXZpetsc::solve(const Field3D& bin, const Field3D& x0in) { if (localmesh->firstX()) { if (inner_boundary_flags & INVERT_AC_GRAD) { // Neumann 0 - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { // Setting the initial guess x0 PetscScalar val = x0(localmesh->xstart - 1, y, z); VecSetValues(xs, 1, &ind, &val, INSERT_VALUES); @@ -816,7 +806,7 @@ Field3D LaplaceXZpetsc::solve(const Field3D& bin, const Field3D& x0in) { } } else if (inner_boundary_flags & INVERT_SET) { // Setting BC from x0 - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { // Setting the initial guess x0 PetscScalar val = x0(localmesh->xstart - 1, y, z); VecSetValues(xs, 1, &ind, &val, INSERT_VALUES); @@ -828,7 +818,7 @@ Field3D LaplaceXZpetsc::solve(const Field3D& bin, const Field3D& x0in) { } } else if (inner_boundary_flags & INVERT_RHS) { // Setting BC from b - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { // Setting the initial guess x0 PetscScalar val = x0(localmesh->xstart - 1, y, z); VecSetValues(xs, 1, &ind, &val, INSERT_VALUES); @@ -840,7 +830,7 @@ Field3D LaplaceXZpetsc::solve(const Field3D& bin, const Field3D& x0in) { } } else { // Default: Neumann on inner x boundary - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { // Setting the initial guess x0 PetscScalar val = x0(localmesh->xstart - 1, y, z); VecSetValues(xs, 1, &ind, &val, INSERT_VALUES); @@ -855,7 +845,7 @@ Field3D LaplaceXZpetsc::solve(const Field3D& bin, const Field3D& x0in) { // Set the inner points for (int x = localmesh->xstart; x <= localmesh->xend; x++) { - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { PetscScalar val = x0(x, y, z); VecSetValues(xs, 1, &ind, &val, INSERT_VALUES); @@ -869,7 +859,7 @@ Field3D LaplaceXZpetsc::solve(const Field3D& bin, const Field3D& x0in) { if (localmesh->lastX()) { if (outer_boundary_flags & INVERT_AC_GRAD) { // Neumann 0 - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { // Setting the initial guess x0 PetscScalar val = x0(localmesh->xend + 1, y, z); VecSetValues(xs, 1, &ind, &val, INSERT_VALUES); @@ -882,7 +872,7 @@ Field3D LaplaceXZpetsc::solve(const Field3D& bin, const Field3D& x0in) { } } else if (outer_boundary_flags & INVERT_SET) { // Setting BC from x0 - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { // Setting the initial guess x0 PetscScalar val = x0(localmesh->xend + 1, y, z); VecSetValues(xs, 1, &ind, &val, INSERT_VALUES); @@ -895,7 +885,7 @@ Field3D LaplaceXZpetsc::solve(const Field3D& bin, const Field3D& x0in) { } } else if (outer_boundary_flags & INVERT_RHS) { // Setting BC from b - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { // Setting the initial guess x0 PetscScalar val = x0(localmesh->xend + 1, y, z); VecSetValues(xs, 1, &ind, &val, INSERT_VALUES); @@ -908,7 +898,7 @@ Field3D LaplaceXZpetsc::solve(const Field3D& bin, const Field3D& x0in) { } } else { //Default: Dirichlet on outer X boundary - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { // Setting the initial guess x0 PetscScalar val = x0(localmesh->xend + 1, y, z); VecSetValues(xs, 1, &ind, &val, INSERT_VALUES); @@ -952,7 +942,7 @@ Field3D LaplaceXZpetsc::solve(const Field3D& bin, const Field3D& x0in) { ind = Istart; // Inner X boundary if (localmesh->firstX()) { - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { PetscScalar val; VecGetValues(xs, 1, &ind, &val); for (int x = localmesh->xstart - 1; x >= 0; --x) { @@ -963,7 +953,7 @@ Field3D LaplaceXZpetsc::solve(const Field3D& bin, const Field3D& x0in) { } for (int x = localmesh->xstart; x <= localmesh->xend; x++) { - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { PetscScalar val; VecGetValues(xs, 1, &ind, &val); result(x, y, z) = val; @@ -973,7 +963,7 @@ Field3D LaplaceXZpetsc::solve(const Field3D& bin, const Field3D& x0in) { // Outer X boundary if (localmesh->lastX()) { - for (int z = 0; z < localmesh->LocalNz; z++) { + for (int z = localmesh->zstart; z <= localmesh->zend; z++) { PetscScalar val; VecGetValues(xs, 1, &ind, &val); for (int x = localmesh->xend + 1; x < localmesh->LocalNx; ++x) { diff --git a/src/invert/parderiv/impls/cyclic/cyclic.cxx b/src/invert/parderiv/impls/cyclic/cyclic.cxx index 5a798ca1d5..c32c3d4b2d 100644 --- a/src/invert/parderiv/impls/cyclic/cyclic.cxx +++ b/src/invert/parderiv/impls/cyclic/cyclic.cxx @@ -46,15 +46,15 @@ #include #include #include -#include -#include - #include +#include #include InvertParCR::InvertParCR(Options* opt, CELL_LOC location, Mesh* mesh_in) : InvertPar(opt, location, mesh_in), A(1.0), B(0.0), C(0.0), D(0.0), E(0.0) { + + bout::fft::assertZSerial(*localmesh, "InvertParCR"); // Number of k equations to solve for each x location nsys = 1 + (localmesh->LocalNz) / 2; @@ -63,7 +63,7 @@ InvertParCR::InvertParCR(Options* opt, CELL_LOC location, Mesh* mesh_in) } const Field3D InvertParCR::solve(const Field3D& f) { - TRACE("InvertParCR::solve(Field3D)"); + ASSERT1(localmesh == f.getMesh()); ASSERT1(location == f.getLocation()); diff --git a/src/invert/pardiv/impls/cyclic/pardiv_cyclic.cxx b/src/invert/pardiv/impls/cyclic/pardiv_cyclic.cxx index be65d5ab0b..aad01c5f2f 100644 --- a/src/invert/pardiv/impls/cyclic/pardiv_cyclic.cxx +++ b/src/invert/pardiv/impls/cyclic/pardiv_cyclic.cxx @@ -46,7 +46,6 @@ #include #include #include -#include #include #include @@ -54,12 +53,13 @@ InvertParDivCR::InvertParDivCR(Options* opt, CELL_LOC location, Mesh* mesh_in) : InvertParDiv(opt, location, mesh_in) { + bout::fft::assertZSerial(*localmesh, "InvertParDivCR"); // Number of k equations to solve for each x location nsys = 1 + (localmesh->LocalNz) / 2; } Field3D InvertParDivCR::solve(const Field3D& f) { - TRACE("InvertParDivCR::solve(Field3D)"); + ASSERT1(localmesh == f.getMesh()); ASSERT1(location == f.getLocation()); diff --git a/src/mesh/boundary_factory.cxx b/src/mesh/boundary_factory.cxx index d112a216ad..fd5bbdc70d 100644 --- a/src/mesh/boundary_factory.cxx +++ b/src/mesh/boundary_factory.cxx @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -228,46 +229,82 @@ BoundaryOpBase* BoundaryFactory::createFromOptions(const string& varname, string prefix("bndry_"); - string side; + std::array sides; + sides[0] = region->label; switch (region->location) { case BNDRY_XIN: { - side = "xin"; + sides[1] = "xin"; break; } case BNDRY_XOUT: { - side = "xout"; + sides[1] = "xout"; break; } case BNDRY_YDOWN: { - side = "ydown"; + sides[1] = "ydown"; break; } case BNDRY_YUP: { - side = "yup"; + sides[1] = "yup"; break; } case BNDRY_PAR_FWD_XIN: { - side = "par_yup_xin"; + sides[1] = "par_yup_xin"; break; } case BNDRY_PAR_FWD_XOUT: { - side = "par_yup_xout"; + sides[1] = "par_yup_xout"; break; } case BNDRY_PAR_BKWD_XIN: { - side = "par_ydown_xin"; + sides[1] = "par_ydown_xin"; break; } case BNDRY_PAR_BKWD_XOUT: { - side = "par_ydown_xout"; + sides[1] = "par_ydown_xout"; break; } default: { - side = "all"; + sides[1] = "all"; break; } } + switch (region->location) { + case BNDRY_PAR_FWD_XIN: + case BNDRY_PAR_BKWD_XIN: { + sides[2] = "par_xin"; + break; + } + case BNDRY_PAR_BKWD_XOUT: + case BNDRY_PAR_FWD_XOUT: { + sides[2] = "par_xout"; + break; + } + default: { + sides[2] = "all"; + break; + } + } + switch (region->location) { + case BNDRY_PAR_FWD_XIN: + case BNDRY_PAR_FWD_XOUT: { + sides[3] = "par_yup"; + break; + } + case BNDRY_PAR_BKWD_XIN: + case BNDRY_PAR_BKWD_XOUT: { + sides[3] = "par_ydown"; + break; + } + default: { + sides[3] = "all"; + break; + } + } + + sides[4] = region->isParallel ? "par_all" : "all"; + // Get options Options* options = Options::getRoot(); @@ -275,27 +312,10 @@ BoundaryOpBase* BoundaryFactory::createFromOptions(const string& varname, Options* varOpts = options->getSection(varname); string set; - /// First try looking for (var, region) - if (varOpts->isSet(prefix + region->label)) { - varOpts->get(prefix + region->label, set, ""); - return create(set, region); - } - - /// Then (var, side) - if (varOpts->isSet(prefix + side)) { - varOpts->get(prefix + side, set, ""); - return create(set, region); - } - - /// Then (var, all) - if (region->isParallel) { - if (varOpts->isSet(prefix + "par_all")) { - varOpts->get(prefix + "par_all", set, ""); - return create(set, region); - } - } else { - if (varOpts->isSet(prefix + "all")) { - varOpts->get(prefix + "all", set, ""); + /// First try looking for (var, ...) + for (const auto& side : sides) { + if (varOpts->isSet(prefix + side)) { + varOpts->get(prefix + side, set, ""); return create(set, region); } } @@ -303,16 +323,13 @@ BoundaryOpBase* BoundaryFactory::createFromOptions(const string& varname, // Get the "all" options varOpts = options->getSection("all"); - /// Then (all, region) - if (varOpts->isSet(prefix + region->label)) { - varOpts->get(prefix + region->label, set, ""); - return create(set, region); - } - - /// Then (all, side) - if (varOpts->isSet(prefix + side)) { - varOpts->get(prefix + side, set, ""); - return create(set, region); + /// First try looking for (all, ...) + for (const auto& side : sides) { + if (varOpts->isSet(prefix + side)) { + varOpts->get(prefix + side, set, + region->isParallel ? "parallel_dirichlet_o2" : "dirichlet"); + return create(set, region); + } } /// Then (all, all) diff --git a/src/mesh/boundary_standard.cxx b/src/mesh/boundary_standard.cxx index 8f2c7df5ec..2e079f59b3 100644 --- a/src/mesh/boundary_standard.cxx +++ b/src/mesh/boundary_standard.cxx @@ -1,12 +1,12 @@ #include #include +#include #include #include #include #include #include #include -#include #include #include #include @@ -29,8 +29,6 @@ using bout::generator::Context; */ #if CHECK > 0 void verifyNumPoints(BoundaryRegion* region, int ptsRequired) { - TRACE("Verifying number of points available for BC"); - int ptsAvailGlobal, ptsAvailLocal, ptsAvail; std::string side, gridType; Mesh* mesh = region->localmesh; @@ -325,7 +323,7 @@ void BoundaryDirichlet::apply(Field3D& f, BoutReal t) { // Outer x boundary for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -346,7 +344,7 @@ void BoundaryDirichlet::apply(Field3D& f, BoutReal t) { if (bndry->bx < 0) { // Inner x boundary. Set one point inwards for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -368,7 +366,7 @@ void BoundaryDirichlet::apply(Field3D& f, BoutReal t) { if (bndry->by != 0) { // y boundaries for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -394,7 +392,7 @@ void BoundaryDirichlet::apply(Field3D& f, BoutReal t) { // Upper y boundary boundary for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -415,7 +413,7 @@ void BoundaryDirichlet::apply(Field3D& f, BoutReal t) { if (bndry->by < 0) { // Lower y boundary. Set one point inwards for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -436,7 +434,7 @@ void BoundaryDirichlet::apply(Field3D& f, BoutReal t) { if (bndry->bx != 0) { // x boundaries for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -469,7 +467,7 @@ void BoundaryDirichlet::apply(Field3D& f, BoutReal t) { * (mesh->GlobalY(bndry->y) // In the guard cell + mesh->GlobalY(bndry->y - bndry->by)); // the grid cell - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate( Context(bndry, zk, loc, t, mesh).set("x", xnorm, "y", ynorm)); @@ -513,7 +511,7 @@ void BoundaryDirichlet::apply(Field3D& f, BoutReal t) { const int yi = bndry->y + (i * bndry->by); const auto xnorm = mesh->GlobalX(xi); const auto ynorm = mesh->GlobalY(yi) * TWOPI; - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate( Context(bndry, zk, loc, t, mesh).set("x", xnorm, "y", ynorm)); @@ -528,7 +526,7 @@ void BoundaryDirichlet::apply(Field3D& f, BoutReal t) { } else { // Standard (non-staggered) case for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -572,7 +570,7 @@ void BoundaryDirichlet::apply(Field3D& f, BoutReal t) { // Set any other guard cells using the values on the cells int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -596,7 +594,7 @@ void BoundaryDirichlet::apply_ddt(Field3D& f) { Field3D* dt = f.timeDeriv(); for (bndry->first(); !bndry->isDone(); bndry->next()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { (*dt)(bndry->x, bndry->y, z) = 0.; // Set time derivative to zero } } @@ -827,7 +825,7 @@ void BoundaryDirichlet_O3::apply(Field3D& f, BoutReal t) { // Outer x boundary for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -848,7 +846,7 @@ void BoundaryDirichlet_O3::apply(Field3D& f, BoutReal t) { if (bndry->bx < 0) { // Inner x boundary. Set one point inwards for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -870,7 +868,7 @@ void BoundaryDirichlet_O3::apply(Field3D& f, BoutReal t) { // y boundaries for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -898,7 +896,7 @@ void BoundaryDirichlet_O3::apply(Field3D& f, BoutReal t) { // Upper y boundary for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -919,7 +917,7 @@ void BoundaryDirichlet_O3::apply(Field3D& f, BoutReal t) { if (bndry->by < 0) { // Lower y boundary. Set one point inwards for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -940,7 +938,7 @@ void BoundaryDirichlet_O3::apply(Field3D& f, BoutReal t) { if (bndry->bx != 0) { // x boundaries for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -974,7 +972,7 @@ void BoundaryDirichlet_O3::apply(Field3D& f, BoutReal t) { * (mesh->GlobalY(bndry->y) // In the guard cell + mesh->GlobalY(bndry->y - bndry->by)); // the grid cell - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate( Context(bndry, zk, loc, t, mesh).set("x", xnorm, "y", ynorm)); @@ -1000,7 +998,7 @@ void BoundaryDirichlet_O3::apply(Field3D& f, BoutReal t) { } else { // Standard (non-staggered) case for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -1038,7 +1036,7 @@ void BoundaryDirichlet_O3::apply_ddt(Field3D& f) { bndry->first(); for (bndry->first(); !bndry->isDone(); bndry->next()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { (*dt)(bndry->x, bndry->y, z) = 0.; // Set time derivative to zero } } @@ -1284,7 +1282,7 @@ void BoundaryDirichlet_O4::apply(Field3D& f, BoutReal t) { // Outer x boundary for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -1306,7 +1304,7 @@ void BoundaryDirichlet_O4::apply(Field3D& f, BoutReal t) { if (bndry->bx < 0) { // Inner x boundary. Set one point inwards for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -1329,7 +1327,7 @@ void BoundaryDirichlet_O4::apply(Field3D& f, BoutReal t) { // y boundaries for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -1358,7 +1356,7 @@ void BoundaryDirichlet_O4::apply(Field3D& f, BoutReal t) { // Outer y boundary for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -1381,7 +1379,7 @@ void BoundaryDirichlet_O4::apply(Field3D& f, BoutReal t) { if (bndry->by < 0) { // Inner y boundary. Set one point inwards for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -1405,7 +1403,7 @@ void BoundaryDirichlet_O4::apply(Field3D& f, BoutReal t) { // x boundaries for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -1440,7 +1438,7 @@ void BoundaryDirichlet_O4::apply(Field3D& f, BoutReal t) { * (mesh->GlobalY(bndry->y) // In the guard cell + mesh->GlobalY(bndry->y - bndry->by)); // the grid cell - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate( Context(bndry, zk, loc, t, mesh).set("x", xnorm, "y", ynorm)); @@ -1468,7 +1466,7 @@ void BoundaryDirichlet_O4::apply(Field3D& f, BoutReal t) { } else { // Standard (non-staggered) case for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); } @@ -1505,7 +1503,7 @@ void BoundaryDirichlet_O4::apply_ddt(Field3D& f) { ASSERT1(mesh == f.getMesh()); Field3D* dt = f.timeDeriv(); for (bndry->first(); !bndry->isDone(); bndry->next()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { (*dt)(bndry->x, bndry->y, z) = 0.; // Set time derivative to zero } } @@ -1547,7 +1545,7 @@ void BoundaryDirichlet_4thOrder::apply(Field3D& f) { // Set (at 4th order) the value at the mid-point between the guard cell and the grid // cell to be val for (bndry->first(); !bndry->isDone(); bndry->next1d()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { f(bndry->x, bndry->y, z) = 128. / 35. * val - 4. * f(bndry->x - bndry->bx, bndry->y - bndry->by, z) + 2. * f(bndry->x - 2 * bndry->bx, bndry->y - 2 * bndry->by, z) @@ -1574,7 +1572,7 @@ void BoundaryDirichlet_4thOrder::apply_ddt(Field3D& f) { ASSERT1(mesh == f.getMesh()); Field3D* dt = f.timeDeriv(); for (bndry->first(); !bndry->isDone(); bndry->next()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { (*dt)(bndry->x, bndry->y, z) = 0.; // Set time derivative to zero } } @@ -1661,7 +1659,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { // Loop over all elements and set equal to the next point in for (bndry->first(); !bndry->isDone(); bndry->next1d()) { #if BOUT_USE_METRIC_3D - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { #else int z = 0; #endif @@ -1680,7 +1678,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { // because derivative values don't exist in boundary region // NOTE: should be fixed to interpolate to boundary line #if not(BOUT_USE_METRIC_3D) - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { #endif BoutReal xshift = g12shift * dfdy(bndry->x - bndry->bx, bndry->y, z) + g13shift * dfdz(bndry->x - bndry->bx, bndry->y, z); @@ -1968,7 +1966,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { if (bndry->bx > 0) { // Outer x boundary for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)) * metric->dx(bndry->x, bndry->y, zk); @@ -1997,7 +1995,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { if (bndry->bx < 0) { // Inner x boundary for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)) * metric->dx(bndry->x - bndry->bx, bndry->y, zk); @@ -2029,7 +2027,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { BoutReal delta = bndry->bx * metric->dx(bndry->x, bndry->y) + bndry->by * metric->dy(bndry->x, bndry->y); #endif - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { #if BOUT_USE_METRIC_3D BoutReal delta = bndry->bx * metric->dx(bndry->x, bndry->y, zk) + bndry->by * metric->dy(bndry->x, bndry->y, zk); @@ -2053,7 +2051,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { if (bndry->by > 0) { // Outer y boundary for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)) * metric->dy(bndry->x, bndry->y, zk); @@ -2081,7 +2079,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { if (bndry->by < 0) { // Inner y boundary. Set one point inwards for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)) * metric->dy(bndry->x, bndry->y - bndry->by, zk); @@ -2115,7 +2113,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { BoutReal delta = bndry->bx * metric->dx(bndry->x, bndry->y, zk) + bndry->by * metric->dy(bndry->x, bndry->y, zk); #endif - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { #if BOUT_USE_METRIC_3D BoutReal delta = bndry->bx * metric->dx(bndry->x, bndry->y, zk) + bndry->by * metric->dy(bndry->x, bndry->y, zk); @@ -2148,7 +2146,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { * (mesh->GlobalY(bndry->y) // In the guard cell + mesh->GlobalY(bndry->y - bndry->by)); // the grid cell - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { BoutReal delta = bndry->bx * metric->dx(bndry->x, bndry->y, zk) + bndry->by * metric->dy(bndry->x, bndry->y, zk); if (fg) { @@ -2167,16 +2165,16 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { } else { throw BoutException("Unrecognized location"); } - } else { + } else { // loc == CELL_CENTRE for (; !bndry->isDone(); bndry->next1d()) { #if BOUT_USE_METRIC_3D - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { BoutReal delta = bndry->bx * metric->dx(bndry->x, bndry->y, zk) + bndry->by * metric->dy(bndry->x, bndry->y, zk); #else BoutReal delta = bndry->bx * metric->dx(bndry->x, bndry->y) + bndry->by * metric->dy(bndry->x, bndry->y); - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { #endif if (fg) { val = fg->generate(Context(bndry, zk, loc, t, mesh)); @@ -2205,7 +2203,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { ASSERT1(mesh == f.getMesh()); Field3D* dt = f.timeDeriv(); for (bndry->first(); !bndry->isDone(); bndry->next()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { (*dt)(bndry->x, bndry->y, z) = 0.; // Set time derivative to zero } } @@ -2303,7 +2301,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { } else { Coordinates* coords = f.getCoordinates(); for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { BoutReal delta = bndry->bx * coords->dx(bndry->x, bndry->y, zk) + bndry->by * coords->dy(bndry->x, bndry->y, zk); if (fg) { @@ -2339,7 +2337,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { ASSERT1(mesh == f.getMesh()); Field3D* dt = f.timeDeriv(); for (bndry->first(); !bndry->isDone(); bndry->next()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { (*dt)(bndry->x, bndry->y, z) = 0.; // Set time derivative to zero } } @@ -2398,7 +2396,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { // grid cell to be val This sets the value of the co-ordinate derivative, i.e. DDX/DDY // not Grad_par/Grad_perp.x for (bndry->first(); !bndry->isDone(); bndry->next1d()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { BoutReal delta = -(bndry->bx * metric->dx(bndry->x, bndry->y, z) + bndry->by * metric->dy(bndry->x, bndry->y, z)); f(bndry->x, bndry->y, z) = @@ -2431,7 +2429,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { ASSERT1(mesh == f.getMesh()); Field3D* dt = f.timeDeriv(); for (bndry->first(); !bndry->isDone(); bndry->next()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { (*dt)(bndry->x, bndry->y, z) = 0.; // Set time derivative to zero } } @@ -2469,7 +2467,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { ASSERT1(mesh == f.getMesh()); Coordinates* metric = f.getCoordinates(); for (bndry->first(); !bndry->isDone(); bndry->next()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { f(bndry->x, bndry->y, z) = f(bndry->x - bndry->bx, bndry->y - bndry->by, z) * sqrt(metric->g_22(bndry->x, bndry->y, z) @@ -2536,7 +2534,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { ASSERT1(mesh == f.getMesh()); if (fabs(bval) < 1.e-12) { for (bndry->first(); !bndry->isDone(); bndry->next()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { f(bndry->x, bndry->y, z) = gval / aval; } } @@ -2546,7 +2544,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { sign = -1.; } for (bndry->first(); !bndry->isDone(); bndry->next()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { f(bndry->x, bndry->y, z) = f(bndry->x - bndry->bx, bndry->y - bndry->by, z) + sign * (gval - aval * f(bndry->x - bndry->bx, bndry->y - bndry->by, z)) @@ -2570,7 +2568,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { Mesh* mesh = bndry->localmesh; ASSERT1(mesh == f.getMesh()); for (bndry->first(); !bndry->isDone(); bndry->next()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { f(bndry->x, bndry->y, z) = 2. * f(bndry->x - bndry->bx, bndry->y - bndry->by, z) - f(bndry->x - 2 * bndry->bx, bndry->y - 2 * bndry->by, z); @@ -2633,6 +2631,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { #if not(BOUT_USE_METRIC_3D) Mesh* mesh = bndry->localmesh; ASSERT1(mesh == f.getMesh()); + bout::fft::assertZSerial(*mesh, "Zero Laplace on Field3D"); int ncz = mesh->LocalNz; Coordinates* metric = f.getCoordinates(); @@ -2736,6 +2735,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { #if not(BOUT_USE_METRIC_3D) Mesh* mesh = bndry->localmesh; ASSERT1(mesh == f.getMesh()); + bout::fft::assertZSerial(*mesh, "Zero Laplace on Field3D"); const int ncz = mesh->LocalNz; ASSERT0(ncz % 2 == 0); // Allocation assumes even number @@ -2845,6 +2845,8 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { Mesh* mesh = bndry->localmesh; ASSERT1(mesh == f.getMesh()); + bout::fft::assertZSerial(*mesh, "Zero Laplace on Field3D"); + Coordinates* metric = f.getCoordinates(); int ncz = mesh->LocalNz; @@ -3191,7 +3193,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { for (int i = 0; i < bndry->width; i++) { int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; @@ -3205,7 +3207,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { // Inner x boundary. Set one point inwards for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { for (int i = -1; i < bndry->width; i++) { int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; @@ -3220,7 +3222,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { for (int i = 0; i < bndry->width; i++) { int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; @@ -3236,7 +3238,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { if (bndry->by > 0) { // Upper y boundary for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { for (int i = 0; i < bndry->width; i++) { int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; @@ -3250,7 +3252,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { // Lower y boundary. Set one point inwards for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { for (int i = -1; i < bndry->width; i++) { int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; @@ -3264,7 +3266,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { // x boundaries for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { for (int i = 0; i < bndry->width; i++) { int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; @@ -3279,7 +3281,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { // Standard (non-staggered) case for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { for (int i = 0; i < bndry->width; i++) { int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; @@ -3303,7 +3305,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { ASSERT1(mesh == f.getMesh()); Field3D* dt = f.timeDeriv(); for (bndry->first(); !bndry->isDone(); bndry->next()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { (*dt)(bndry->x, bndry->y, z) = 0.; // Set time derivative to zero } } @@ -3453,7 +3455,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { for (int i = 0; i < bndry->width; i++) { int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; @@ -3468,7 +3470,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { // Inner x boundary. Set one point inwards for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { for (int i = -1; i < bndry->width; i++) { int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; @@ -3484,7 +3486,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { for (int i = 0; i < bndry->width; i++) { int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; @@ -3501,7 +3503,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { if (bndry->by > 0) { // Upper y boundary for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { for (int i = 0; i < bndry->width; i++) { int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; @@ -3516,7 +3518,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { // Lower y boundary. Set one point inwards for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { for (int i = -1; i < bndry->width; i++) { int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; @@ -3531,7 +3533,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { // x boundaries for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { for (int i = 0; i < bndry->width; i++) { int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; @@ -3547,7 +3549,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { // Standard (non-staggered) case for (; !bndry->isDone(); bndry->next1d()) { - for (int zk = 0; zk < mesh->LocalNz; zk++) { + for (int zk = mesh->zstart; zk <= mesh->zend; zk++) { for (int i = 0; i < bndry->width; i++) { int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; @@ -3572,7 +3574,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { ASSERT1(mesh == f.getMesh()); Field3D* dt = f.timeDeriv(); for (bndry->first(); !bndry->isDone(); bndry->next()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { (*dt)(bndry->x, bndry->y, z) = 0.; // Set time derivative to zero } } @@ -3605,7 +3607,6 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { } void BoundaryRelax::apply_ddt(Field2D & f) { - TRACE("BoundaryRelax::apply_ddt(Field2D)"); // Make a copy of f Field2D g = f; @@ -3621,7 +3622,6 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { } void BoundaryRelax::apply_ddt(Field3D & f) { - TRACE("BoundaryRelax::apply_ddt(Field3D)"); Mesh* mesh = bndry->localmesh; ASSERT1(mesh == f.getMesh()); @@ -3632,7 +3632,7 @@ void BoundaryNeumann_NonOrthogonal::apply(Field3D& f) { op->apply(g); // Set time-derivatives for (bndry->first(); !bndry->isDone(); bndry->next()) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { ddt(f)(bndry->x, bndry->y, z) = r * (g(bndry->x, bndry->y, z) - f(bndry->x, bndry->y, z)); } diff --git a/src/mesh/coordinates.cxx b/src/mesh/coordinates.cxx index 3dfee6a553..d2634d3f88 100644 --- a/src/mesh/coordinates.cxx +++ b/src/mesh/coordinates.cxx @@ -5,9 +5,9 @@ **************************************************************************/ #include +#include #include #include -#include #include #include #include @@ -225,7 +225,7 @@ Field3D interpolateAndExtrapolate(const Field3D& f_, CELL_LOC location, ASSERT1(bndry->bx == 0 or localmesh->xstart > 1); ASSERT1(bndry->by == 0 or localmesh->ystart > 1); // note that either bx or by is >0 here - for (int zi = 0; zi < localmesh->LocalNz; ++zi) { + for (int zi = localmesh->zstart; zi <= localmesh->zend; ++zi) { result(bndry->x, bndry->y, zi) = (9. * (f(bndry->x - bndry->bx, bndry->y - bndry->by, zi) @@ -256,7 +256,7 @@ Field3D interpolateAndExtrapolate(const Field3D& f_, CELL_LOC location, for (int i = extrap_start; i < bndry->width; i++) { int xi = bndry->x + i * bndry->bx; int yi = bndry->y + i * bndry->by; - for (int zi = 0; zi < localmesh->LocalNz; ++zi) { + for (int zi = localmesh->zstart; zi <= localmesh->zend; ++zi) { result(xi, yi, zi) = 3.0 * result(xi - bndry->bx, yi - bndry->by, zi) - 3.0 * result(xi - 2 * bndry->bx, yi - 2 * bndry->by, zi) @@ -266,7 +266,7 @@ Field3D interpolateAndExtrapolate(const Field3D& f_, CELL_LOC location, } else { // not enough grid points to extrapolate, set equal to last grid point for (int i = extrap_start; i < bndry->width; i++) { - for (int zi = 0; zi < localmesh->LocalNz; ++zi) { + for (int zi = localmesh->zstart; zi <= localmesh->zend; ++zi) { result(bndry->x + i * bndry->bx, bndry->y + i * bndry->by, zi) = result(bndry->x - bndry->bx, bndry->y - bndry->by, zi); } @@ -931,7 +931,7 @@ const Field2D& Coordinates::zlength() const { int Coordinates::geometry(bool recalculate_staggered, bool force_interpolate_from_centre) { - TRACE("Coordinates::geometry"); + localmesh->communicate_no_slices(dx, dy, dz, g11, g22, g33, g12, g13, g23, g_11, g_22, g_33, g_12, g_13, g_23, J, Bxy); @@ -1201,7 +1201,6 @@ int Coordinates::geometry(bool recalculate_staggered, } int Coordinates::calcCovariant(const std::string& region) { - TRACE("Coordinates::calcCovariant"); // Make sure metric elements are allocated g_11.allocate(); @@ -1233,7 +1232,7 @@ int Coordinates::calcCovariant(const std::string& region) { a(0, 2) = a(2, 0) = g13[i]; if (const auto det = bout::invert3x3(a); det.has_value()) { - output_error.write("\tERROR: metric tensor is singular at {}, determinant: {:d}\n", + output_error.write("\tERROR: metric tensor is singular at {}, determinant: {:e}\n", i, det.value()); return 1; } @@ -1264,7 +1263,6 @@ int Coordinates::calcCovariant(const std::string& region) { } int Coordinates::calcContravariant(const std::string& region) { - TRACE("Coordinates::calcContravariant"); // Make sure metric elements are allocated g11.allocate(); @@ -1289,7 +1287,7 @@ int Coordinates::calcContravariant(const std::string& region) { a(0, 2) = a(2, 0) = g_13[i]; if (const auto det = bout::invert3x3(a); det.has_value()) { - output_error.write("\tERROR: metric tensor is singular at {}, determinant: {:d}\n", + output_error.write("\tERROR: metric tensor is singular at {}, determinant: {:e}\n", i, det.value()); return 1; } @@ -1319,7 +1317,7 @@ int Coordinates::calcContravariant(const std::string& region) { } int Coordinates::jacobian() { - TRACE("Coordinates::jacobian"); + // calculate Jacobian using g^-1 = det[g^ij], J = sqrt(g) const bool extrapolate_x = not localmesh->sourceHasXBoundaryGuards(); @@ -1521,7 +1519,7 @@ Field3D Coordinates::DDZ(const Field3D& f, CELL_LOC outloc, const std::string& m Coordinates::FieldMetric Coordinates::Grad_par(const Field2D& var, [[maybe_unused]] CELL_LOC outloc, const std::string& UNUSED(method)) { - TRACE("Coordinates::Grad_par( Field2D )"); + ASSERT1(location == outloc || (outloc == CELL_DEFAULT && location == var.getLocation())); @@ -1530,7 +1528,7 @@ Coordinates::FieldMetric Coordinates::Grad_par(const Field2D& var, Field3D Coordinates::Grad_par(const Field3D& var, CELL_LOC outloc, const std::string& method) { - TRACE("Coordinates::Grad_par( Field3D )"); + ASSERT1(location == outloc || outloc == CELL_DEFAULT); return ::DDY(var, outloc, method) * invSg(); @@ -1560,7 +1558,7 @@ Field3D Coordinates::Vpar_Grad_par(const Field3D& v, const Field3D& f, CELL_LOC Coordinates::FieldMetric Coordinates::Div_par(const Field2D& f, CELL_LOC outloc, const std::string& method) { - TRACE("Coordinates::Div_par( Field2D )"); + ASSERT1(location == outloc || outloc == CELL_DEFAULT); // Need Bxy at location of f, which might be different from location of this @@ -1572,12 +1570,12 @@ Coordinates::FieldMetric Coordinates::Div_par(const Field2D& f, CELL_LOC outloc, Field3D Coordinates::Div_par(const Field3D& f, CELL_LOC outloc, const std::string& method) { - TRACE("Coordinates::Div_par( Field3D )"); + ASSERT1(location == outloc || outloc == CELL_DEFAULT); // Need Bxy at location of f, which might be different from location of this // Coordinates object - auto Bxy_floc = f.getCoordinates()->Bxy; + const auto& Bxy_floc = f.getCoordinates()->Bxy; if (!f.hasParallelSlices()) { // No yup/ydown fields. The Grad_par operator will @@ -1601,7 +1599,7 @@ Field3D Coordinates::Div_par(const Field3D& f, CELL_LOC outloc, Coordinates::FieldMetric Coordinates::Grad2_par2(const Field2D& f, CELL_LOC outloc, const std::string& method) { - TRACE("Coordinates::Grad2_par2( Field2D )"); + ASSERT1(location == outloc || (outloc == CELL_DEFAULT && location == f.getLocation())); auto result = Grad2_par2_DDY_invSg(outloc, method) * DDY(f, outloc, method) @@ -1612,7 +1610,7 @@ Coordinates::FieldMetric Coordinates::Grad2_par2(const Field2D& f, CELL_LOC outl Field3D Coordinates::Grad2_par2(const Field3D& f, CELL_LOC outloc, const std::string& method) { - TRACE("Coordinates::Grad2_par2( Field3D )"); + if (outloc == CELL_DEFAULT) { outloc = f.getLocation(); } @@ -1636,7 +1634,7 @@ Field3D Coordinates::Grad2_par2(const Field3D& f, CELL_LOC outloc, Coordinates::FieldMetric Coordinates::Delp2(const Field2D& f, CELL_LOC outloc, bool UNUSED(useFFT)) { - TRACE("Coordinates::Delp2( Field2D )"); + ASSERT1(location == outloc || outloc == CELL_DEFAULT); auto result = G1 * DDX(f, outloc) + g11 * D2DX2(f, outloc); @@ -1645,7 +1643,6 @@ Coordinates::FieldMetric Coordinates::Delp2(const Field2D& f, CELL_LOC outloc, } Field3D Coordinates::Delp2(const Field3D& f, CELL_LOC outloc, bool useFFT) { - TRACE("Coordinates::Delp2( Field3D )"); if (outloc == CELL_DEFAULT) { outloc = f.getLocation(); @@ -1662,7 +1659,7 @@ Field3D Coordinates::Delp2(const Field3D& f, CELL_LOC outloc, bool useFFT) { Field3D result{emptyFrom(f).setLocation(outloc)}; - if (useFFT and not bout::build::use_metric_3d) { + if (useFFT and not bout::build::use_metric_3d and localmesh->getNZPE() == 1) { int ncz = localmesh->LocalNz; // Allocate memory @@ -1711,7 +1708,6 @@ Field3D Coordinates::Delp2(const Field3D& f, CELL_LOC outloc, bool useFFT) { } FieldPerp Coordinates::Delp2(const FieldPerp& f, CELL_LOC outloc, bool useFFT) { - TRACE("Coordinates::Delp2( FieldPerp )"); if (outloc == CELL_DEFAULT) { outloc = f.getLocation(); @@ -1731,7 +1727,7 @@ FieldPerp Coordinates::Delp2(const FieldPerp& f, CELL_LOC outloc, bool useFFT) { int jy = f.getIndex(); result.setIndex(jy); - if (useFFT) { + if (useFFT and localmesh->getNZPE() == 1) { int ncz = localmesh->LocalNz; // Allocate memory @@ -1788,7 +1784,7 @@ Field3D Coordinates::Laplace_par(const Field3D& f, CELL_LOC outloc) { Coordinates::FieldMetric Coordinates::Laplace(const Field2D& f, CELL_LOC outloc, const std::string& dfdy_boundary_conditions, const std::string& dfdy_dy_region) { - TRACE("Coordinates::Laplace( Field2D )"); + ASSERT1(location == outloc || outloc == CELL_DEFAULT); auto result = G1 * DDX(f, outloc) + G2 * DDY(f, outloc) + g11 * D2DX2(f, outloc) @@ -1803,7 +1799,7 @@ Coordinates::FieldMetric Coordinates::Laplace(const Field2D& f, CELL_LOC outloc, Field3D Coordinates::Laplace(const Field3D& f, CELL_LOC outloc, const std::string& dfdy_boundary_conditions, const std::string& dfdy_dy_region) { - TRACE("Coordinates::Laplace( Field3D )"); + ASSERT1(location == outloc || outloc == CELL_DEFAULT); Field3D result = G1 * ::DDX(f, outloc) + G2 * ::DDY(f, outloc) + G3 * ::DDZ(f, outloc) @@ -1822,7 +1818,7 @@ Field3D Coordinates::Laplace(const Field3D& f, CELL_LOC outloc, // solver Field2D Coordinates::Laplace_perpXY([[maybe_unused]] const Field2D& A, [[maybe_unused]] const Field2D& f) { - TRACE("Coordinates::Laplace_perpXY( Field2D )"); + #if not(BOUT_USE_METRIC_3D) Field2D result; result.allocate(); diff --git a/src/mesh/coordinates_accessor.cxx b/src/mesh/coordinates_accessor.cxx index aff546c2b0..c874ea3ac2 100644 --- a/src/mesh/coordinates_accessor.cxx +++ b/src/mesh/coordinates_accessor.cxx @@ -1,4 +1,5 @@ #include "bout/coordinates_accessor.hxx" +#include #include "bout/mesh.hxx" @@ -44,8 +45,10 @@ CoordinatesAccessor::CoordinatesAccessor(const Coordinates* coords) { data[stripe_size * ind.ind + static_cast(Offset::symbol)] = coords->symbol[ind]; // Implement copy for each argument -#define COPY_STRIPE(...) \ - { MACRO_FOR_EACH(COPY_STRIPE1, __VA_ARGS__) } +#define COPY_STRIPE(...) \ + { \ + MACRO_FOR_EACH(COPY_STRIPE1, __VA_ARGS__) \ + } // Iterate over all points in the field // Note this could be 2D or 3D, depending on FieldMetric type diff --git a/src/mesh/data/gridfromfile.cxx b/src/mesh/data/gridfromfile.cxx index b9cc1c4e89..ee09d0055f 100644 --- a/src/mesh/data/gridfromfile.cxx +++ b/src/mesh/data/gridfromfile.cxx @@ -3,10 +3,10 @@ #include #include +#include #include #include #include -#include #include #include #include @@ -20,7 +20,6 @@ GridFile::GridFile(std::string gridfilename) : GridDataSource(true), data(bout::OptionsIO::create(gridfilename)->read()), filename(std::move(gridfilename)) { - TRACE("GridFile constructor"); // Get number of y-boundary guard cells saved in the grid file grid_yguards = data["y_boundary_guards"].withDefault(0); @@ -60,7 +59,7 @@ bool GridFile::hasVar(const std::string& name) { return data.isSet(name); } bool GridFile::get(Mesh* UNUSED(m), std::string& sval, const std::string& name, const std::string& def) { Timer timer("io"); - TRACE("GridFile::get(std::string)"); + const bool success = data.isSet(name); if (not success) { // Override any previously set defaults @@ -93,7 +92,7 @@ bool GridFile::get(Mesh* UNUSED(m), std::string& sval, const std::string& name, */ bool GridFile::get(Mesh* UNUSED(m), int& ival, const std::string& name, int def) { Timer timer("io"); - TRACE("GridFile::get(int)"); + const bool success = data.isSet(name); if (not success) { // Override any previously set defaults @@ -110,7 +109,7 @@ bool GridFile::get(Mesh* UNUSED(m), int& ival, const std::string& name, int def) bool GridFile::get(Mesh* UNUSED(m), BoutReal& rval, const std::string& name, BoutReal def) { Timer timer("io"); - TRACE("GridFile::get(BoutReal)"); + const bool success = data.isSet(name); if (not success) { // Override any previously set defaults @@ -122,7 +121,7 @@ bool GridFile::get(Mesh* UNUSED(m), BoutReal& rval, const std::string& name, /*! * Reads a 2D, 3D or FieldPerp field variable from a file - * + * * Successfully reads Field2D or FieldPerp if the variable in the file is 0-D or 2-D. * Successfully reads Field3D if the variable in the file is 0-D, 2-D or 3-D. */ @@ -143,7 +142,6 @@ bool GridFile::getField(Mesh* m, T& var, const std::string& name, BoutReal def, "templated GridFile::getField only works for Field2D, Field3D or FieldPerp"); Timer timer("io"); - AUTO_TRACE(); if (not data.isSet(name)) { // Variable not found @@ -384,7 +382,7 @@ void GridFile::readField(Mesh* m, const std::string& name, int ys, int yd, int n for (int x = xs; x < xs + nx_to_read; ++x) { for (int y = ys; y < ys + ny_to_read; ++y) { - BoutReal const value = full_var(x, y); + const BoutReal value = full_var(x, y); for (int z = 0; z < var.getNz(); z++) { var(x - xs + xd, y - ys + yd, z) = value; } @@ -476,7 +474,6 @@ bool GridFile::get([[maybe_unused]] Mesh* m, [[maybe_unused]] std::vector& [[maybe_unused]] const std::string& name, [[maybe_unused]] int len, [[maybe_unused]] int offset, [[maybe_unused]] GridDataSource::Direction dir) { - TRACE("GridFile::get(vector)"); if (not data.isSet(name)) { return false; @@ -502,7 +499,6 @@ bool GridFile::get([[maybe_unused]] Mesh* m, [[maybe_unused]] std::vector& bool GridFile::get(Mesh* UNUSED(m), std::vector& var, const std::string& name, int len, int offset, GridDataSource::Direction UNUSED(dir)) { - TRACE("GridFile::get(vector)"); if (not data.isSet(name)) { return false; diff --git a/src/mesh/data/gridfromoptions.cxx b/src/mesh/data/gridfromoptions.cxx index 379e279de0..662329c526 100644 --- a/src/mesh/data/gridfromoptions.cxx +++ b/src/mesh/data/gridfromoptions.cxx @@ -146,8 +146,7 @@ bool GridFromOptions::get(Mesh* m, std::vector& var, const std::string } case GridDataSource::Z: { for (int z = 0; z < len; z++) { - pos.set("z", - (TWOPI * (z - m->OffsetZ + offset)) / static_cast(m->LocalNz)); + pos.set("z", TWOPI * m->GlobalZ(z - m->OffsetZ + offset)); var[z] = gen->generate(pos); } break; diff --git a/src/mesh/difops.cxx b/src/mesh/difops.cxx index 42fa4d6ca5..09433b0685 100644 --- a/src/mesh/difops.cxx +++ b/src/mesh/difops.cxx @@ -30,7 +30,6 @@ #include #include #include -#include #include #include #include @@ -458,8 +457,6 @@ Field2D Laplace_perpXY(const Field2D& A, const Field2D& f) { Coordinates::FieldMetric b0xGrad_dot_Grad(const Field2D& phi, const Field2D& A, CELL_LOC outloc) { - TRACE("b0xGrad_dot_Grad( Field2D , Field2D )"); - if (outloc == CELL_DEFAULT) { outloc = A.getLocation(); } @@ -489,7 +486,6 @@ Coordinates::FieldMetric b0xGrad_dot_Grad(const Field2D& phi, const Field2D& A, } Field3D b0xGrad_dot_Grad(const Field2D& phi, const Field3D& A, CELL_LOC outloc) { - TRACE("b0xGrad_dot_Grad( Field2D , Field3D )"); if (outloc == CELL_DEFAULT) { outloc = A.getLocation(); @@ -531,7 +527,6 @@ Field3D b0xGrad_dot_Grad(const Field2D& phi, const Field3D& A, CELL_LOC outloc) } Field3D b0xGrad_dot_Grad(const Field3D& p, const Field2D& A, CELL_LOC outloc) { - TRACE("b0xGrad_dot_Grad( Field3D , Field2D )"); if (outloc == CELL_DEFAULT) { outloc = A.getLocation(); @@ -566,7 +561,6 @@ Field3D b0xGrad_dot_Grad(const Field3D& p, const Field2D& A, CELL_LOC outloc) { } Field3D b0xGrad_dot_Grad(const Field3D& phi, const Field3D& A, CELL_LOC outloc) { - TRACE("b0xGrad_dot_Grad( Field3D , Field3D )"); if (outloc == CELL_DEFAULT) { outloc = A.getLocation(); @@ -614,7 +608,6 @@ Field3D b0xGrad_dot_Grad(const Field3D& phi, const Field3D& A, CELL_LOC outloc) Coordinates::FieldMetric bracket(const Field2D& f, const Field2D& g, BRACKET_METHOD method, CELL_LOC outloc, Solver* UNUSED(solver)) { - TRACE("bracket(Field2D, Field2D)"); ASSERT1_FIELDS_COMPATIBLE(f, g); if (outloc == CELL_DEFAULT) { @@ -637,7 +630,6 @@ Coordinates::FieldMetric bracket(const Field2D& f, const Field2D& g, Field3D bracket(const Field3D& f, const Field2D& g, BRACKET_METHOD method, CELL_LOC outloc, Solver* solver) { - TRACE("bracket(Field3D, Field2D)"); ASSERT1_FIELDS_COMPATIBLE(f, g); if (outloc == CELL_DEFAULT) { @@ -771,46 +763,6 @@ Field3D bracket(const Field3D& f, const Field2D& g, BRACKET_METHOD method, break; } - case BRACKET_ARAKAWA_OLD: { -#if not(BOUT_USE_METRIC_3D) - const int ncz = mesh->LocalNz; - BOUT_OMP_PERF(parallel for) - for (int jx = mesh->xstart; jx <= mesh->xend; jx++) { - for (int jy = mesh->ystart; jy <= mesh->yend; jy++) { - const BoutReal partialFactor = 1.0 / (12 * metric->dz(jx, jy)); - const BoutReal spacingFactor = partialFactor / metric->dx(jx, jy); - for (int jz = 0; jz < mesh->LocalNz; jz++) { - const int jzp = jz + 1 < ncz ? jz + 1 : 0; - // Above is alternative to const int jzp = (jz + 1) % ncz; - const int jzm = jz - 1 >= 0 ? jz - 1 : ncz - 1; - // Above is alternative to const int jzmTmp = (jz - 1 + ncz) % ncz; - - // J++ = DDZ(f)*DDX(g) - DDX(f)*DDZ(g) - BoutReal Jpp = - ((f(jx, jy, jzp) - f(jx, jy, jzm)) * (g(jx + 1, jy) - g(jx - 1, jy)) - - (f(jx + 1, jy, jz) - f(jx - 1, jy, jz)) * (g(jx, jy) - g(jx, jy))); - - // J+x - BoutReal Jpx = (g(jx + 1, jy) * (f(jx + 1, jy, jzp) - f(jx + 1, jy, jzm)) - - g(jx - 1, jy) * (f(jx - 1, jy, jzp) - f(jx - 1, jy, jzm)) - - g(jx, jy) * (f(jx + 1, jy, jzp) - f(jx - 1, jy, jzp)) - + g(jx, jy) * (f(jx + 1, jy, jzm) - f(jx - 1, jy, jzm))); - - // Jx+ - BoutReal Jxp = (g(jx + 1, jy) * (f(jx, jy, jzp) - f(jx + 1, jy, jz)) - - g(jx - 1, jy) * (f(jx - 1, jy, jz) - f(jx, jy, jzm)) - - g(jx - 1, jy) * (f(jx, jy, jzp) - f(jx - 1, jy, jz)) - + g(jx + 1, jy) * (f(jx + 1, jy, jz) - f(jx, jy, jzm))); - - result(jx, jy, jz) = (Jpp + Jpx + Jxp) * spacingFactor; - } - } - } -#else - throw BoutException("BRACKET_ARAKAWA_OLD not valid with 3D metrics yet."); -#endif - break; - } case BRACKET_SIMPLE: { // Use a subset of terms for comparison to BOUT-06 result = VDDX(DDZ(f, outloc), g, outloc); @@ -826,7 +778,6 @@ Field3D bracket(const Field3D& f, const Field2D& g, BRACKET_METHOD method, Field3D bracket(const Field2D& f, const Field3D& g, BRACKET_METHOD method, CELL_LOC outloc, Solver* solver) { - TRACE("bracket(Field2D, Field3D)"); ASSERT1_FIELDS_COMPATIBLE(f, g); if (outloc == CELL_DEFAULT) { @@ -863,8 +814,6 @@ Field3D bracket(const Field2D& f, const Field3D& g, BRACKET_METHOD method, Field3D bracket(const Field3D& f, const Field3D& g, BRACKET_METHOD method, CELL_LOC outloc, [[maybe_unused]] Solver* solver) { - TRACE("Field3D, Field3D"); - ASSERT1_FIELDS_COMPATIBLE(f, g); if (outloc == CELL_DEFAULT) { outloc = g.getLocation(); @@ -904,7 +853,7 @@ Field3D bracket(const Field3D& f, const Field3D& g, BRACKET_METHOD method, int ncz = mesh->LocalNz; for (int y = mesh->ystart; y <= mesh->yend; y++) { for (int x = 1; x <= mesh->LocalNx - 2; x++) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { int zm = (z - 1 + ncz) % ncz; int zp = (z + 1) % ncz; @@ -1090,63 +1039,6 @@ Field3D bracket(const Field3D& f, const Field3D& g, BRACKET_METHOD method, } break; } - case BRACKET_ARAKAWA_OLD: { - // Arakawa scheme for perpendicular flow - - const int ncz = mesh->LocalNz; - - // We need to discard const qualifier in order to manipulate - // storage array directly - Field3D f_temp = f; - Field3D g_temp = g; - - BOUT_OMP_PERF(parallel for) - for (int jx = mesh->xstart; jx <= mesh->xend; jx++) { - for (int jy = mesh->ystart; jy <= mesh->yend; jy++) { -#if not(BOUT_USE_METRIC_3D) - const BoutReal spacingFactor = - 1.0 / (12 * metric->dz(jx, jy) * metric->dx(jx, jy)); -#endif - const BoutReal* Fxm = f_temp(jx - 1, jy); - const BoutReal* Fx = f_temp(jx, jy); - const BoutReal* Fxp = f_temp(jx + 1, jy); - const BoutReal* Gxm = g_temp(jx - 1, jy); - const BoutReal* Gx = g_temp(jx, jy); - const BoutReal* Gxp = g_temp(jx + 1, jy); - for (int jz = 0; jz < mesh->LocalNz; jz++) { -#if BOUT_USE_METRIC_3D - const BoutReal spacingFactor = - 1.0 / (12 * metric->dz(jx, jy, jz) * metric->dx(jx, jy, jz)); -#endif - const int jzp = jz + 1 < ncz ? jz + 1 : 0; - // Above is alternative to const int jzp = (jz + 1) % ncz; - const int jzm = jz - 1 >= 0 ? jz - 1 : ncz - 1; - // Above is alternative to const int jzm = (jz - 1 + ncz) % ncz; - - // J++ = DDZ(f)*DDX(g) - DDX(f)*DDZ(g) - // NOLINTNEXTLINE - BoutReal Jpp = ((Fx[jzp] - Fx[jzm]) * (Gxp[jz] - Gxm[jz]) - - (Fxp[jz] - Fxm[jz]) * (Gx[jzp] - Gx[jzm])); - - // J+x - // NOLINTNEXTLINE - BoutReal Jpx = - (Gxp[jz] * (Fxp[jzp] - Fxp[jzm]) - Gxm[jz] * (Fxm[jzp] - Fxm[jzm]) - - Gx[jzp] * (Fxp[jzp] - Fxm[jzp]) + Gx[jzm] * (Fxp[jzm] - Fxm[jzm])); - - // Jx+ - // NOLINTNEXTLINE - BoutReal Jxp = - (Gxp[jzp] * (Fx[jzp] - Fxp[jz]) - Gxm[jzm] * (Fxm[jz] - Fx[jzm]) - - Gxm[jzp] * (Fx[jzp] - Fxm[jz]) + Gxp[jzm] * (Fxp[jz] - Fx[jzm])); - - result(jx, jy, jz) = (Jpp + Jpx + Jxp) * spacingFactor; - } - } - } - - break; - } case BRACKET_SIMPLE: { // Use a subset of terms for comparison to BOUT-06 result = VDDX(DDZ(f, outloc), g, outloc) + VDDZ(-DDX(f, outloc), g, outloc); diff --git a/src/mesh/fv_ops.cxx b/src/mesh/fv_ops.cxx index fe5422b4d1..fab8beb794 100644 --- a/src/mesh/fv_ops.cxx +++ b/src/mesh/fv_ops.cxx @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -48,7 +47,7 @@ Field3D Div_a_Grad_perp(const Field3D& a, const Field3D& f) { for (int i = xs; i <= xe; i++) { for (int j = mesh->ystart; j <= mesh->yend; j++) { - for (int k = 0; k < mesh->LocalNz; k++) { + for (int k = mesh->zstart; k <= mesh->zend; k++) { // Calculate flux from i to i+1 BoutReal fout = 0.5 * (a(i, j, k) + a(i + 1, j, k)) @@ -181,7 +180,6 @@ Field3D Div_a_Grad_perp(const Field3D& a, const Field3D& f) { const Field3D Div_par_K_Grad_par(const Field3D& Kin, const Field3D& fin, bool bndry_flux) { - TRACE("FV::Div_par_K_Grad_par"); ASSERT2(Kin.getLocation() == fin.getLocation()); @@ -282,7 +280,7 @@ const Field3D D4DY4(const Field3D& d_in, const Field3D& f_in) { mesh->yend; for (int j = ystart; j <= yend; j++) { - for (int k = 0; k < mesh->LocalNz; k++) { + for (int k = mesh->zstart; k <= mesh->zend; k++) { BoutReal dy3 = SQ(coord->dy(i, j, k)) * coord->dy(i, j, k); // 3rd derivative at upper boundary @@ -329,7 +327,7 @@ const Field3D D4DY4_Index(const Field3D& f_in, bool bndry_flux) { if (j != mesh->yend || !has_upper_boundary) { - for (int k = 0; k < mesh->LocalNz; k++) { + for (int k = mesh->zstart; k <= mesh->zend; k++) { // Right boundary common factors const BoutReal common_factor = 0.25 * (coord->dy(i, j, k) + coord->dy(i, j + 1, k)) @@ -353,7 +351,7 @@ const Field3D D4DY4_Index(const Field3D& f_in, bool bndry_flux) { // At a domain boundary // Use a one-sided difference formula - for (int k = 0; k < mesh->LocalNz; k++) { + for (int k = mesh->zstart; k <= mesh->zend; k++) { // Right boundary common factors const BoutReal common_factor = 0.25 * (coord->dy(i, j, k) + coord->dy(i, j + 1, k)) @@ -383,7 +381,7 @@ const Field3D D4DY4_Index(const Field3D& f_in, bool bndry_flux) { // Calculate the fluxes if (j != mesh->ystart || !has_lower_boundary) { - for (int k = 0; k < mesh->LocalNz; k++) { + for (int k = mesh->zstart; k <= mesh->zend; k++) { const BoutReal common_factor = 0.25 * (coord->dy(i, j, k) + coord->dy(i, j + 1, k)) * (coord->J(i, j, k) + coord->J(i, j - 1, k)); @@ -402,7 +400,7 @@ const Field3D D4DY4_Index(const Field3D& f_in, bool bndry_flux) { } } else { // On a domain (Y) boundary - for (int k = 0; k < mesh->LocalNz; k++) { + for (int k = mesh->zstart; k <= mesh->zend; k++) { const BoutReal common_factor = 0.25 * (coord->dy(i, j, k) + coord->dy(i, j + 1, k)) * (coord->J(i, j, k) + coord->J(i, j - 1, k)); @@ -463,7 +461,7 @@ void communicateFluxes(Field3D& f) { mesh->wait(xin); // Add to cells for (int y = mesh->ystart; y <= mesh->yend; y++) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { f(2, y, z) += f(0, y, z); } } @@ -472,7 +470,7 @@ void communicateFluxes(Field3D& f) { mesh->wait(xout); // Add to cells for (int y = mesh->ystart; y <= mesh->yend; y++) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { f(mesh->LocalNx - 3, y, z) += f(mesh->LocalNx - 1, y, z); } } diff --git a/src/mesh/impls/bout/boutmesh.cxx b/src/mesh/impls/bout/boutmesh.cxx index 8917fa3753..83cd16d76c 100644 --- a/src/mesh/impls/bout/boutmesh.cxx +++ b/src/mesh/impls/bout/boutmesh.cxx @@ -26,24 +26,45 @@ #include "boutmesh.hxx" +#include #include +#include #include #include #include #include #include #include +#include +#include +#include +#include +#include #include #include #include #include #include +#include +#include +#include #include #include +#include +#include + #include +#include +#include #include +#include +#include +#include #include +#include +#include +#include /// MPI type of BoutReal for communications #define PVEC_REAL_MPI_TYPE MPI_DOUBLE @@ -57,6 +78,8 @@ If you want the old setting, you have to specify mesh:symmetricGlobalY=false in << optionfile << "\n"; } OPTION(options, symmetricGlobalY, true); + OPTION(options, symmetricGlobalZ, false); // The default should be updated to true but + // this breaks backwards compatibility comm_x = MPI_COMM_NULL; comm_inner = MPI_COMM_NULL; @@ -84,6 +107,9 @@ BoutMesh::~BoutMesh() { if (comm_outer != MPI_COMM_NULL) { MPI_Comm_free(&comm_outer); } + if (comm_xz != MPI_COMM_NULL) { + MPI_Comm_free(&comm_xz); + } } BoutMesh::YDecompositionIndices @@ -163,13 +189,13 @@ CheckMeshResult checkBoutMeshYDecomposition(int num_y_processors, int ny, // Check size of Y mesh if we've got multiple processors in Y if (num_local_y_points < num_y_guards and num_y_processors != 1) { return {false, - fmt::format(_("\t -> ny/NYPE ({:d}/{:d} = {:d}) must be >= MYG ({:d})\n"), ny, - num_y_processors, num_local_y_points, num_y_guards)}; + fmt::format(_f("\t -> ny/NYPE ({:d}/{:d} = {:d}) must be >= MYG ({:d})\n"), + ny, num_y_processors, num_local_y_points, num_y_guards)}; } // Check branch cuts if ((jyseps1_1 + 1) % num_local_y_points != 0) { - return {false, fmt::format(_("\t -> Leg region jyseps1_1+1 ({:d}) must be a " - "multiple of MYSUB ({:d})\n"), + return {false, fmt::format(_f("\t -> Leg region jyseps1_1+1 ({:d}) must be a " + "multiple of MYSUB ({:d})\n"), jyseps1_1 + 1, num_local_y_points)}; } @@ -179,50 +205,51 @@ CheckMeshResult checkBoutMeshYDecomposition(int num_y_processors, int ny, if ((jyseps2_1 - jyseps1_1) % num_local_y_points != 0) { return { false, - fmt::format(_("\t -> Core region jyseps2_1-jyseps1_1 ({:d}-{:d} = {:d}) must " - "be a multiple of MYSUB ({:d})\n"), + fmt::format(_f("\t -> Core region jyseps2_1-jyseps1_1 ({:d}-{:d} = {:d}) must " + "be a multiple of MYSUB ({:d})\n"), jyseps2_1, jyseps1_1, jyseps2_1 - jyseps1_1, num_local_y_points)}; } if ((jyseps2_2 - jyseps1_2) % num_local_y_points != 0) { return { false, - fmt::format(_("\t -> Core region jyseps2_2-jyseps1_2 ({:d}-{:d} = {:d}) must " - "be a multiple of MYSUB ({:d})\n"), + fmt::format(_f("\t -> Core region jyseps2_2-jyseps1_2 ({:d}-{:d} = {:d}) must " + "be a multiple of MYSUB ({:d})\n"), jyseps2_2, jyseps1_2, jyseps2_2 - jyseps1_2, num_local_y_points)}; } // Check upper legs if ((ny_inner - jyseps2_1 - 1) % num_local_y_points != 0) { - return { - false, - fmt::format(_("\t -> leg region ny_inner-jyseps2_1-1 ({:d}-{:d}-1 = {:d}) must " - "be a multiple of MYSUB ({:d})\n"), - ny_inner, jyseps2_1, ny_inner - jyseps2_1 - 1, num_local_y_points)}; + return {false, + fmt::format( + _f("\t -> leg region ny_inner-jyseps2_1-1 ({:d}-{:d}-1 = {:d}) must " + "be a multiple of MYSUB ({:d})\n"), + ny_inner, jyseps2_1, ny_inner - jyseps2_1 - 1, num_local_y_points)}; } if ((jyseps1_2 - ny_inner + 1) % num_local_y_points != 0) { - return { - false, - fmt::format(_("\t -> leg region jyseps1_2-ny_inner+1 ({:d}-{:d}+1 = {:d}) must " - "be a multiple of MYSUB ({:d})\n"), - jyseps1_2, ny_inner, jyseps1_2 - ny_inner + 1, num_local_y_points)}; + return {false, + fmt::format( + _f("\t -> leg region jyseps1_2-ny_inner+1 ({:d}-{:d}+1 = {:d}) must " + "be a multiple of MYSUB ({:d})\n"), + jyseps1_2, ny_inner, jyseps1_2 - ny_inner + 1, num_local_y_points)}; } } else { // Single Null if ((jyseps2_2 - jyseps1_1) % num_local_y_points != 0) { return { false, - fmt::format(_("\t -> Core region jyseps2_2-jyseps1_1 ({:d}-{:d} = {:d}) must " - "be a multiple of MYSUB ({:d})\n"), + fmt::format(_f("\t -> Core region jyseps2_2-jyseps1_1 ({:d}-{:d} = {:d}) must " + "be a multiple of MYSUB ({:d})\n"), jyseps2_2, jyseps1_1, jyseps2_2 - jyseps1_1, num_local_y_points)}; } } if ((ny - jyseps2_2 - 1) % num_local_y_points != 0) { - return {false, fmt::format( - _("\t -> leg region ny-jyseps2_2-1 ({:d}-{:d}-1 = {:d}) must be a " - "multiple of MYSUB ({:d})\n"), - ny, jyseps2_2, ny - jyseps2_2 - 1, num_local_y_points)}; + return { + false, + fmt::format(_f("\t -> leg region ny-jyseps2_2-1 ({:d}-{:d}-1 = {:d}) must be a " + "multiple of MYSUB ({:d})\n"), + ny, jyseps2_2, ny - jyseps2_2 - 1, num_local_y_points)}; } return {true, ""}; @@ -242,7 +269,7 @@ void BoutMesh::chooseProcessorSplit(Options& options) { .withDefault(1); if ((NPES % NXPE) != 0) { throw BoutException( - _("Number of processors ({:d}) not divisible by NPs in x direction ({:d})\n"), + _f("Number of processors ({:d}) not divisible by NPs in x direction ({:d})\n"), NPES, NXPE); } @@ -255,7 +282,7 @@ void BoutMesh::chooseProcessorSplit(Options& options) { .withDefault(1); if ((NPES % NYPE) != 0) { throw BoutException( - _("Number of processors ({:d}) not divisible by NPs in y direction ({:d})\n"), + _f("Number of processors ({:d}) not divisible by NPs in y direction ({:d})\n"), NPES, NYPE); } @@ -278,14 +305,14 @@ void BoutMesh::findProcessorSplit() { // Results in square domains const BoutReal ideal = sqrt(MX * NPES / static_cast(ny)); - output_info.write(_("Finding value for NXPE (ideal = {:f})\n"), ideal); + output_info.write(_f("Finding value for NXPE (ideal = {:f})\n"), ideal); for (int i = 1; i <= NPES; i++) { // Loop over all possibilities if ((NPES % i == 0) && // Processors divide equally (MX % i == 0) && // Mesh in X divides equally (ny % (NPES / i) == 0)) { // Mesh in Y divides equally - output_info.write(_("\tCandidate value: {:d}\n"), i); + output_info.write(_f("\tCandidate value: {:d}\n"), i); const int nyp = NPES / i; @@ -312,15 +339,15 @@ void BoutMesh::findProcessorSplit() { NYPE = NPES / NXPE; - output_progress.write(_("\tDomain split (NXPE={:d}, NYPE={:d}) into domains " - "(localNx={:d}, localNy={:d})\n"), + output_progress.write(_f("\tDomain split (NXPE={:d}, NYPE={:d}) into domains " + "(localNx={:d}, localNy={:d})\n"), NXPE, NYPE, MX / NXPE, ny / NYPE); } void BoutMesh::setDerivedGridSizes() { // Check that nx is large enough if (nx <= 2 * MXG) { - throw BoutException(_("Error: nx must be greater than 2 times MXG (2 * {:d})"), MXG); + throw BoutException(_f("Error: nx must be greater than 2 times MXG (2 * {:d})"), MXG); } GlobalNx = nx; @@ -345,8 +372,8 @@ void BoutMesh::setDerivedGridSizes() { MX = nx - 2 * MXG; MXSUB = MX / NXPE; if ((MX % NXPE) != 0) { - throw BoutException(_("Cannot split {:d} X points equally between {:d} processors\n"), - MX, NXPE); + throw BoutException( + _f("Cannot split {:d} X points equally between {:d} processors\n"), MX, NXPE); } // NOTE: No grid data reserved for Y boundary cells - copy from neighbours @@ -354,7 +381,7 @@ void BoutMesh::setDerivedGridSizes() { MYSUB = MY / NYPE; if ((MY % NYPE) != 0) { throw BoutException( - _("\tERROR: Cannot split {:d} Y points equally between {:d} processors\n"), MY, + _f("\tERROR: Cannot split {:d} Y points equally between {:d} processors\n"), MY, NYPE); } @@ -362,7 +389,7 @@ void BoutMesh::setDerivedGridSizes() { MZSUB = MZ / NZPE; if ((MZ % NZPE) != 0) { throw BoutException( - _("\tERROR: Cannot split {:d} Z points equally between {:d} processors\n"), MZ, + _f("\tERROR: Cannot split {:d} Z points equally between {:d} processors\n"), MZ, NZPE); } @@ -431,7 +458,6 @@ void BoutMesh::setDerivedGridSizes() { } int BoutMesh::load() { - TRACE("BoutMesh::load()"); output_progress << _("Loading mesh") << endl; @@ -464,8 +490,8 @@ int BoutMesh::load() { if (!is_pow2(nz)) { // Should be a power of 2 for efficient FFTs output_warn.write( - _("WARNING: Number of toroidal points should be 2^n for efficient " - "FFT performance -- consider changing MZ ({:d}) if using FFTs\n"), + _f("WARNING: Number of toroidal points should be 2^n for efficient " + "FFT performance -- consider changing MZ ({:d}) if using FFTs\n"), nz); } } else { @@ -484,8 +510,19 @@ int BoutMesh::load() { } ASSERT0(MXG >= 0); - if (Mesh::get(MYG, "MYG") != 0) { - MYG = options["MYG"].doc("Number of guard cells on each side in Y").withDefault(2); + const bool meshHasMyg = Mesh::get(MYG, "MYG") == 0; + if (!meshHasMyg) { + MYG = 2; + } + int meshMyg = MYG; + + if (options.isSet("MYG") or (!meshHasMyg)) { + MYG = options["MYG"].doc("Number of guard cells on each side in Y").withDefault(MYG); + } + if (meshHasMyg && MYG != meshMyg) { + output_warn.write(_f("Options changed the number of y-guard cells. Grid has {} but " + "option specified {}! Continuing with {}"), + meshMyg, MYG, MYG); } ASSERT0(MYG >= 0); @@ -517,9 +554,12 @@ int BoutMesh::load() { findProcessorSplit(); } - // Get X and Y processor indices + // Get X, Y, Z processor indices PE_YIND = MYPE / NXPE; PE_XIND = MYPE % NXPE; + PE_ZIND = 0; + + ASSERT2(MYPE == getProcIndex(PE_XIND, PE_YIND, PE_ZIND)); // Set the other grid sizes from nx, ny, nz setDerivedGridSizes(); @@ -629,10 +669,43 @@ int BoutMesh::load() { return 0; } +namespace { +auto make_XZ_communicator(const BoutMesh& mesh, MPI_Group group_world) -> MPI_Comm { + std::vector ranks; + + const int yp = mesh.getYProcIndex(); + + // All processors with the same Y index + for (int xp = 0; xp < mesh.getNXPE(); ++xp) { + for (int zp = 0; zp < mesh.getNZPE(); ++zp) { + ranks.push_back(mesh.getProcIndex(xp, yp, zp)); + } + } + MPI_Group group{}; + if (MPI_Group_incl(group_world, static_cast(ranks.size()), ranks.data(), &group) + != MPI_SUCCESS) { + throw BoutException("Could not create X-Z communication group for ranks {}", + fmt::join(ranks, ", ")); + } + + MPI_Comm comm_xz{}; + if (MPI_Comm_create(BoutComm::get(), group, &comm_xz) != MPI_SUCCESS) { + throw BoutException("Could not create X-Z communicator for yp={} (xind={}, yind={}, " + "zind={}) ranks={}", + yp, mesh.getXProcIndex(), mesh.getYProcIndex(), + mesh.getZProcIndex(), fmt::join(ranks, ", ")); + } + + return comm_xz; +} +} // namespace + void BoutMesh::createCommunicators() { MPI_Group group_world{}; MPI_Comm_group(BoutComm::get(), &group_world); // Get the entire group + comm_xz = make_XZ_communicator(*this, group_world); + ////////////////////////////////////////////////////// /// Communicator in X @@ -1002,6 +1075,10 @@ void BoutMesh::createXBoundaries() { } } +int BoutMesh::getProcIndex(int X, int Y, int Z) const { + return (((Z * NYPE) + Y) * NXPE) + X; +} + void BoutMesh::createYBoundaries() { if (MYG <= 0) { return; @@ -1363,7 +1440,6 @@ comm_handle BoutMesh::sendY(FieldGroup& g, comm_handle handle) { } int BoutMesh::wait(comm_handle handle) { - TRACE("BoutMesh::wait(comm_handle)"); if (handle == nullptr) { return 1; @@ -1521,13 +1597,17 @@ int BoutMesh::wait(comm_handle handle) { * Non-Local Communications ***************************************************************/ -int BoutMesh::getNXPE() { return NXPE; } +int BoutMesh::getNXPE() const { return NXPE; } + +int BoutMesh::getNYPE() const { return NYPE; } -int BoutMesh::getNYPE() { return NYPE; } +int BoutMesh::getNZPE() const { return NZPE; } -int BoutMesh::getXProcIndex() { return PE_XIND; } +int BoutMesh::getXProcIndex() const { return PE_XIND; } -int BoutMesh::getYProcIndex() { return PE_YIND; } +int BoutMesh::getYProcIndex() const { return PE_YIND; } + +int BoutMesh::getZProcIndex() const { return PE_ZIND; } /**************************************************************** * X COMMUNICATIONS @@ -1542,7 +1622,7 @@ bool BoutMesh::lastX() const { return PE_XIND == NXPE - 1; } int BoutMesh::sendXOut(BoutReal* buffer, int size, int tag) { Timer timer("comms"); - int proc {-1}; + int proc{-1}; if (PE_XIND == NXPE - 1) { if (periodicX) { // Wrap around to first processor in X @@ -1554,8 +1634,7 @@ int BoutMesh::sendXOut(BoutReal* buffer, int size, int tag) { proc = PROC_NUM(PE_XIND + 1, PE_YIND); } - mpi->MPI_Send(buffer, size, PVEC_REAL_MPI_TYPE, proc, tag, - BoutComm::get()); + mpi->MPI_Send(buffer, size, PVEC_REAL_MPI_TYPE, proc, tag, BoutComm::get()); return 0; } @@ -1563,7 +1642,7 @@ int BoutMesh::sendXOut(BoutReal* buffer, int size, int tag) { int BoutMesh::sendXIn(BoutReal* buffer, int size, int tag) { Timer timer("comms"); - int proc {-1}; + int proc{-1}; if (PE_XIND == 0) { if (periodicX) { // Wrap around to last processor in X @@ -1575,8 +1654,7 @@ int BoutMesh::sendXIn(BoutReal* buffer, int size, int tag) { proc = PROC_NUM(PE_XIND - 1, PE_YIND); } - mpi->MPI_Send(buffer, size, PVEC_REAL_MPI_TYPE, proc, tag, - BoutComm::get()); + mpi->MPI_Send(buffer, size, PVEC_REAL_MPI_TYPE, proc, tag, BoutComm::get()); return 0; } @@ -1584,7 +1662,7 @@ int BoutMesh::sendXIn(BoutReal* buffer, int size, int tag) { comm_handle BoutMesh::irecvXOut(BoutReal* buffer, int size, int tag) { Timer timer("comms"); - int proc {-1}; + int proc{-1}; if (PE_XIND == NXPE - 1) { if (periodicX) { // Wrap around to first processor in X @@ -1599,8 +1677,8 @@ comm_handle BoutMesh::irecvXOut(BoutReal* buffer, int size, int tag) { // Get a communications handle. Not fussy about size of arrays CommHandle* ch = get_handle(0, 0); - mpi->MPI_Irecv(buffer, size, PVEC_REAL_MPI_TYPE, proc, tag, - BoutComm::get(), ch->request); + mpi->MPI_Irecv(buffer, size, PVEC_REAL_MPI_TYPE, proc, tag, BoutComm::get(), + ch->request); ch->in_progress = true; @@ -1610,7 +1688,7 @@ comm_handle BoutMesh::irecvXOut(BoutReal* buffer, int size, int tag) { comm_handle BoutMesh::irecvXIn(BoutReal* buffer, int size, int tag) { Timer timer("comms"); - int proc {-1}; + int proc{-1}; if (PE_XIND == 0) { if (periodicX) { // Wrap around to last processor in X @@ -1625,8 +1703,8 @@ comm_handle BoutMesh::irecvXIn(BoutReal* buffer, int size, int tag) { // Get a communications handle. Not fussy about size of arrays CommHandle* ch = get_handle(0, 0); - mpi->MPI_Irecv(buffer, size, PVEC_REAL_MPI_TYPE, proc, tag, - BoutComm::get(), ch->request); + mpi->MPI_Irecv(buffer, size, PVEC_REAL_MPI_TYPE, proc, tag, BoutComm::get(), + ch->request); ch->in_progress = true; @@ -1691,35 +1769,32 @@ int BoutMesh::PROC_NUM(int xind, int yind) const { return -1; } - return yind * NXPE + xind; + return (yind * NXPE) + xind; } -/// Returns the global X index given a local index -int BoutMesh::XGLOBAL(BoutReal xloc, BoutReal& xglo) const { - xglo = xloc + PE_XIND * MXSUB; - return static_cast(xglo); +BoutReal BoutMesh::getGlobalXIndex(BoutReal xloc) const { + return xloc + (PE_XIND * MXSUB); } -int BoutMesh::getGlobalXIndex(int xlocal) const { return xlocal + PE_XIND * MXSUB; } +int BoutMesh::getGlobalXIndex(int xlocal) const { return xlocal + (PE_XIND * MXSUB); } int BoutMesh::getGlobalXIndexNoBoundaries(int xlocal) const { - return xlocal + PE_XIND * MXSUB - MXG; + return xlocal + (PE_XIND * MXSUB) - MXG; } -int BoutMesh::getLocalXIndex(int xglobal) const { return xglobal - PE_XIND * MXSUB; } +int BoutMesh::getLocalXIndex(int xglobal) const { return xglobal - (PE_XIND * MXSUB); } int BoutMesh::getLocalXIndexNoBoundaries(int xglobal) const { - return xglobal - PE_XIND * MXSUB + MXG; + return xglobal - (PE_XIND * MXSUB) + MXG; } -int BoutMesh::YGLOBAL(BoutReal yloc, BoutReal& yglo) const { - yglo = yloc + PE_YIND * MYSUB - MYG; - return static_cast(yglo); +BoutReal BoutMesh::getGlobalYIndex(BoutReal yloc) const { + return yloc + (PE_YIND * MYSUB) - MYG; } int BoutMesh::getGlobalYIndex(int ylocal) const { - int yglobal = ylocal + PE_YIND * MYSUB; - if (jyseps1_2 > jyseps2_1 and PE_YIND * MYSUB + 2 * MYG + 1 > ny_inner) { + int yglobal = ylocal + (PE_YIND * MYSUB); + if (jyseps1_2 > jyseps2_1 and (PE_YIND * MYSUB) + (2 * MYG) + 1 > ny_inner) { // Double null, and we are past the upper target yglobal += 2 * MYG; } @@ -1727,12 +1802,12 @@ int BoutMesh::getGlobalYIndex(int ylocal) const { } int BoutMesh::getGlobalYIndexNoBoundaries(int ylocal) const { - return ylocal + PE_YIND * MYSUB - MYG; + return ylocal + (PE_YIND * MYSUB) - MYG; } int BoutMesh::getLocalYIndex(int yglobal) const { - int ylocal = yglobal - PE_YIND * MYSUB; - if (jyseps1_2 > jyseps2_1 and PE_YIND * MYSUB + 2 * MYG + 1 > ny_inner) { + int ylocal = yglobal - (PE_YIND * MYSUB); + if (jyseps1_2 > jyseps2_1 and (PE_YIND * MYSUB) + (2 * MYG) + 1 > ny_inner) { // Double null, and we are past the upper target ylocal -= 2 * MYG; } @@ -1740,19 +1815,25 @@ int BoutMesh::getLocalYIndex(int yglobal) const { } int BoutMesh::getLocalYIndexNoBoundaries(int yglobal) const { - return yglobal - PE_YIND * MYSUB + MYG; + return yglobal - (PE_YIND * MYSUB) + MYG; } -int BoutMesh::YGLOBAL(int yloc, int yproc) const { return yloc + yproc * MYSUB - MYG; } +int BoutMesh::YGLOBAL(int yloc, int yproc) const { return yloc + (yproc * MYSUB) - MYG; } -int BoutMesh::YLOCAL(int yglo, int yproc) const { return yglo - yproc * MYSUB + MYG; } +int BoutMesh::YLOCAL(int yglo, int yproc) const { return yglo - (yproc * MYSUB) + MYG; } -int BoutMesh::getGlobalZIndex(int zlocal) const { return zlocal; } +int BoutMesh::getGlobalZIndex(int zlocal) const { return zlocal + (PE_ZIND * MZSUB); } -int BoutMesh::getGlobalZIndexNoBoundaries(int zlocal) const { return zlocal; } +int BoutMesh::getGlobalZIndexNoBoundaries(int zlocal) const { + return zlocal + (PE_ZIND * MZSUB) - MZG; +} int BoutMesh::getLocalZIndex(int zglobal) const { return zglobal; } +BoutReal BoutMesh::getGlobalZIndex(BoutReal zloc) const { + return zloc + (PE_ZIND * MZSUB); +} + int BoutMesh::getLocalZIndexNoBoundaries(int zglobal) const { return zglobal; } int BoutMesh::YPROC(int yind) const { @@ -1820,16 +1901,15 @@ BoutMesh::BoutMesh(int input_nx, int input_ny, int input_nz, int mxg, int myg, i BoutMesh::BoutMesh(int input_nx, int input_ny, int input_nz, int mxg, int myg, int nxpe, int nype, int pe_xind, int pe_yind, bool symmetric_X, bool symmetric_Y, - bool periodicX_, int ixseps1_, int ixseps2_, int jyseps1_1_, + bool periodic_X_, int ixseps1_, int ixseps2_, int jyseps1_1_, int jyseps2_1_, int jyseps1_2_, int jyseps2_2_, int ny_inner_, bool create_regions) : nx(input_nx), ny(input_ny), nz(input_nz), NPES(nxpe * nype), - MYPE(nxpe * pe_yind + pe_xind), PE_YIND(pe_yind), NYPE(nype), NZPE(1), - ixseps1(ixseps1_), ixseps2(ixseps2_), symmetricGlobalX(symmetric_X), + MYPE((nxpe * pe_yind) + pe_xind), PE_XIND(pe_xind), NXPE(nxpe), PE_YIND(pe_yind), + NYPE(nype), ixseps1(ixseps1_), ixseps2(ixseps2_), symmetricGlobalX(symmetric_X), symmetricGlobalY(symmetric_Y), MXG(mxg), MYG(myg), MZG(0) { - NXPE = nxpe; - PE_XIND = pe_xind; - periodicX = periodicX_; + + periodicX = periodic_X_; setYDecompositionIndices(jyseps1_1_, jyseps2_1_, jyseps1_2_, jyseps2_2_, ny_inner_); setDerivedGridSizes(); topology(); @@ -2177,9 +2257,9 @@ void BoutMesh::topology() { } for (int i = 0; i < limiter_count; ++i) { - int const yind = limiter_yinds[i]; - int const xstart = limiter_xstarts[i]; - int const xend = limiter_xends[i]; + const int yind = limiter_yinds[i]; + const int xstart = limiter_xstarts[i]; + const int xend = limiter_xends[i]; output_info.write("Adding a limiter between y={} and {}. X indices {} to {}\n", yind, yind + 1, xstart, xend); add_target(yind, xstart, xend); @@ -2356,72 +2436,112 @@ void BoutMesh::overlapHandleMemory(BoutMesh* yup, BoutMesh* ydown, BoutMesh* xin * Communication utilities ****************************************************************/ -int BoutMesh::pack_data(const std::vector& var_list, int xge, int xlt, - int yge, int ylt, BoutReal* buffer) { +int BoutMesh::pack_data(const std::vector& var_list, int xge, int xlt, int yge, + int ylt, BoutReal* buffer) const { + using enum Field::FieldType; int len = 0; + const int zge = 0; + const int zlt = LocalNz; - /// Loop over variables for (const auto& var : var_list) { - if (var->is3D()) { - // 3D variable - auto* var3d_ref_ptr = dynamic_cast(var); + switch (var->field_type()) { + case field3d: { + const auto* var3d_ref_ptr = dynamic_cast(var); ASSERT0(var3d_ref_ptr != nullptr); - auto& var3d_ref = *var3d_ref_ptr; + const auto& var3d_ref = *var3d_ref_ptr; ASSERT2(var3d_ref.isAllocated()); - for (int jx = xge; jx != xlt; jx++) { + for (int jx = xge; jx < xlt; jx++) { for (int jy = yge; jy < ylt; jy++) { - for (int jz = 0; jz < LocalNz; jz++, len++) { + for (int jz = zge; jz < zlt; jz++, len++) { buffer[len] = var3d_ref(jx, jy, jz); } } } - } else { - // 2D variable - auto* var2d_ref_ptr = dynamic_cast(var); + break; + } + case field2d: { + const auto* var2d_ref_ptr = dynamic_cast(var); ASSERT0(var2d_ref_ptr != nullptr); - auto& var2d_ref = *var2d_ref_ptr; + const auto& var2d_ref = *var2d_ref_ptr; ASSERT2(var2d_ref.isAllocated()); - for (int jx = xge; jx != xlt; jx++) { + for (int jx = xge; jx < xlt; jx++) { for (int jy = yge; jy < ylt; jy++, len++) { buffer[len] = var2d_ref(jx, jy); } } + break; + } + case fieldperp: { + const auto* varperp_ref_ptr = dynamic_cast(var); + ASSERT0(varperp_ref_ptr != nullptr); + const auto& varperp_ref = *varperp_ref_ptr; + ASSERT2(varperp_ref.isAllocated()); + for (int jx = xge; jx < xlt; jx++) { + for (int jz = zge; jz < zlt; jz++, len++) { + buffer[len] = varperp_ref(jx, jz); + } + } + break; + } } } - return (len); + return len; } -int BoutMesh::unpack_data(const std::vector& var_list, int xge, int xlt, - int yge, int ylt, BoutReal* buffer) { +int BoutMesh::unpack_data(const std::vector& var_list, int xge, int xlt, int yge, + int ylt, const BoutReal* buffer) const { + using enum Field::FieldType; int len = 0; + const int zge = 0; + const int zlt = LocalNz; - /// Loop over variables for (const auto& var : var_list) { - if (var->is3D()) { - // 3D variable - auto& var3d_ref = *dynamic_cast(var); - for (int jx = xge; jx != xlt; jx++) { + switch (var->field_type()) { + case field3d: { + auto* var3d_ref_ptr = dynamic_cast(var); + ASSERT0(var3d_ref_ptr != nullptr); + auto& var3d_ref = *var3d_ref_ptr; + ASSERT2(var3d_ref.isAllocated()); + for (int jx = xge; jx < xlt; jx++) { for (int jy = yge; jy < ylt; jy++) { - for (int jz = 0; jz < LocalNz; jz++, len++) { + for (int jz = zge; jz < zlt; jz++, len++) { var3d_ref(jx, jy, jz) = buffer[len]; } } } - } else { - // 2D variable - auto& var2d_ref = *dynamic_cast(var); - for (int jx = xge; jx != xlt; jx++) { + break; + } + case field2d: { + auto* var2d_ref_ptr = dynamic_cast(var); + ASSERT0(var2d_ref_ptr != nullptr); + auto& var2d_ref = *var2d_ref_ptr; + ASSERT2(var2d_ref.isAllocated()); + for (int jx = xge; jx < xlt; jx++) { for (int jy = yge; jy < ylt; jy++, len++) { var2d_ref(jx, jy) = buffer[len]; } } + break; + } + case fieldperp: { + auto* varperp_ref_ptr = dynamic_cast(var); + ASSERT0(varperp_ref_ptr != nullptr); + auto& varperp_ref = *varperp_ref_ptr; + ASSERT2(varperp_ref.isAllocated()); + for (int jx = xge; jx < xlt; jx++) { + for (int jz = zge; jz < zlt; jz++, len++) { + varperp_ref(jx, jz) = buffer[len]; + } + } + break; + } } } - return (len); + return len; } /**************************************************************** @@ -2446,9 +2566,8 @@ bool BoutMesh::periodicY(int jx, BoutReal& ts) const { int BoutMesh::numberOfYBoundaries() const { if (jyseps2_1 != jyseps1_2) { return 2; - } else { - return 1; } + return 1; } std::pair BoutMesh::hasBranchCutLower(int jx) const { @@ -3115,47 +3234,6 @@ void BoutMesh::addBoundaryPar(std::shared_ptr bndry, par_boundary[static_cast(BoundaryParType::all)].push_back(bndry); } -Field3D BoutMesh::smoothSeparatrix(const Field3D& f) { - Field3D result{emptyFrom(f)}; - if ((ixseps_inner > 0) && (ixseps_inner < nx - 1)) { - if (XPROC(ixseps_inner) == PE_XIND) { - int x = getLocalXIndex(ixseps_inner); - for (int y = 0; y < LocalNy; y++) { - for (int z = 0; z < LocalNz; z++) { - result(x, y, z) = 0.5 * (f(x, y, z) + f(x - 1, y, z)); - } - } - } - if (XPROC(ixseps_inner - 1) == PE_XIND) { - int x = getLocalXIndex(ixseps_inner - 1); - for (int y = 0; y < LocalNy; y++) { - for (int z = 0; z < LocalNz; z++) { - result(x, y, z) = 0.5 * (f(x, y, z) + f(x + 1, y, z)); - } - } - } - } - if ((ixseps_outer > 0) && (ixseps_outer < nx - 1) && (ixseps_outer != ixseps_inner)) { - if (XPROC(ixseps_outer) == PE_XIND) { - int x = getLocalXIndex(ixseps_outer); - for (int y = 0; y < LocalNy; y++) { - for (int z = 0; z < LocalNz; z++) { - result(x, y, z) = 0.5 * (f(x, y, z) + f(x - 1, y, z)); - } - } - } - if (XPROC(ixseps_outer - 1) == PE_XIND) { - int x = getLocalXIndex(ixseps_outer - 1); - for (int y = 0; y < LocalNy; y++) { - for (int z = 0; z < LocalNz; z++) { - result(x, y, z) = 0.5 * (f(x, y, z) + f(x + 1, y, z)); - } - } - } - } - return result; -} - BoutReal BoutMesh::GlobalX(int jx) const { if (symmetricGlobalX) { // With this definition the boundary sits dx/2 away form the first/last inner points @@ -3167,8 +3245,7 @@ BoutReal BoutMesh::GlobalX(int jx) const { BoutReal BoutMesh::GlobalX(BoutReal jx) const { // Get global X index as a BoutReal - BoutReal xglo; - XGLOBAL(jx, xglo); + const BoutReal xglo = getGlobalXIndex(jx); if (symmetricGlobalX) { // With this definition the boundary sits dx/2 away form the first/last inner points @@ -3222,8 +3299,7 @@ BoutReal BoutMesh::GlobalY(int jy) const { BoutReal BoutMesh::GlobalY(BoutReal jy) const { // Get global Y index as a BoutReal - BoutReal yglo; - YGLOBAL(jy, yglo); + BoutReal yglo = getGlobalYIndex(jy); if (symmetricGlobalY) { BoutReal yi = yglo; @@ -3265,6 +3341,28 @@ BoutReal BoutMesh::GlobalY(BoutReal jy) const { return yglo / static_cast(nycore); } +BoutReal BoutMesh::GlobalZ(int jz) const { + if (symmetricGlobalZ) { + // With this definition the boundary sits dz/2 away form the first/last inner points + return (0.5 + getGlobalZIndexNoBoundaries(jz) - (nz - MZ) * 0.5) + / static_cast(MZ); + } + return static_cast(getGlobalZIndexNoBoundaries(jz)) + / static_cast(MZ); +} + +BoutReal BoutMesh::GlobalZ(BoutReal jz) const { + + // Get global Z index as a BoutReal + const BoutReal zglo = getGlobalZIndex(jz); + + if (symmetricGlobalZ) { + // With this definition the boundary sits dz/2 away form the first/last inner points + return (0.5 + zglo - (nz - MZ) * 0.5) / static_cast(MZ); + } + return zglo / static_cast(MZ); +} + void BoutMesh::outputVars(Options& output_options) { Timer time("io"); output_options["zperiod"].force(zperiod, "BoutMesh"); diff --git a/src/mesh/impls/bout/boutmesh.hxx b/src/mesh/impls/bout/boutmesh.hxx index a22e0e4473..07453c2676 100644 --- a/src/mesh/impls/bout/boutmesh.hxx +++ b/src/mesh/impls/bout/boutmesh.hxx @@ -4,15 +4,17 @@ #include "mpi.h" +#include "bout/bout_types.hxx" #include "bout/unused.hxx" #include -#include #include #include #include #include +class Field; + /// Implementation of Mesh (mostly) compatible with BOUT /// /// Topology and communications compatible with BOUT @@ -58,10 +60,13 @@ public: ///////////////////////////////////////////// // non-local communications - int getNXPE() override; ///< The number of processors in the X direction - int getNYPE() override; ///< The number of processors in the Y direction - int getXProcIndex() override; ///< This processor's index in X direction - int getYProcIndex() override; ///< This processor's index in Y direction + int getNXPE() const override; ///< The number of processors in the X direction + int getNYPE() const override; ///< The number of processors in the Y direction + int getNZPE() const override; ///< The number of processors in the Z direction + int getXProcIndex() const override; ///< This processor's index in X direction + int getYProcIndex() const override; ///< This processor's index in Y direction + int getZProcIndex() const override; ///< This processor's index in Z direction + int getProcIndex(int X, int Y, int Z) const override; ///////////////////////////////////////////// // X communications @@ -105,6 +110,7 @@ public: MPI_Comm getXcomm(int UNUSED(jy)) const override { return comm_x; } /// Return communicator containing all processors in Y MPI_Comm getYcomm(int xpos) const override; + MPI_Comm getXZcomm() const override { return comm_xz; } /// Is local X index \p jx periodic in Y? /// @@ -167,15 +173,15 @@ public: BoundaryParType type) override; std::set getPossibleBoundaries() const override; - Field3D smoothSeparatrix(const Field3D& f) override; - int getNx() const { return nx; } int getNy() const { return ny; } BoutReal GlobalX(int jx) const override; BoutReal GlobalY(int jy) const override; + BoutReal GlobalZ(int jz) const override; BoutReal GlobalX(BoutReal jx) const override; BoutReal GlobalY(BoutReal jy) const override; + BoutReal GlobalZ(BoutReal jz) const override; BoutReal getIxseps1() const { return ixseps1; } BoutReal getIxseps2() const { return ixseps2; } @@ -208,7 +214,7 @@ protected: /// `getPossibleBoundaries`. \p create_regions controls whether or /// not the various `Region`s are created on the new mesh BoutMesh(int input_nx, int input_ny, int input_nz, int mxg, int myg, int nxpe, int nype, - int pe_xind, int pe_yind, bool symmetric_X, bool symmetric_Y, bool periodic_X, + int pe_xind, int pe_yind, bool symmetric_X, bool symmetric_Y, bool periodic_X_, int ixseps1_, int ixseps2_, int jyseps1_1_, int jyseps2_1_, int jyseps1_2_, int jyseps2_2_, int ny_inner_, bool create_regions = true); @@ -297,16 +303,24 @@ private: int NPES; ///< Number of processors int MYPE; ///< Rank of this processor + int PE_XIND; ///< X index of this processor + int NXPE; ///< Number of processors in the X direction + int PE_YIND; ///< Y index of this processor - int NYPE; // Number of processors in the Y direction + int NYPE; ///< Number of processors in the Y direction - int NZPE; + int PE_ZIND{0}; ///< Z index of this processor + int NZPE{1}; ///< Number of processors in the Z direction /// Is this processor in the core region? bool MYPE_IN_CORE{false}; - int XGLOBAL(BoutReal xloc, BoutReal& xglo) const; - int YGLOBAL(BoutReal yloc, BoutReal& yglo) const; + /// Returns the global X index given a local index + BoutReal getGlobalXIndex(BoutReal xloc) const; + /// Returns the global Y index given a local index + BoutReal getGlobalYIndex(BoutReal yloc) const; + /// Returns the global Z index given a local index + BoutReal getGlobalZIndex(BoutReal zloc) const; // Topology int ixseps1, ixseps2, jyseps1_1, jyseps2_1, jyseps1_2, jyseps2_2; @@ -357,8 +371,9 @@ private: // Settings bool TwistShift; // Use a twist-shift condition in core? - bool symmetricGlobalX; ///< Use a symmetric definition in GlobalX() function - bool symmetricGlobalY; + bool symmetricGlobalX; ///< Use a symmetric definition in `GlobalX()` function + bool symmetricGlobalY; ///< Use a symmetric definition in `GlobalY()` function + bool symmetricGlobalZ{false}; ///< Use a symmetric definition in `GlobalZ()` function int zperiod; BoutReal ZMIN, ZMAX; // Range of the Z domain (in fractions of 2pi) @@ -444,6 +459,8 @@ private: /// Communicator containing all processors in X MPI_Comm comm_x{MPI_COMM_NULL}; + /// Communicator for all processors in an XZ plane + MPI_Comm comm_xz{MPI_COMM_NULL}; ////////////////////////////////////////////////// // Surface communications @@ -465,12 +482,11 @@ private: void post_receiveY(CommHandle& ch); /// Take data from objects and put into a buffer - int pack_data(const std::vector& var_list, int xge, int xlt, int yge, - int ylt, BoutReal* buffer); + int pack_data(const std::vector& var_list, int xge, int xlt, int yge, int ylt, + BoutReal* buffer) const; /// Copy data from a buffer back into the fields - - int unpack_data(const std::vector& var_list, int xge, int xlt, int yge, - int ylt, BoutReal* buffer); + int unpack_data(const std::vector& var_list, int xge, int xlt, int yge, int ylt, + const BoutReal* buffer) const; }; namespace { diff --git a/src/mesh/index_derivs.cxx b/src/mesh/index_derivs.cxx index add67a77c8..70fc47b538 100644 --- a/src/mesh/index_derivs.cxx +++ b/src/mesh/index_derivs.cxx @@ -25,7 +25,6 @@ #include "bout/traits.hxx" #include #include -#include #include /******************************************************************************* @@ -34,7 +33,6 @@ /// Initialise the derivative methods. Must be called before any derivatives are used void Mesh::derivs_init(Options* options) { - TRACE("Initialising derivatives"); // For each direction need to set what the default method is for each type // of derivative. DerivativeStore::getInstance().initialise(options); @@ -45,7 +43,6 @@ void Mesh::derivs_init(Options* options) { STAGGER Mesh::getStagger(const CELL_LOC inloc, const CELL_LOC outloc, const CELL_LOC allowedStaggerLoc) const { - TRACE("Mesh::getStagger -- three arguments"); ASSERT1(outloc == inloc || (outloc == CELL_CENTRE && inloc == allowedStaggerLoc) || (outloc == allowedStaggerLoc && inloc == CELL_CENTRE)); @@ -61,7 +58,6 @@ STAGGER Mesh::getStagger(const CELL_LOC inloc, const CELL_LOC outloc, STAGGER Mesh::getStagger(const CELL_LOC vloc, [[maybe_unused]] const CELL_LOC inloc, const CELL_LOC outloc, const CELL_LOC allowedStaggerLoc) const { - TRACE("Mesh::getStagger -- four arguments"); ASSERT1(inloc == outloc); ASSERT1(vloc == inloc || (vloc == CELL_CENTRE && inloc == allowedStaggerLoc) || (vloc == allowedStaggerLoc && inloc == CELL_CENTRE)); @@ -423,7 +419,7 @@ class FFTDerivativeType { public: template void standard(const T& var, T& result, const std::string& region) const { - AUTO_TRACE(); + ASSERT2(meta.derivType == DERIV::Standard) ASSERT2(var.getMesh()->getNguard(direction) >= nGuards); ASSERT2(direction == DIRECTION::Z); // Only in Z for now @@ -431,6 +427,7 @@ class FFTDerivativeType { ASSERT2(bout::utils::is_Field3D_v); // Should never need to call this with Field2D auto* theMesh = var.getMesh(); + ASSERT2(theMesh->getNZPE() == 1); // Only works if serial in Z for FFTs // Calculate how many Z wavenumbers will be removed const int ncz = theMesh->getNpoints(direction); @@ -479,7 +476,7 @@ class FFTDerivativeType { template void upwindOrFlux(const T& UNUSED(vel), const T& UNUSED(var), T& UNUSED(result), const std::string& UNUSED(region)) const { - AUTO_TRACE(); + throw BoutException("The FFT METHOD isn't available in upwind/Flux"); } static constexpr metaData meta{"FFT", 0, DERIV::Standard}; @@ -489,7 +486,7 @@ class FFT2ndDerivativeType { public: template void standard(const T& var, T& result, const std::string& region) const { - AUTO_TRACE(); + ASSERT2(meta.derivType == DERIV::StandardSecond); ASSERT2(var.getMesh()->getNguard(direction) >= nGuards); ASSERT2(direction == DIRECTION::Z); // Only in Z for now @@ -497,6 +494,7 @@ class FFT2ndDerivativeType { ASSERT2(bout::utils::is_Field3D_v); // Should never need to call this with Field2D auto* theMesh = var.getMesh(); + ASSERT2(theMesh->getNZPE() == 1); // Only works if serial in Z for FFTs // Calculate how many Z wavenumbers will be removed const int ncz = theMesh->getNpoints(direction); @@ -536,7 +534,7 @@ class FFT2ndDerivativeType { template void upwindOrFlux(const T& UNUSED(vel), const T& UNUSED(var), T& UNUSED(result), const std::string& UNUSED(region)) const { - AUTO_TRACE(); + throw BoutException("The FFT METHOD isn't available in upwind/Flux"); } static constexpr metaData meta{"FFT", 0, DERIV::StandardSecond}; @@ -552,14 +550,14 @@ class SplitFluxDerivativeType { public: template void standard(const T&, T&, const std::string) const { - AUTO_TRACE(); + throw BoutException("The SPLIT method isn't available for standard"); } template void upwindOrFlux(const T& vel, const T& var, T& result, const std::string region) const { - AUTO_TRACE(); + // Split into an upwind and a central differencing part // d/dx(v*f) = v*d/dx(f) + f*d/dx(v) result = bout::derivatives::index::flowDerivative( diff --git a/src/mesh/interpolation/hermite_spline_xz.cxx b/src/mesh/interpolation/hermite_spline_xz.cxx index 06d5b393a2..0aa1f2b066 100644 --- a/src/mesh/interpolation/hermite_spline_xz.cxx +++ b/src/mesh/interpolation/hermite_spline_xz.cxx @@ -1,7 +1,7 @@ /************************************************************************** - * Copyright 2015-2018 B.D.Dudson, P. Hill + * Copyright 2015 - 2026 BOUT++ contributors * - * Contact: Ben Dudson, bd512@york.ac.uk + * Contact: Ben Dudson, dudson2@llnl.gov * * This file is part of BOUT++. * @@ -21,10 +21,18 @@ **************************************************************************/ #include "../impls/bout/boutmesh.hxx" +#include "../parallel/fci_comm.hxx" +#include "bout/bout.hxx" +#include "bout/boutexception.hxx" +#include "bout/build_defines.hxx" #include "bout/globals.hxx" #include "bout/index_derivs_interface.hxx" #include "bout/interpolation_xz.hxx" +#include "bout/mask.hxx" +#include +#include +#include #include class IndConverter { @@ -34,9 +42,9 @@ class IndConverter { xstart(mesh->xstart), ystart(mesh->ystart), zstart(0), lnx(mesh->LocalNx - 2 * xstart), lny(mesh->LocalNy - 2 * ystart), lnz(mesh->LocalNz - 2 * zstart) {} - // ix and iy are global indices + // ix and iz are global indices // iy is local - int fromMeshToGlobal(int ix, int iy, int iz) { + int fromMeshToGlobal(int ix, int iy, int iz) const { const int xstart = mesh->xstart; const int lnx = mesh->LocalNx - xstart * 2; // x-proc-id @@ -70,12 +78,12 @@ class IndConverter { return fromLocalToGlobal(ix - pex * lnx, iy - pey_offset * lny, iz - pez * lnz, pex, pey, 0); } - int fromLocalToGlobal(const int ilocalx, const int ilocaly, const int ilocalz) { + int fromLocalToGlobal(const int ilocalx, const int ilocaly, const int ilocalz) const { return fromLocalToGlobal(ilocalx, ilocaly, ilocalz, mesh->getXProcIndex(), mesh->getYProcIndex(), 0); } int fromLocalToGlobal(const int ilocalx, const int ilocaly, const int ilocalz, - const int pex, const int pey, const int pez) { + const int pex, const int pey, const int pez) const { ASSERT3(ilocalx >= 0); ASSERT3(ilocaly >= 0); ASSERT3(ilocalz >= 0); @@ -101,11 +109,21 @@ class IndConverter { } }; -XZHermiteSpline::XZHermiteSpline(int y_offset, Mesh* meshin) +template +XZHermiteSplineBase::XZHermiteSplineBase(int y_offset, Mesh* meshin, + Options* options) : XZInterpolation(y_offset, meshin), h00_x(localmesh), h01_x(localmesh), h10_x(localmesh), h11_x(localmesh), h00_z(localmesh), h01_z(localmesh), h10_z(localmesh), h11_z(localmesh) { + if constexpr (monotonic) { + if (options == nullptr) { + options = &Options::root()["mesh:paralleltransform:xzinterpolation"]; + } + abs_fac_monotonic = (*options)["abs_tol"].withDefault(abs_fac_monotonic); + rel_fac_monotonic = (*options)["rel_tol"].withDefault(rel_fac_monotonic); + } + // Index arrays contain guard cells in order to get subscripts right i_corner.reallocate(localmesh->LocalNx, localmesh->LocalNy, localmesh->LocalNz); k_corner.reallocate(localmesh->LocalNx, localmesh->LocalNy, localmesh->LocalNz); @@ -124,44 +142,48 @@ XZHermiteSpline::XZHermiteSpline(int y_offset, Mesh* meshin) h10_z.allocate(); h11_z.allocate(); -#if USE_NEW_WEIGHTS - newWeights.reserve(16); - for (int w = 0; w < 16; ++w) { - newWeights.emplace_back(localmesh); - newWeights[w].allocate(); + if constexpr (imp_type == bout::details::implementation_type::new_weights + || imp_type == bout::details::implementation_type::petsc) { + newWeights.reserve(16); + for (int w = 0; w < 16; ++w) { + newWeights.emplace_back(localmesh); + newWeights[w].allocate(); + } } -#ifdef HS_USE_PETSC - petsclib = new PetscLib( - &Options::root()["mesh:paralleltransform:xzinterpolation:hermitespline"]); - // MatCreate(MPI_Comm comm,Mat *A) - // MatCreate(MPI_COMM_WORLD, &petscWeights); - // MatSetSizes(petscWeights, m, m, M, M); - // PetscErrorCode MatCreateAIJ(MPI_Comm comm, PetscInt m, PetscInt n, PetscInt M, - // PetscInt N, PetscInt d_nz, const PetscInt d_nnz[], - // PetscInt o_nz, const PetscInt o_nnz[], Mat *A) - // MatSetSizes(Mat A,PetscInt m,PetscInt n,PetscInt M,PetscInt N) - const int m = localmesh->LocalNx * localmesh->LocalNy * localmesh->LocalNz; - const int M = m * localmesh->getNXPE() * localmesh->getNYPE(); - MatCreateAIJ(MPI_COMM_WORLD, m, m, M, M, 16, nullptr, 16, nullptr, &petscWeights); -#endif + if constexpr (imp_type == bout::details::implementation_type::petsc) { +#if BOUT_HAS_PETSC + petsclib = new PetscLib( + &Options::root()["mesh:paralleltransform:xzinterpolation:hermitespline"]); + const int m = localmesh->LocalNx * localmesh->LocalNy * localmesh->LocalNz; + const int M = m * localmesh->getNXPE() * localmesh->getNYPE() * localmesh->getNZPE(); + MatCreateAIJ(BoutComm::get(), m, m, M, M, 16, nullptr, 16, nullptr, &petscWeights); #endif -#ifndef HS_USE_PETSC - if (localmesh->getNXPE() > 1) { - throw BoutException("Require PETSc for MPI splitting in X"); } -#endif + if constexpr (monotonic) { + gf3daccess = std::make_unique(localmesh); + g3dinds.reallocate(localmesh->LocalNx, localmesh->LocalNy, localmesh->LocalNz); + } + if constexpr (imp_type == bout::details::implementation_type::new_weights + || imp_type == bout::details::implementation_type::legacy) { + if (localmesh->getNXPE() > 1) { + throw BoutException("Require PETSc for MPI splitting in X"); + } + } } -void XZHermiteSpline::calcWeights(const Field3D& delta_x, const Field3D& delta_z, - const std::string& region) { +template +void XZHermiteSplineBase::calcWeights( + const Field3D& delta_x, const Field3D& delta_z, + [[maybe_unused]] const std::string& region) { const int ny = localmesh->LocalNy; const int nz = localmesh->LocalNz; const int xend = (localmesh->xend - localmesh->xstart + 1) * localmesh->getNXPE() + localmesh->xstart - 1; -#ifdef HS_USE_PETSC - IndConverter conv{localmesh}; -#endif + [[maybe_unused]] const IndConverter conv{localmesh}; + + [[maybe_unused]] const int y_global_offset = + localmesh->getYProcIndex() * (localmesh->yend - localmesh->ystart + 1); BOUT_FOR(i, getRegion(region)) { const int x = i.x(); const int y = i.y(); @@ -177,7 +199,19 @@ void XZHermiteSpline::calcWeights(const Field3D& delta_x, const Field3D& delta_z BoutReal t_x = delta_x(x, y, z) - static_cast(i_corn); BoutReal t_z = delta_z(x, y, z) - static_cast(k_corner(x, y, z)); - // NOTE: A (small) hack to avoid one-sided differences + // NOTE: A (small) hack to avoid one-sided differences. We need at + // least 2 interior points due to an awkwardness with the + // boundaries. The splines need derivatives in x, but we don't + // know the value in the boundaries, so _any_ interpolation in the + // last interior cell can't be done. Instead, we fudge the + // interpolation in the last cell to be at the extreme right-hand + // edge of the previous cell (that is, exactly on the last + // interior point). However, this doesn't work with only one + // interior point, because we have to do something similar to the + // _first_ cell, and these two fudges cancel out and we end up + // indexing into the boundary anyway. + // TODO(peter): Can we remove this if we apply (dirichlet?) BCs to + // the X derivatives? Note that we need at least _2_ if (i_corn >= xend) { i_corn = xend - 1; t_x = 1.0; @@ -217,96 +251,112 @@ void XZHermiteSpline::calcWeights(const Field3D& delta_x, const Field3D& delta_z h11_x[i] = (t_x * t_x * t_x) - (t_x * t_x); h11_z[i] = (t_z * t_z * t_z) - (t_z * t_z); -#if USE_NEW_WEIGHTS - - for (int w = 0; w < 16; ++w) { - newWeights[w][i] = 0; - } - // The distribution of our weights: - // 0 4 8 12 - // 1 5 9 13 - // 2 6 10 14 - // 3 7 11 15 - // e.g. 1 == ic.xm(); 4 == ic.zm(); 5 == ic; 7 == ic.zp(2); - - // f[ic] * h00_x[i] + f[icxp] * h01_x[i] + fx[ic] * h10_x[i] + fx[icxp] * h11_x[i]; - newWeights[5][i] += h00_x[i] * h00_z[i]; - newWeights[9][i] += h01_x[i] * h00_z[i]; - newWeights[9][i] += h10_x[i] * h00_z[i] / 2; - newWeights[1][i] -= h10_x[i] * h00_z[i] / 2; - newWeights[13][i] += h11_x[i] * h00_z[i] / 2; - newWeights[5][i] -= h11_x[i] * h00_z[i] / 2; - - // f[iczp] * h00_x[i] + f[icxpzp] * h01_x[i] + - // fx[iczp] * h10_x[i] + fx[icxpzp] * h11_x[i]; - newWeights[6][i] += h00_x[i] * h01_z[i]; - newWeights[10][i] += h01_x[i] * h01_z[i]; - newWeights[10][i] += h10_x[i] * h01_z[i] / 2; - newWeights[2][i] -= h10_x[i] * h01_z[i] / 2; - newWeights[14][i] += h11_x[i] * h01_z[i] / 2; - newWeights[6][i] -= h11_x[i] * h01_z[i] / 2; - - // fz[ic] * h00_x[i] + fz[icxp] * h01_x[i] + - // fxz[ic] * h10_x[i]+ fxz[icxp] * h11_x[i]; - newWeights[6][i] += h00_x[i] * h10_z[i] / 2; - newWeights[4][i] -= h00_x[i] * h10_z[i] / 2; - newWeights[10][i] += h01_x[i] * h10_z[i] / 2; - newWeights[8][i] -= h01_x[i] * h10_z[i] / 2; - newWeights[10][i] += h10_x[i] * h10_z[i] / 4; - newWeights[8][i] -= h10_x[i] * h10_z[i] / 4; - newWeights[2][i] -= h10_x[i] * h10_z[i] / 4; - newWeights[0][i] += h10_x[i] * h10_z[i] / 4; - newWeights[14][i] += h11_x[i] * h10_z[i] / 4; - newWeights[12][i] -= h11_x[i] * h10_z[i] / 4; - newWeights[6][i] -= h11_x[i] * h10_z[i] / 4; - newWeights[4][i] += h11_x[i] * h10_z[i] / 4; - - // fz[iczp] * h00_x[i] + fz[icxpzp] * h01_x[i] + - // fxz[iczp] * h10_x[i] + fxz[icxpzp] * h11_x[i]; - newWeights[7][i] += h00_x[i] * h11_z[i] / 2; - newWeights[5][i] -= h00_x[i] * h11_z[i] / 2; - newWeights[11][i] += h01_x[i] * h11_z[i] / 2; - newWeights[9][i] -= h01_x[i] * h11_z[i] / 2; - newWeights[11][i] += h10_x[i] * h11_z[i] / 4; - newWeights[9][i] -= h10_x[i] * h11_z[i] / 4; - newWeights[3][i] -= h10_x[i] * h11_z[i] / 4; - newWeights[1][i] += h10_x[i] * h11_z[i] / 4; - newWeights[15][i] += h11_x[i] * h11_z[i] / 4; - newWeights[13][i] -= h11_x[i] * h11_z[i] / 4; - newWeights[7][i] -= h11_x[i] * h11_z[i] / 4; - newWeights[5][i] += h11_x[i] * h11_z[i] / 4; -#ifdef HS_USE_PETSC - PetscInt idxn[1] = {conv.fromLocalToGlobal(x, y + y_offset, z)}; - // output.write("debug: {:d} -> {:d}: {:d}:{:d} -> {:d}:{:d}\n", - // conv.fromLocalToGlobal(x, y + y_offset, z), - // conv.fromMeshToGlobal(i_corn, y + y_offset, k_corner(x, y, z)), - // x, z, i_corn, k_corner(x, y, z)); - // ixstep = mesh->LocalNx * mesh->LocalNz; - for (int j = 0; j < 4; ++j) { - PetscInt idxm[4]; - PetscScalar vals[4]; - for (int k = 0; k < 4; ++k) { - idxm[k] = conv.fromMeshToGlobal(i_corn - 1 + j, y + y_offset, - k_corner(x, y, z) - 1 + k); - vals[k] = newWeights[j * 4 + k][i]; + if constexpr (imp_type == bout::details::implementation_type::new_weights + || imp_type == bout::details::implementation_type::petsc) { + for (int w = 0; w < 16; ++w) { + newWeights[w][i] = 0; } - MatSetValues(petscWeights, 1, idxn, 4, idxm, vals, INSERT_VALUES); - } -#endif + // The distribution of our weights: + // 0 4 8 12 + // 1 5 9 13 + // 2 6 10 14 + // 3 7 11 15 + // e.g. 1 == ic.xm(); 4 == ic.zm(); 5 == ic; 7 == ic.zp(2); + + // f[ic] * h00_x[i] + f[icxp] * h01_x[i] + fx[ic] * h10_x[i] + fx[icxp] * h11_x[i]; + newWeights[5][i] += h00_x[i] * h00_z[i]; + newWeights[9][i] += h01_x[i] * h00_z[i]; + newWeights[9][i] += h10_x[i] * h00_z[i] / 2; + newWeights[1][i] -= h10_x[i] * h00_z[i] / 2; + newWeights[13][i] += h11_x[i] * h00_z[i] / 2; + newWeights[5][i] -= h11_x[i] * h00_z[i] / 2; + + // f[iczp] * h00_x[i] + f[icxpzp] * h01_x[i] + + // fx[iczp] * h10_x[i] + fx[icxpzp] * h11_x[i]; + newWeights[6][i] += h00_x[i] * h01_z[i]; + newWeights[10][i] += h01_x[i] * h01_z[i]; + newWeights[10][i] += h10_x[i] * h01_z[i] / 2; + newWeights[2][i] -= h10_x[i] * h01_z[i] / 2; + newWeights[14][i] += h11_x[i] * h01_z[i] / 2; + newWeights[6][i] -= h11_x[i] * h01_z[i] / 2; + + // fz[ic] * h00_x[i] + fz[icxp] * h01_x[i] + + // fxz[ic] * h10_x[i]+ fxz[icxp] * h11_x[i]; + newWeights[6][i] += h00_x[i] * h10_z[i] / 2; + newWeights[4][i] -= h00_x[i] * h10_z[i] / 2; + newWeights[10][i] += h01_x[i] * h10_z[i] / 2; + newWeights[8][i] -= h01_x[i] * h10_z[i] / 2; + newWeights[10][i] += h10_x[i] * h10_z[i] / 4; + newWeights[8][i] -= h10_x[i] * h10_z[i] / 4; + newWeights[2][i] -= h10_x[i] * h10_z[i] / 4; + newWeights[0][i] += h10_x[i] * h10_z[i] / 4; + newWeights[14][i] += h11_x[i] * h10_z[i] / 4; + newWeights[12][i] -= h11_x[i] * h10_z[i] / 4; + newWeights[6][i] -= h11_x[i] * h10_z[i] / 4; + newWeights[4][i] += h11_x[i] * h10_z[i] / 4; + + // fz[iczp] * h00_x[i] + fz[icxpzp] * h01_x[i] + + // fxz[iczp] * h10_x[i] + fxz[icxpzp] * h11_x[i]; + newWeights[7][i] += h00_x[i] * h11_z[i] / 2; + newWeights[5][i] -= h00_x[i] * h11_z[i] / 2; + newWeights[11][i] += h01_x[i] * h11_z[i] / 2; + newWeights[9][i] -= h01_x[i] * h11_z[i] / 2; + newWeights[11][i] += h10_x[i] * h11_z[i] / 4; + newWeights[9][i] -= h10_x[i] * h11_z[i] / 4; + newWeights[3][i] -= h10_x[i] * h11_z[i] / 4; + newWeights[1][i] += h10_x[i] * h11_z[i] / 4; + newWeights[15][i] += h11_x[i] * h11_z[i] / 4; + newWeights[13][i] -= h11_x[i] * h11_z[i] / 4; + newWeights[7][i] -= h11_x[i] * h11_z[i] / 4; + newWeights[5][i] += h11_x[i] * h11_z[i] / 4; + if (imp_type == bout::details::implementation_type::petsc) { +#if BOUT_HAS_PETSC + PetscInt idxn[1] = {conv.fromLocalToGlobal(x, y + y_offset, z)}; + // output.write("debug: {:d} -> {:d}: {:d}:{:d} -> {:d}:{:d}\n", + // conv.fromLocalToGlobal(x, y + y_offset, z), + // conv.fromMeshToGlobal(i_corn, y + y_offset, k_corner(x, y, z)), + // x, z, i_corn, k_corner(x, y, z)); + // ixstep = mesh->LocalNx * mesh->LocalNz; + for (int j = 0; j < 4; ++j) { + PetscInt idxm[4]; + PetscScalar vals[4]; + for (int k = 0; k < 4; ++k) { + idxm[k] = conv.fromMeshToGlobal(i_corn - 1 + j, y + y_offset, + k_corner(x, y, z) - 1 + k); + vals[k] = newWeights[(j * 4) + k][i]; + } + MatSetValues(petscWeights, 1, idxn, 4, idxm, vals, INSERT_VALUES); + } #endif + } + } + if constexpr (monotonic) { + const auto gind = gf3daccess->xyzglobal(i_corn, y + y_offset + y_global_offset, + k_corner(x, y, z)); + gf3daccess->request(gind); + gf3daccess->request(gind.xp(1)); + gf3daccess->request(gind.zp(1)); + gf3daccess->request(gind.xp(1).zp(1)); + g3dinds[i] = {gind.ind, gind.xp(1).ind, gind.zp(1).ind, gind.xp(1).zp(1).ind}; + } } -#ifdef HS_USE_PETSC - MatAssemblyBegin(petscWeights, MAT_FINAL_ASSEMBLY); - MatAssemblyEnd(petscWeights, MAT_FINAL_ASSEMBLY); - if (!isInit) { - MatCreateVecs(petscWeights, &rhs, &result); - } - isInit = true; + if constexpr (imp_type == bout::details::implementation_type::petsc) { +#if BOUT_HAS_PETSC + MatAssemblyBegin(petscWeights, MAT_FINAL_ASSEMBLY); + MatAssemblyEnd(petscWeights, MAT_FINAL_ASSEMBLY); + if (!isInit) { + MatCreateVecs(petscWeights, &rhs, &result); + } + isInit = true; #endif + } } -void XZHermiteSpline::calcWeights(const Field3D& delta_x, const Field3D& delta_z, - const BoutMask& mask, const std::string& region) { +template +void XZHermiteSplineBase::calcWeights(const Field3D& delta_x, + const Field3D& delta_z, + const BoutMask& mask, + const std::string& region) { setMask(mask); calcWeights(delta_x, delta_z, region); } @@ -327,8 +377,14 @@ void XZHermiteSpline::calcWeights(const Field3D& delta_x, const Field3D& delta_z * (i, j+1, k+1) h01_z + h10_z / 2 * (i, j+1, k+2) h11_z / 2 */ +template std::vector -XZHermiteSpline::getWeightsForYApproximation(int i, int j, int k, int yoffset) { +XZHermiteSplineBase::getWeightsForYApproximation(int i, int j, int k, + int yoffset) { + if (localmesh->getNXPE() > 1) { + throw BoutException("It is likely that the function calling this is not handling the " + "result correctly."); + } const int nz = localmesh->LocalNz; const int k_mod = k_corner(i, j, k); const int k_mod_m1 = (k_mod > 0) ? (k_mod - 1) : (nz - 1); @@ -341,99 +397,158 @@ XZHermiteSpline::getWeightsForYApproximation(int i, int j, int k, int yoffset) { {i, j + yoffset, k_mod_p2, 0.5 * h11_z(i, j, k)}}; } -Field3D XZHermiteSpline::interpolate(const Field3D& f, const std::string& region) const { +template +Field3D XZHermiteSplineBase::interpolate( + const Field3D& f, [[maybe_unused]] const std::string& region) const { ASSERT1(f.getMesh() == localmesh); Field3D f_interp{emptyFrom(f)}; - const auto region2 = - y_offset == 0 ? "RGN_NOY" : fmt::format("RGN_YPAR_{:+d}", y_offset); - -#if USE_NEW_WEIGHTS -#ifdef HS_USE_PETSC - BoutReal* ptr; - const BoutReal* cptr; - VecGetArray(rhs, &ptr); - BOUT_FOR(i, f.getRegion("RGN_NOY")) { ptr[int(i)] = f[i]; } - VecRestoreArray(rhs, &ptr); - MatMult(petscWeights, rhs, result); - VecGetArrayRead(result, &cptr); - BOUT_FOR(i, f.getRegion(region2)) { - f_interp[i] = cptr[int(i)]; - ASSERT2(std::isfinite(cptr[int(i)])); + const auto region2 = y_offset != 0 ? fmt::format("RGN_YPAR_{:+d}", y_offset) : region; + + std::unique_ptr gf; + if constexpr (monotonic) { + gf = gf3daccess->communicate_asPtr(f); } - VecRestoreArrayRead(result, &cptr); -#else - BOUT_FOR(i, getRegion(region)) { - auto ic = i_corner[i]; - auto iyp = i.yp(y_offset); - - f_interp[iyp] = 0; - for (int w = 0; w < 4; ++w) { - f_interp[iyp] += newWeights[w * 4 + 0][i] * f[ic.zm().xp(w - 1)]; - f_interp[iyp] += newWeights[w * 4 + 1][i] * f[ic.xp(w - 1)]; - f_interp[iyp] += newWeights[w * 4 + 2][i] * f[ic.zp().xp(w - 1)]; - f_interp[iyp] += newWeights[w * 4 + 3][i] * f[ic.zp(2).xp(w - 1)]; + + if constexpr (imp_type == bout::details::implementation_type::petsc) { +#if BOUT_HAS_PETSC + BoutReal* ptr = nullptr; + const BoutReal* cptr = nullptr; + VecGetArray(rhs, &ptr); + BOUT_FOR(i, f.getRegion("RGN_NOY")) { ptr[int(i)] = f[i]; } + VecRestoreArray(rhs, &ptr); + MatMult(petscWeights, rhs, result); + VecGetArrayRead(result, &cptr); + BOUT_FOR(i, f.getRegion(region2)) { + f_interp[i] = cptr[int(i)]; + if constexpr (monotonic) { + const auto iyp = i; + const auto i = iyp.ym(y_offset); + const auto corners = {(*gf)[IndG3D(g3dinds[i][0])], (*gf)[IndG3D(g3dinds[i][1])], + (*gf)[IndG3D(g3dinds[i][2])], (*gf)[IndG3D(g3dinds[i][3])]}; + const auto minmax = std::minmax(corners); + const auto diff = + ((minmax.second - minmax.first) * rel_fac_monotonic) + abs_fac_monotonic; + f_interp[iyp] = std::max(f_interp[iyp], minmax.first - diff); + f_interp[iyp] = std::min(f_interp[iyp], minmax.second + diff); + } + ASSERT2(std::isfinite(cptr[int(i)])); } - } + VecRestoreArrayRead(result, &cptr); #endif -#else - // Derivatives are used for tension and need to be on dimensionless - // coordinates - - // f has been communcated, and thus we can assume that the x-boundaries are - // also valid in the y-boundary. Thus the differentiated field needs no - // extra comms. - Field3D fx = bout::derivatives::index::DDX(f, CELL_DEFAULT, "DEFAULT", region2); - Field3D fz = bout::derivatives::index::DDZ(f, CELL_DEFAULT, "DEFAULT", region2); - Field3D fxz = bout::derivatives::index::DDZ(fx, CELL_DEFAULT, "DEFAULT", region2); - - BOUT_FOR(i, getRegion(region)) { - const auto iyp = i.yp(y_offset); - - const auto ic = i_corner[i]; - const auto iczp = ic.zp(); - const auto icxp = ic.xp(); - const auto icxpzp = iczp.xp(); - - // Interpolate f in X at Z - const BoutReal f_z = - f[ic] * h00_x[i] + f[icxp] * h01_x[i] + fx[ic] * h10_x[i] + fx[icxp] * h11_x[i]; - - // Interpolate f in X at Z+1 - const BoutReal f_zp1 = f[iczp] * h00_x[i] + f[icxpzp] * h01_x[i] + fx[iczp] * h10_x[i] - + fx[icxpzp] * h11_x[i]; - - // Interpolate fz in X at Z - const BoutReal fz_z = fz[ic] * h00_x[i] + fz[icxp] * h01_x[i] + fxz[ic] * h10_x[i] - + fxz[icxp] * h11_x[i]; - - // Interpolate fz in X at Z+1 - const BoutReal fz_zp1 = fz[iczp] * h00_x[i] + fz[icxpzp] * h01_x[i] - + fxz[iczp] * h10_x[i] + fxz[icxpzp] * h11_x[i]; - - // Interpolate in Z - f_interp[iyp] = - +f_z * h00_z[i] + f_zp1 * h01_z[i] + fz_z * h10_z[i] + fz_zp1 * h11_z[i]; + } - ASSERT2(std::isfinite(f_interp[iyp]) || i.x() < localmesh->xstart - || i.x() > localmesh->xend); + if constexpr (imp_type == bout::details::implementation_type::new_weights) { + BOUT_FOR(i, getRegion(region)) { + auto ic = i_corner[i]; + auto iyp = i.yp(y_offset); + + f_interp[iyp] = 0; + for (int w = 0; w < 4; ++w) { + f_interp[iyp] += newWeights[(w * 4) + 0][i] * f[ic.zm().xp(w - 1)]; + f_interp[iyp] += newWeights[(w * 4) + 1][i] * f[ic.xp(w - 1)]; + f_interp[iyp] += newWeights[(w * 4) + 2][i] * f[ic.zp().xp(w - 1)]; + f_interp[iyp] += newWeights[(w * 4) + 3][i] * f[ic.zp(2).xp(w - 1)]; + } + if constexpr (monotonic) { + const auto corners = {(*gf)[IndG3D(g3dinds[i][0])], (*gf)[IndG3D(g3dinds[i][1])], + (*gf)[IndG3D(g3dinds[i][2])], (*gf)[IndG3D(g3dinds[i][3])]}; + const auto minmax = std::minmax(corners); + const auto diff = + ((minmax.second - minmax.first) * rel_fac_monotonic) + abs_fac_monotonic; + f_interp[iyp] = std::max(f_interp[iyp], minmax.first - diff); + f_interp[iyp] = std::min(f_interp[iyp], minmax.second + diff); + } + ASSERT2(std::isfinite(f_interp[iyp])); + } + } + if constexpr (imp_type == bout::details::implementation_type::legacy) { + // Legacy interpolation + // TODO(peter): Should we apply dirichlet BCs to derivatives? + // Derivatives are used for tension and need to be on dimensionless + // coordinates + + // f has been communcated, and thus we can assume that the x-boundaries are + // also valid in the y-boundary. Thus the differentiated field needs no + // extra comms. + // TODO(dave) Add assert that we do not use z-splitting or z-guards. + Field3D fx = bout::derivatives::index::DDX(f, CELL_DEFAULT, "DEFAULT", region2); + Field3D fz = bout::derivatives::index::DDZ(f, CELL_DEFAULT, "DEFAULT", region2); + Field3D fxz = bout::derivatives::index::DDZ(fx, CELL_DEFAULT, "DEFAULT", region2); + + BOUT_FOR(i, getRegion(region)) { + const auto iyp = i.yp(y_offset); + + const auto ic = i_corner[i]; + const auto iczp = ic.zp(); + const auto icxp = ic.xp(); + const auto icxpzp = iczp.xp(); + + // Interpolate f in X at Z + const BoutReal f_z = + f[ic] * h00_x[i] + f[icxp] * h01_x[i] + fx[ic] * h10_x[i] + fx[icxp] * h11_x[i]; + + // Interpolate f in X at Z+1 + const BoutReal f_zp1 = f[iczp] * h00_x[i] + f[icxpzp] * h01_x[i] + + fx[iczp] * h10_x[i] + fx[icxpzp] * h11_x[i]; + + // Interpolate fz in X at Z + const BoutReal fz_z = fz[ic] * h00_x[i] + fz[icxp] * h01_x[i] + fxz[ic] * h10_x[i] + + fxz[icxp] * h11_x[i]; + + // Interpolate fz in X at Z+1 + const BoutReal fz_zp1 = fz[iczp] * h00_x[i] + fz[icxpzp] * h01_x[i] + + fxz[iczp] * h10_x[i] + fxz[icxpzp] * h11_x[i]; + + // Interpolate in Z + f_interp[iyp] = + +f_z * h00_z[i] + f_zp1 * h01_z[i] + fz_z * h10_z[i] + fz_zp1 * h11_z[i]; + + if constexpr (monotonic) { + const auto corners = {(*gf)[IndG3D(g3dinds[i][0])], (*gf)[IndG3D(g3dinds[i][1])], + (*gf)[IndG3D(g3dinds[i][2])], (*gf)[IndG3D(g3dinds[i][3])]}; + const auto minmax = std::minmax(corners); + const auto diff = + ((minmax.second - minmax.first) * rel_fac_monotonic) + abs_fac_monotonic; + f_interp[iyp] = std::max(f_interp[iyp], minmax.first - diff); + f_interp[iyp] = std::min(f_interp[iyp], minmax.second + diff); + } + ASSERT2(std::isfinite(f_interp[iyp]) || i.x() < localmesh->xstart + || i.x() > localmesh->xend); + } } -#endif f_interp.setRegion(region2); ASSERT2(f_interp.getRegionID()); return f_interp; } -Field3D XZHermiteSpline::interpolate(const Field3D& f, const Field3D& delta_x, - const Field3D& delta_z, const std::string& region) { +template +Field3D XZHermiteSplineBase::interpolate(const Field3D& f, + const Field3D& delta_x, + const Field3D& delta_z, + const std::string& region) { calcWeights(delta_x, delta_z, region); return interpolate(f, region); } -Field3D XZHermiteSpline::interpolate(const Field3D& f, const Field3D& delta_x, - const Field3D& delta_z, const BoutMask& mask, - const std::string& region) { +template +Field3D XZHermiteSplineBase::interpolate(const Field3D& f, + const Field3D& delta_x, + const Field3D& delta_z, + const BoutMask& mask, + const std::string& region) { calcWeights(delta_x, delta_z, mask, region); return interpolate(f, region); } + +// ensure they are instantiated +template class XZHermiteSplineBase; +template class XZHermiteSplineBase; +template class XZHermiteSplineBase; +template class XZHermiteSplineBase; +#if BOUT_HAS_PETSC +template class XZHermiteSplineBase; +template class XZHermiteSplineBase; +#endif diff --git a/src/mesh/interpolation/interpolation_z.cxx b/src/mesh/interpolation/interpolation_z.cxx index 8d39e6baa8..e743ec8555 100644 --- a/src/mesh/interpolation/interpolation_z.cxx +++ b/src/mesh/interpolation/interpolation_z.cxx @@ -20,8 +20,10 @@ * **************************************************************************/ +#include #include #include +#include ZInterpolation::ZInterpolation(int y_offset, Mesh* mesh, Region region_in) : localmesh(mesh == nullptr ? bout::globals::mesh : mesh), region(region_in), diff --git a/src/mesh/interpolation/lagrange_4pt_xz.cxx b/src/mesh/interpolation/lagrange_4pt_xz.cxx index e16b2699d1..c82c9fa36e 100644 --- a/src/mesh/interpolation/lagrange_4pt_xz.cxx +++ b/src/mesh/interpolation/lagrange_4pt_xz.cxx @@ -20,6 +20,7 @@ * **************************************************************************/ +#include "fmt/format.h" #include "bout/globals.hxx" #include "bout/interpolation_xz.hxx" #include "bout/mesh.hxx" @@ -133,7 +134,10 @@ Field3D XZLagrange4pt::interpolate(const Field3D& f, const std::string& region) // Then in X f_interp(x, y_next, z) = lagrange_4pt(xvals, t_x(x, y, z)); + ASSERT2(std::isfinite(f_interp(x, y_next, z))); } + const auto region2 = y_offset != 0 ? fmt::format("RGN_YPAR_{:+d}", y_offset) : region; + f_interp.setRegion(region2); return f_interp; } diff --git a/src/mesh/interpolation/monotonic_hermite_spline_xz.cxx b/src/mesh/interpolation/monotonic_hermite_spline_xz.cxx deleted file mode 100644 index f23bfd499e..0000000000 --- a/src/mesh/interpolation/monotonic_hermite_spline_xz.cxx +++ /dev/null @@ -1,101 +0,0 @@ -/************************************************************************** - * Copyright 2018 B.D.Dudson, P. Hill - * - * Contact: Ben Dudson, bd512@york.ac.uk - * - * This file is part of BOUT++. - * - * BOUT++ is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * BOUT++ is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with BOUT++. If not, see . - * - **************************************************************************/ - -#include "bout/globals.hxx" -#include "bout/index_derivs_interface.hxx" -#include "bout/interpolation_xz.hxx" -#include "bout/mesh.hxx" - -#include - -Field3D XZMonotonicHermiteSpline::interpolate(const Field3D& f, - const std::string& region) const { - ASSERT1(f.getMesh() == localmesh); - Field3D f_interp(f.getMesh()); - f_interp.allocate(); - - // Derivatives are used for tension and need to be on dimensionless - // coordinates - Field3D fx = bout::derivatives::index::DDX(f, CELL_DEFAULT, "DEFAULT"); - Field3D fz = bout::derivatives::index::DDZ(f, CELL_DEFAULT, "DEFAULT", "RGN_ALL"); - localmesh->communicate_no_slices(fx, fz); - Field3D fxz = bout::derivatives::index::DDX(fz, CELL_DEFAULT, "DEFAULT"); - localmesh->communicate_no_slices(fxz); - - const auto curregion{getRegion(region)}; - BOUT_FOR(i, curregion) { - const auto iyp = i.yp(y_offset); - - const auto ic = i_corner[i]; - const auto iczp = ic.zp(); - const auto icxp = ic.xp(); - const auto icxpzp = iczp.xp(); - - // Interpolate f in X at Z - const BoutReal f_z = - f[ic] * h00_x[i] + f[icxp] * h01_x[i] + fx[ic] * h10_x[i] + fx[icxp] * h11_x[i]; - - // Interpolate f in X at Z+1 - const BoutReal f_zp1 = f[iczp] * h00_x[i] + f[icxpzp] * h01_x[i] + fx[iczp] * h10_x[i] - + fx[icxpzp] * h11_x[i]; - - // Interpolate fz in X at Z - const BoutReal fz_z = fz[ic] * h00_x[i] + fz[icxp] * h01_x[i] + fxz[ic] * h10_x[i] - + fxz[icxp] * h11_x[i]; - - // Interpolate fz in X at Z+1 - const BoutReal fz_zp1 = fz[iczp] * h00_x[i] + fz[icxpzp] * h01_x[i] - + fxz[iczp] * h10_x[i] + fxz[icxpzp] * h11_x[i]; - - // Interpolate in Z - BoutReal result = - +f_z * h00_z[i] + f_zp1 * h01_z[i] + fz_z * h10_z[i] + fz_zp1 * h11_z[i]; - - ASSERT2(std::isfinite(result) || i.x() < localmesh->xstart - || i.x() > localmesh->xend); - - // Monotonicity - // Force the interpolated result to be in the range of the - // neighbouring cell values. This prevents unphysical overshoots, - // but also degrades accuracy near maxima and minima. - // Perhaps should only impose near boundaries, since that is where - // problems most obviously occur. - const BoutReal localmax = BOUTMAX(f[ic], f[icxp], f[iczp], f[icxpzp]); - - const BoutReal localmin = BOUTMIN(f[ic], f[icxp], f[iczp], f[icxpzp]); - - ASSERT2(std::isfinite(localmax) || i.x() < localmesh->xstart - || i.x() > localmesh->xend); - ASSERT2(std::isfinite(localmin) || i.x() < localmesh->xstart - || i.x() > localmesh->xend); - - if (result > localmax) { - result = localmax; - } - if (result < localmin) { - result = localmin; - } - - f_interp[iyp] = result; - } - return f_interp; -} diff --git a/src/mesh/interpolation_xz.cxx b/src/mesh/interpolation_xz.cxx index f7f0b457f2..ec8bcc0502 100644 --- a/src/mesh/interpolation_xz.cxx +++ b/src/mesh/interpolation_xz.cxx @@ -23,9 +23,9 @@ * **************************************************************************/ +#include "parallel/fci_comm.hxx" #include #include -#include #include #include @@ -36,7 +36,6 @@ const char* strLocation(CELL_LOC loc) { return toString(loc).c_str(); } const Field3D interpolate(const Field3D& f, const Field3D& delta_x, const Field3D& delta_z) { - TRACE("Interpolating 3D field"); XZLagrange4pt interpolateMethod{f.getMesh()}; return interpolateMethod.interpolate(f, delta_x, delta_z); } @@ -47,8 +46,6 @@ const Field3D interpolate(const Field2D& f, const Field3D& delta_x, } const Field3D interpolate(const Field2D& f, const Field3D& delta_x) { - TRACE("interpolate(Field2D, Field3D)"); - Mesh* mesh = f.getMesh(); ASSERT1(mesh == delta_x.getMesh()); Field3D result{emptyFrom(delta_x)}; @@ -90,6 +87,14 @@ namespace { RegisterXZInterpolation registerinterphermitespline{"hermitespline"}; RegisterXZInterpolation registerinterpmonotonichermitespline{ "monotonichermitespline"}; +RegisterXZInterpolation registerinterphermitesplines{ + "hermitesplineserial"}; +RegisterXZInterpolation + registerinterpmonotonichermitesplines{"monotonichermitesplineserial"}; +RegisterXZInterpolation registerinterphermitesplinel{ + "hermitesplinelegacy"}; +RegisterXZInterpolation + registerinterpmonotonichermitesplinel{"monotonichermitesplinelegacy"}; RegisterXZInterpolation registerinterplagrange4pt{"lagrange4pt"}; RegisterXZInterpolation registerinterpbilinear{"bilinear"}; } // namespace diff --git a/src/mesh/invert3x3.hxx b/src/mesh/invert3x3.hxx index c011f55bf7..746765d913 100644 --- a/src/mesh/invert3x3.hxx +++ b/src/mesh/invert3x3.hxx @@ -37,7 +37,6 @@ /// return. Otherwise, an empty `std::optional` is return namespace bout { inline std::optional invert3x3(Matrix& a) { - TRACE("invert3x3"); // Calculate the first co-factors const BoutReal A = a(1, 1) * a(2, 2) - a(1, 2) * a(2, 1); diff --git a/src/mesh/mesh.cxx b/src/mesh/mesh.cxx index 1d3f3b0dbe..9a98b5848a 100644 --- a/src/mesh/mesh.cxx +++ b/src/mesh/mesh.cxx @@ -1,14 +1,38 @@ +#include +#include #include +#include +#include #include #include +#include +#include +#include +#include +#include #include #include #include #include +#include +#include #include +#include +#include +#include +#include #include - -#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include #include "impls/bout/boutmesh.hxx" @@ -298,7 +322,6 @@ bool Mesh::sourceHasYBoundaryGuards() { return source->hasYBoundaryGuards(); } **************************************************************************/ void Mesh::communicateXZ(FieldGroup& g) { - TRACE("Mesh::communicate(FieldGroup&)"); // Send data comm_handle h = sendX(g); @@ -308,7 +331,6 @@ void Mesh::communicateXZ(FieldGroup& g) { } void Mesh::communicateYZ(FieldGroup& g) { - TRACE("Mesh::communicate(FieldGroup&)"); // Send data comm_handle h = sendY(g); @@ -325,7 +347,6 @@ void Mesh::communicateYZ(FieldGroup& g) { } void Mesh::communicate(FieldGroup& g) { - TRACE("Mesh::communicate(FieldGroup&)"); if (include_corner_cells) { // Send data in y-direction @@ -355,38 +376,28 @@ void Mesh::communicate(FieldGroup& g) { } } -/// This is a bit of a hack for now to get FieldPerp communications -/// The FieldData class needs to be changed to accomodate FieldPerp objects -void Mesh::communicate(FieldPerp& f) { - comm_handle recv[2]; - - int nin = xstart; // Number of x points in inner guard cell - int nout = LocalNx - xend - 1; // Number of x points in outer guard cell - - // Post receives for guard cell regions - - recv[0] = irecvXIn(f[0], nin * LocalNz, 0); - recv[1] = irecvXOut(f[xend + 1], nout * LocalNz, 1); - - // Send data - sendXIn(f[xstart], nin * LocalNz, 1); - sendXOut(f[xend - nout + 1], nout * LocalNz, 0); +int Mesh::msg_len(const std::vector& var_list, int xge, int xlt, int yge, + int ylt) const { + int len = 0; - // Wait for receive - wait(recv[0]); - wait(recv[1]); -} + using enum Field::FieldType; -int Mesh::msg_len(const std::vector& var_list, int xge, int xlt, int yge, - int ylt) { - int len = 0; + const auto x_length = xlt - xge; + const auto y_length = ylt - yge; + const auto z_length = LocalNz; /// Loop over variables for (const auto& var : var_list) { - if (var->is3D()) { - len += (xlt - xge) * (ylt - yge) * LocalNz * var->elementSize(); - } else { - len += (xlt - xge) * (ylt - yge) * var->elementSize(); + switch (var->field_type()) { + case field3d: + len += x_length * y_length * z_length * var->elementSize(); + break; + case field2d: + len += x_length * y_length * var->elementSize(); + break; + case fieldperp: + len += x_length * z_length * var->elementSize(); + break; } } @@ -495,7 +506,7 @@ int Mesh::globalStartIndex2D() { int Mesh::globalStartIndexPerp() { int localSize = localSizePerp(); int cumulativeSize = 0; - mpi->MPI_Scan(&localSize, &cumulativeSize, 1, MPI_INT, MPI_SUM, getXcomm()); + mpi->MPI_Scan(&localSize, &cumulativeSize, 1, MPI_INT, MPI_SUM, getXZcomm()); return cumulativeSize - localSize; } @@ -512,11 +523,11 @@ const std::vector Mesh::readInts(const std::string& name, int n) { if (source->hasVar(name)) { if (!source->get(this, result, name, n, 0)) { // Error reading - throw BoutException(_("Could not read integer array '{:s}'\n"), name.c_str()); + throw BoutException(_f("Could not read integer array '{:s}'\n"), name.c_str()); } } else { // Not found - throw BoutException(_("Missing integer array {:s}\n"), name.c_str()); + throw BoutException(_f("Missing integer array {:s}\n"), name.c_str()); } return result; @@ -540,7 +551,7 @@ Mesh::createDefaultCoordinates(const CELL_LOC location, const Region<>& Mesh::getRegion3D(const std::string& region_name) const { const auto found = regionMap3D.find(region_name); if (found == end(regionMap3D)) { - throw BoutException(_("Couldn't find region {:s} in regionMap3D"), region_name); + throw BoutException(_f("Couldn't find region {:s} in regionMap3D"), region_name); } return region3D[found->second]; } @@ -548,7 +559,7 @@ const Region<>& Mesh::getRegion3D(const std::string& region_name) const { size_t Mesh::getRegionID(const std::string& region_name) const { const auto found = regionMap3D.find(region_name); if (found == end(regionMap3D)) { - throw BoutException(_("Couldn't find region {:s} in regionMap3D"), region_name); + throw BoutException(_f("Couldn't find region {:s} in regionMap3D"), region_name); } return found->second; } @@ -556,7 +567,7 @@ size_t Mesh::getRegionID(const std::string& region_name) const { const Region& Mesh::getRegion2D(const std::string& region_name) const { const auto found = regionMap2D.find(region_name); if (found == end(regionMap2D)) { - throw BoutException(_("Couldn't find region {:s} in regionMap2D"), region_name); + throw BoutException(_f("Couldn't find region {:s} in regionMap2D"), region_name); } return found->second; } @@ -564,7 +575,7 @@ const Region& Mesh::getRegion2D(const std::string& region_name) const { const Region& Mesh::getRegionPerp(const std::string& region_name) const { const auto found = regionMapPerp.find(region_name); if (found == end(regionMapPerp)) { - throw BoutException(_("Couldn't find region {:s} in regionMapPerp"), region_name); + throw BoutException(_f("Couldn't find region {:s} in regionMapPerp"), region_name); } return found->second; } @@ -583,8 +594,8 @@ bool Mesh::hasRegionPerp(const std::string& region_name) const { void Mesh::addRegion3D(const std::string& region_name, const Region<>& region) { if (regionMap3D.count(region_name)) { - throw BoutException(_("Trying to add an already existing region {:s} to regionMap3D"), - region_name); + throw BoutException( + _f("Trying to add an already existing region {:s} to regionMap3D"), region_name); } std::optional id; @@ -601,27 +612,28 @@ void Mesh::addRegion3D(const std::string& region_name, const Region<>& region) { regionMap3D[region_name] = id.value(); - output_verbose.write(_("Registered region 3D {:s}"), region_name); + output_verbose.write(_f("Registered region 3D {:s}"), region_name); output_verbose << "\n:\t" << region.getStats() << "\n"; } void Mesh::addRegion2D(const std::string& region_name, const Region& region) { if (regionMap2D.count(region_name)) { - throw BoutException(_("Trying to add an already existing region {:s} to regionMap2D"), - region_name); + throw BoutException( + _f("Trying to add an already existing region {:s} to regionMap2D"), region_name); } regionMap2D[region_name] = region; - output_verbose.write(_("Registered region 2D {:s}"), region_name); + output_verbose.write(_f("Registered region 2D {:s}"), region_name); output_verbose << "\n:\t" << region.getStats() << "\n"; } void Mesh::addRegionPerp(const std::string& region_name, const Region& region) { if (regionMapPerp.count(region_name)) { throw BoutException( - _("Trying to add an already existing region {:s} to regionMapPerp"), region_name); + _f("Trying to add an already existing region {:s} to regionMapPerp"), + region_name); } regionMapPerp[region_name] = region; - output_verbose.write(_("Registered region Perp {:s}"), region_name); + output_verbose.write(_f("Registered region Perp {:s}"), region_name); output_verbose << "\n:\t" << region.getStats() << "\n"; } @@ -657,6 +669,12 @@ void Mesh::createDefaultRegions() { + getRegion3D("RGN_YGUARDS") + getRegion3D("RGN_ZGUARDS")) .unique()); + for (int offset_ = -ystart; offset_ <= ystart; ++offset_) { + const auto region = fmt::format("RGN_YPAR_{:+d}", offset_); + addRegion3D(region, Region(xstart, xend, ystart + offset_, yend + offset_, 0, + LocalNz - 1, LocalNy, LocalNz)); + } + //2D regions addRegion2D("RGN_ALL", Region(0, LocalNx - 1, 0, LocalNy - 1, 0, 0, LocalNy, 1, maxregionblocksize)); diff --git a/src/mesh/parallel/fci.cxx b/src/mesh/parallel/fci.cxx index e8d3af1cdb..fa965a335a 100644 --- a/src/mesh/parallel/fci.cxx +++ b/src/mesh/parallel/fci.cxx @@ -37,79 +37,100 @@ **************************************************************************/ #include "fci.hxx" + +#include "bout/assert.hxx" +#include "bout/bout_types.hxx" +#include "bout/boutexception.hxx" +#include "bout/field2d.hxx" +#include "bout/field3d.hxx" +#include "bout/field_data.hxx" +#include "bout/mesh.hxx" +#include "bout/msg_stack.hxx" +#include "bout/options.hxx" #include "bout/parallel_boundary_op.hxx" #include "bout/parallel_boundary_region.hxx" -#include -#include -#include -#include -#include +#include "bout/paralleltransform.hxx" +#include "bout/region.hxx" + +#include +#include +#include +#include +#include +#include #include +#include -FCIMap::FCIMap(Mesh& mesh, const Coordinates::FieldMetric& UNUSED(dy), Options& options, - int offset_, const std::shared_ptr& inner_boundary, +using namespace std::string_view_literals; + +FCIMap::FCIMap(Mesh& mesh, [[maybe_unused]] const Coordinates::FieldMetric& dy, + Options& options, int offset, + const std::shared_ptr& inner_boundary, const std::shared_ptr& outer_boundary, bool zperiodic) - : map_mesh(mesh), offset(offset_), - region_no_boundary(map_mesh.getRegion("RGN_NOBNDRY")), + : map_mesh(&mesh), offset_(offset), + region_no_boundary(map_mesh->getRegion("RGN_NOBNDRY")), corner_boundary_mask(map_mesh) { - TRACE("Creating FCIMAP for direction {:d}", offset); + TRACE("Creating FCIMAP for direction {:d}", offset_); - if (offset == 0) { + if (offset_ == 0) { throw BoutException( "FCIMap called with offset = 0; You probably didn't mean to do that"); } auto& interpolation_options = options["xzinterpolation"]; - interp = - XZInterpolationFactory::getInstance().create(&interpolation_options, &map_mesh); - interp->setYOffset(offset); + interp = XZInterpolationFactory::getInstance().create(&interpolation_options, map_mesh); + interp->setYOffset(offset_); interp_corner = - XZInterpolationFactory::getInstance().create(&interpolation_options, &map_mesh); - interp_corner->setYOffset(offset); + XZInterpolationFactory::getInstance().create(&interpolation_options, map_mesh); + interp_corner->setYOffset(offset_); // Index-space coordinates of forward/backward points - Field3D xt_prime{&map_mesh}, zt_prime{&map_mesh}; + Field3D xt_prime{map_mesh}; + Field3D zt_prime{map_mesh}; // Real-space coordinates of grid points - Field3D R{&map_mesh}, Z{&map_mesh}; + Field3D R{map_mesh}; + Field3D Z{map_mesh}; // Real-space coordinates of forward/backward points - Field3D R_prime{&map_mesh}, Z_prime{&map_mesh}; + Field3D R_prime{map_mesh}; + Field3D Z_prime{map_mesh}; - map_mesh.get(R, "R", 0.0, false); - map_mesh.get(Z, "Z", 0.0, false); + map_mesh->get(R, "R", 0.0, false); + map_mesh->get(Z, "Z", 0.0, false); // Get a unique name for a field based on the sign/magnitude of the offset - const auto parallel_slice_field_name = [&](std::string field) -> std::string { - const std::string direction = (offset > 0) ? "forward" : "backward"; + const auto parallel_slice_field_name = [&](std::string_view field) -> std::string { + const auto direction = (offset_ > 0) ? "forward"sv : "backward"sv; // We only have a suffix for parallel slices beyond the first // This is for backwards compatibility - const std::string slice_suffix = - (std::abs(offset) > 1) ? "_" + std::to_string(std::abs(offset)) : ""; - return direction + "_" + field + slice_suffix; + if (std::abs(offset_) == 1) { + return fmt::format("{}_{}", direction, field); + } + return fmt::format("{}_{}_{}", direction, field, std::abs(offset_)); }; // If we can't read in any of these fields, things will silently not // work, so best throw - if (map_mesh.get(xt_prime, parallel_slice_field_name("xt_prime"), 0.0, false) != 0) { + if (map_mesh->get(xt_prime, parallel_slice_field_name("xt_prime"), 0.0, false) != 0) { throw BoutException("Could not read {:s} from grid file!\n" " Either add it to the grid file, or reduce MYG", parallel_slice_field_name("xt_prime")); } - if (map_mesh.get(zt_prime, parallel_slice_field_name("zt_prime"), 0.0, false) != 0) { + if (map_mesh->get(zt_prime, parallel_slice_field_name("zt_prime"), 0.0, false) != 0) { throw BoutException("Could not read {:s} from grid file!\n" " Either add it to the grid file, or reduce MYG", parallel_slice_field_name("zt_prime")); } - if (map_mesh.get(R_prime, parallel_slice_field_name("R"), 0.0, false) != 0) { + if (map_mesh->get(R_prime, parallel_slice_field_name("R"), 0.0, false) != 0) { throw BoutException("Could not read {:s} from grid file!\n" " Either add it to the grid file, or reduce MYG", parallel_slice_field_name("R")); } - if (map_mesh.get(Z_prime, parallel_slice_field_name("Z"), 0.0, false) != 0) { + if (map_mesh->get(Z_prime, parallel_slice_field_name("Z"), 0.0, false) != 0) { throw BoutException("Could not read {:s} from grid file!\n" " Either add it to the grid file, or reduce MYG", parallel_slice_field_name("Z")); @@ -157,25 +178,26 @@ FCIMap::FCIMap(Mesh& mesh, const Coordinates::FieldMetric& UNUSED(dy), Options& interp->calcWeights(xt_prime, zt_prime); } - const int ncz = map_mesh.LocalNz; + const int ncz = map_mesh->LocalNz; BoutMask to_remove(map_mesh); - const int xend = - map_mesh.xstart + (map_mesh.xend - map_mesh.xstart + 1) * map_mesh.getNXPE() - 1; + const int xend = map_mesh->xstart + + ((map_mesh->xend - map_mesh->xstart + 1) * map_mesh->getNXPE()) - 1; // Serial loop because call to BoundaryRegionPar::addPoint // (probably?) can't be done in parallel BOUT_FOR_SERIAL(i, xt_prime.getRegion("RGN_NOBNDRY")) { // z is periodic, so make sure the z-index wraps around if (zperiodic) { - zt_prime[i] = zt_prime[i] - - ncz * (static_cast(zt_prime[i] / static_cast(ncz))); + zt_prime[i] = + zt_prime[i] + - (ncz * (static_cast(zt_prime[i] / static_cast(ncz)))); if (zt_prime[i] < 0.0) { zt_prime[i] += ncz; } } - if ((xt_prime[i] >= map_mesh.xstart) and (xt_prime[i] <= xend)) { + if ((xt_prime[i] >= map_mesh->xstart) and (xt_prime[i] <= xend)) { // Not a boundary continue; } @@ -215,7 +237,7 @@ FCIMap::FCIMap(Mesh& mesh, const Coordinates::FieldMetric& UNUSED(dy), Options& const BoutReal dR_dz = 0.5 * (R[i_zp] - R[i_zm]); const BoutReal dZ_dz = 0.5 * (Z[i_zp] - Z[i_zm]); - const BoutReal det = dR_dx * dZ_dz - dR_dz * dZ_dx; // Determinant of 2x2 matrix + const BoutReal det = (dR_dx * dZ_dz) - (dR_dz * dZ_dx); // Determinant of 2x2 matrix const BoutReal dR = R_prime[i] - R[i]; const BoutReal dZ = Z_prime[i] - Z[i]; @@ -228,9 +250,9 @@ FCIMap::FCIMap(Mesh& mesh, const Coordinates::FieldMetric& UNUSED(dy), Options& // outer boundary. However, if any of the surrounding points are negative, // that also means inner. So to differentiate between inner and outer we // need at least 2 points in the domain. - ASSERT2(map_mesh.xend - map_mesh.xstart >= 2); - auto boundary = (xt_prime[i] < map_mesh.xstart) ? inner_boundary : outer_boundary; - boundary->add_point(x, y, z, x + dx, y + 0.5 * offset, + ASSERT2(map_mesh->xend - map_mesh->xstart >= 2); + auto boundary = (xt_prime[i] < map_mesh->xstart) ? inner_boundary : outer_boundary; + boundary->add_point(x, y, z, x + dx, y + (0.5 * offset_), z + dz, // Intersection point in local index space 0.5, // Distance to intersection 1 // Default to that there is a point in the other direction @@ -239,22 +261,12 @@ FCIMap::FCIMap(Mesh& mesh, const Coordinates::FieldMetric& UNUSED(dy), Options& region_no_boundary = region_no_boundary.mask(to_remove); interp->setRegion(region_no_boundary); - - const auto region = fmt::format("RGN_YPAR_{:+d}", offset); - if (not map_mesh.hasRegion3D(region)) { - // The valid region for this slice - map_mesh.addRegion3D( - region, Region(map_mesh.xstart, map_mesh.xend, map_mesh.ystart + offset, - map_mesh.yend + offset, 0, map_mesh.LocalNz - 1, - map_mesh.LocalNy, map_mesh.LocalNz)); - } } Field3D FCIMap::integrate(Field3D& f) const { - TRACE("FCIMap::integrate"); ASSERT1(f.getDirectionY() == YDirectionType::Standard); - ASSERT1(&map_mesh == f.getMesh()); + ASSERT1(map_mesh == f.getMesh()); // Cell centre values Field3D centre = interp->interpolate(f); @@ -269,7 +281,7 @@ Field3D FCIMap::integrate(Field3D& f) const { #endif BOUT_FOR(i, region_no_boundary) { - const auto inext = i.yp(offset); + const auto inext = i.yp(offset_); const BoutReal f_c = centre[inext]; const auto izm = i.zm(); const int x = i.x(); @@ -278,7 +290,7 @@ Field3D FCIMap::integrate(Field3D& f) const { const int zm = izm.z(); if (corner_boundary_mask(x, y, z) || corner_boundary_mask(x - 1, y, z) || corner_boundary_mask(x, y, zm) || corner_boundary_mask(x - 1, y, zm) - || (x == map_mesh.xstart)) { + || (x == map_mesh->xstart)) { // One of the corners leaves the domain. // Use the cell centre value, since boundary conditions are not // currently applied to corners. @@ -299,24 +311,74 @@ Field3D FCIMap::integrate(Field3D& f) const { return result; } +FCITransform::FCITransform(Mesh& mesh, const Coordinates::FieldMetric& dy, bool zperiodic, + Options* opt) + : ParallelTransform(mesh, opt), R{&mesh}, Z{&mesh} { + + // check the coordinate system used for the grid data source + FCITransform::checkInputGrid(); + + // Real-space coordinates of grid cells + mesh.get(R, "R", 0.0, false); + mesh.get(Z, "Z", 0.0, false); + + auto forward_boundary_xin = + std::make_shared("FCI_forward", BNDRY_PAR_FWD_XIN, +1, &mesh); + auto backward_boundary_xin = + std::make_shared("FCI_backward", BNDRY_PAR_BKWD_XIN, -1, &mesh); + auto forward_boundary_xout = + std::make_shared("FCI_forward", BNDRY_PAR_FWD_XOUT, +1, &mesh); + auto backward_boundary_xout = + std::make_shared("FCI_backward", BNDRY_PAR_BKWD_XOUT, -1, &mesh); + + // Add the boundary region to the mesh's vector of parallel boundaries + mesh.addBoundaryPar(forward_boundary_xin, BoundaryParType::xin_fwd); + mesh.addBoundaryPar(backward_boundary_xin, BoundaryParType::xin_bwd); + mesh.addBoundaryPar(forward_boundary_xout, BoundaryParType::xout_fwd); + mesh.addBoundaryPar(backward_boundary_xout, BoundaryParType::xout_bwd); + + field_line_maps.reserve(static_cast(mesh.ystart) * 2); + for (int offset = 1; offset < mesh.ystart + 1; ++offset) { + field_line_maps.emplace_back(mesh, dy, options, offset, forward_boundary_xin, + forward_boundary_xout, zperiodic); + field_line_maps.emplace_back(mesh, dy, options, -offset, backward_boundary_xin, + backward_boundary_xout, zperiodic); + } + ASSERT0(mesh.ystart == 1); + const std::array bndries = {forward_boundary_xin, forward_boundary_xout, + backward_boundary_xin, backward_boundary_xout}; + for (const auto& bndry : bndries) { + for (const auto& bndry2 : bndries) { + if (bndry->dir == bndry2->dir) { + continue; + } + for (bndry->first(); !bndry->isDone(); bndry->next()) { + if (bndry2->contains(*bndry)) { + bndry->setValid(0); + } + } + } + } +} + void FCITransform::checkInputGrid() { std::string parallel_transform; if (mesh.isDataSourceGridFile() - && !mesh.get(parallel_transform, "parallel_transform")) { + && (mesh.get(parallel_transform, "parallel_transform") == 0)) { if (parallel_transform != "fci") { throw BoutException( "Incorrect parallel transform type '" + parallel_transform + "' used " "to generate metric components for FCITransform. Should be 'fci'."); } - } // else: parallel_transform variable not found in grid input, indicates older input - // file or grid from options so must rely on the user having ensured the type is - // correct + } + // else: parallel_transform variable not found in grid input, indicates older input + // file or grid from options so must rely on the user having ensured the type is + // correct } void FCITransform::calcParallelSlices(Field3D& f) { - TRACE("FCITransform::calcParallelSlices"); - + ASSERT1(f.areCalcParallelSlicesAllowed()); ASSERT1(f.getDirectionY() == YDirectionType::Standard); // Only have forward_map/backward_map for CELL_CENTRE, so can only deal with // CELL_CENTRE inputs @@ -327,13 +389,12 @@ void FCITransform::calcParallelSlices(Field3D& f) { // Interpolate f onto yup and ydown fields for (const auto& map : field_line_maps) { - f.ynext(map.offset) = map.interpolate(f); - f.ynext(map.offset).setRegion(fmt::format("RGN_YPAR_{:+d}", map.offset)); + f.ynext(map.offset()) = map.interpolate(f); + f.ynext(map.offset()).setRegion(fmt::format("RGN_YPAR_{:+d}", map.offset())); } } void FCITransform::integrateParallelSlices(Field3D& f) { - TRACE("FCITransform::integrateParallelSlices"); ASSERT1(f.getDirectionY() == YDirectionType::Standard); // Only have forward_map/backward_map for CELL_CENTRE, so can only deal with @@ -345,7 +406,7 @@ void FCITransform::integrateParallelSlices(Field3D& f) { // Integrate f onto yup and ydown fields for (const auto& map : field_line_maps) { - f.ynext(map.offset) = map.integrate(f); + f.ynext(map.offset()) = map.integrate(f); } } diff --git a/src/mesh/parallel/fci.hxx b/src/mesh/parallel/fci.hxx index 1a02f558e1..65529a4c4e 100644 --- a/src/mesh/parallel/fci.hxx +++ b/src/mesh/parallel/fci.hxx @@ -26,6 +26,11 @@ #ifndef BOUT_FCITRANSFORM_H #define BOUT_FCITRANSFORM_H +#include "bout/assert.hxx" +#include "bout/bout_types.hxx" +#include "bout/boutexception.hxx" +#include "bout/coordinates.hxx" +#include "bout/region.hxx" #include #include #include @@ -33,25 +38,26 @@ #include #include +#include #include +class BoundaryRegionPar; +class FieldPerp; +class Field2D; +class Field3D; +class Options; + /// Field line map - contains the coefficients for interpolation class FCIMap { /// Interpolation objects std::unique_ptr interp; // Cell centre std::unique_ptr interp_corner; // Cell corner at (x+1, z+1) -public: - FCIMap() = delete; - FCIMap(Mesh& mesh, const Coordinates::FieldMetric& dy, Options& options, int offset, - const std::shared_ptr& inner_boundary, - const std::shared_ptr& outer_boundary, bool zperiodic); - // The mesh this map was created on - Mesh& map_mesh; + Mesh* map_mesh; /// Direction of map - const int offset; + int offset_; /// region containing all points where the field line has not left the /// domain @@ -59,8 +65,17 @@ public: /// If any of the integration area has left the domain BoutMask corner_boundary_mask; +public: + FCIMap() = delete; + FCIMap(Mesh& mesh, const Coordinates::FieldMetric& dy, Options& options, int offset, + const std::shared_ptr& inner_boundary, + const std::shared_ptr& outer_boundary, bool zperiodic); + + /// Direction of map + int offset() const { return offset_; } + Field3D interpolate(Field3D& f) const { - ASSERT1(&map_mesh == f.getMesh()); + ASSERT1(map_mesh == f.getMesh()); return interp->interpolate(f); } @@ -72,55 +87,7 @@ class FCITransform : public ParallelTransform { public: FCITransform() = delete; FCITransform(Mesh& mesh, const Coordinates::FieldMetric& dy, bool zperiodic = true, - Options* opt = nullptr) - : ParallelTransform(mesh, opt), R{&mesh}, Z{&mesh} { - - // check the coordinate system used for the grid data source - FCITransform::checkInputGrid(); - - // Real-space coordinates of grid cells - mesh.get(R, "R", 0.0, false); - mesh.get(Z, "Z", 0.0, false); - - auto forward_boundary_xin = - std::make_shared("FCI_forward", BNDRY_PAR_FWD_XIN, +1, &mesh); - auto backward_boundary_xin = std::make_shared( - "FCI_backward", BNDRY_PAR_BKWD_XIN, -1, &mesh); - auto forward_boundary_xout = - std::make_shared("FCI_forward", BNDRY_PAR_FWD_XOUT, +1, &mesh); - auto backward_boundary_xout = std::make_shared( - "FCI_backward", BNDRY_PAR_BKWD_XOUT, -1, &mesh); - - // Add the boundary region to the mesh's vector of parallel boundaries - mesh.addBoundaryPar(forward_boundary_xin, BoundaryParType::xin_fwd); - mesh.addBoundaryPar(backward_boundary_xin, BoundaryParType::xin_bwd); - mesh.addBoundaryPar(forward_boundary_xout, BoundaryParType::xout_fwd); - mesh.addBoundaryPar(backward_boundary_xout, BoundaryParType::xout_bwd); - - field_line_maps.reserve(mesh.ystart * 2); - for (int offset = 1; offset < mesh.ystart + 1; ++offset) { - field_line_maps.emplace_back(mesh, dy, options, offset, forward_boundary_xin, - forward_boundary_xout, zperiodic); - field_line_maps.emplace_back(mesh, dy, options, -offset, backward_boundary_xin, - backward_boundary_xout, zperiodic); - } - ASSERT0(mesh.ystart == 1); - std::shared_ptr bndries[]{ - forward_boundary_xin, forward_boundary_xout, backward_boundary_xin, - backward_boundary_xout}; - for (auto& bndry : bndries) { - for (const auto& bndry2 : bndries) { - if (bndry->dir == bndry2->dir) { - continue; - } - for (bndry->first(); !bndry->isDone(); bndry->next()) { - if (bndry2->contains(*bndry)) { - bndry->setValid(0); - } - } - } - } - } + Options* opt = nullptr); void calcParallelSlices(Field3D& f) override; diff --git a/src/mesh/parallel/fci_comm.cxx b/src/mesh/parallel/fci_comm.cxx new file mode 100644 index 0000000000..3d213ad6e4 --- /dev/null +++ b/src/mesh/parallel/fci_comm.cxx @@ -0,0 +1,224 @@ +/************************************************************************** + * Communication for Flux-coordinate Independent interpolation + * + ************************************************************************** + * Copyright 2025 BOUT++ contributors + * + * Contact: Ben Dudson, dudson2@llnl.gov + * + * This file is part of BOUT++. + * + * BOUT++ is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * BOUT++ is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with BOUT++. If not, see . + * + **************************************************************************/ + +#include "fci_comm.hxx" +#include "bout/assert.hxx" +#include "bout/bout_types.hxx" +#include "bout/field3d.hxx" +#include "bout/region.hxx" + +#include +#include +#include +#include + +fci_comm::ProcLocal fci_comm::GlobalToLocal1D::convert(int id) const { + if (periodic) { + while (id < mg) { + id += global; + } + while (id >= global + mg) { + id -= global; + } + } + const int idwo = id - mg; + int proc = idwo / local; + if (not periodic) { + if (proc >= npe) { + proc = npe - 1; + } + } + const int loc = id - (local * proc); +#if CHECK > 1 + ASSERT1(loc >= 0); + ASSERT1(loc <= localwith); + ASSERT1(proc >= 0); + ASSERT1(proc < npe); + if (periodic and (loc < mg or loc >= local + mg)) { + throw BoutException( + "GlobalToLocal1D failure - expected {} < {} < {} because we are periodic\n", mg, + loc, local + mg); + } +#endif + return {proc, loc}; +} + +void GlobalField3DAccess::setup() { + // We need to send a list of data to every processor of which data + // we want We also get a list of all the data that every other + // processor wants Some of theses lists may be empty. We still need + // to send them, later we can skip that. We also compute where the + // requested data will be stored later on. This is currently + // implemented as a map, to memory efficient, as the data is + // sparse. We could also store the mapping as a dense array, if it + // turns out this lookup is not fast enough, but that may limit + // scaling at some point. + ASSERT2(is_setup == false); +#ifdef _OPENMP + for (auto& o_id : openmp_ids) { + ids.merge(o_id); + } + openmp_ids.clear(); +#endif + toGet.resize(static_cast(global2local_x.getNPE() * global2local_y.getNPE() + * global2local_z.getNPE())); + for (const auto id : ids) { + const IndG3D gind{id, global2local_y.getGlobalWith(), global2local_z.getGlobalWith()}; + const auto pix = global2local_x.convert(gind.x()); + const auto piy = global2local_y.convert(gind.y()); + const auto piz = global2local_z.convert(gind.z()); + ASSERT3(piz.proc == 0); + toGet[mesh->getProcIndex(pix.proc, piy.proc, piz.proc)].push_back( + xyzlocal.convert(pix.index, piy.index, piz.index).ind); + } + for (auto& v : toGet) { + std::sort(v.begin(), v.end()); + } + commCommLists(); + { + int offset = 0; + for (const auto& get : toGet) { + getOffsets.push_back(offset); + offset += get.size(); + } + getOffsets.push_back(offset); + } + for (const auto id : ids) { + const IndG3D gind{id, global2local_y.getGlobalWith(), global2local_z.getGlobalWith()}; + const auto pix = global2local_x.convert(gind.x()); + const auto piy = global2local_y.convert(gind.y()); + const auto piz = global2local_z.convert(gind.z()); + ASSERT3(piz.proc == 0); + const auto proc = mesh->getProcIndex(pix.proc, piy.proc, piz.proc); + const auto& vec = toGet[proc]; + const auto tofind = xyzlocal.convert(pix.index, piy.index, piz.index).ind; + auto it = std::lower_bound(vec.begin(), vec.end(), tofind); + ASSERT3(it != vec.end()); + ASSERT3(*it == tofind); + mapping[id] = std::distance(vec.begin(), it) + getOffsets[proc]; + } + is_setup = true; +} + +void GlobalField3DAccess::commCommLists() { + toSend.resize(toGet.size()); + std::vector toGetSizes(toGet.size(), -1); + std::vector toSendSizes(toSend.size(), -1); +#if CHECK > 3 + { + int thisproc; + MPI_Comm_rank(comm, &thisproc); + ASSERT0(thisproc + == mesh->getProcIndex(mesh->getXProcIndex(), mesh->getYProcIndex(), + mesh->getZProcIndex())); + } +#endif + std::vector reqs(toSend.size()); + for (size_t proc = 0; proc < toGet.size(); ++proc) { + auto ret = MPI_Irecv(&toSendSizes[proc], 1, MPI_INT, proc, 666, comm, &reqs[proc]); + ASSERT0(ret == MPI_SUCCESS); + } + for (size_t proc = 0; proc < toGet.size(); ++proc) { + toGetSizes[proc] = toGet[proc].size(); + auto ret = MPI_Send(&toGetSizes[proc], 1, MPI_INT, proc, 666, comm); + ASSERT0(ret == MPI_SUCCESS); + } + std::vector reqs2(toSend.size()); + int cnt = 0; + for ([[maybe_unused]] auto dummy : reqs) { + int ind{0}; + auto ret = MPI_Waitany(reqs.size(), reqs.data(), &ind, MPI_STATUS_IGNORE); + ASSERT0(ret == MPI_SUCCESS); + ASSERT3(ind != MPI_UNDEFINED); + ASSERT2(static_cast(ind) < toSend.size()); + ASSERT3(toSendSizes[ind] >= 0); + if (toSendSizes[ind] == 0) { + continue; + } + sendBufferSize += toSendSizes[ind]; + toSend[ind].resize(toSendSizes[ind], -1); + + ret = MPI_Irecv(toSend[ind].data(), toSend[ind].size(), MPI_INT, ind, 666 * 666, comm, + reqs2.data() + cnt++); + ASSERT0(ret == MPI_SUCCESS); + } + for (size_t proc = 0; proc < toGet.size(); ++proc) { + if (!toGet[proc].empty()) { + const auto ret = MPI_Send(toGet[proc].data(), toGet[proc].size(), MPI_INT, proc, + 666 * 666, comm); + ASSERT0(ret == MPI_SUCCESS); + } + } + for (int c = 0; c < cnt; c++) { + int ind{0}; + const auto ret = MPI_Waitany(cnt, reqs2.data(), &ind, MPI_STATUS_IGNORE); + ASSERT0(ret == MPI_SUCCESS); + ASSERT3(ind != MPI_UNDEFINED); + } +} + +std::vector GlobalField3DAccess::communicate_data(const Field3D& f) { + // Ensure setup is called, to setup communication pattern + if (not is_setup) { + setup(); + } + // We now send the previosly requested data to every processor, that wanted some. + // We also get the data we requested. + ASSERT2(f.getMesh() == mesh); + std::vector data(getOffsets.back()); + std::vector sendBuffer(sendBufferSize); + std::vector reqs(toSend.size()); + int cnt1 = 0; + for (size_t proc = 0; proc < toGet.size(); ++proc) { + if (toGet[proc].empty()) { + continue; + } + auto ret = MPI_Irecv(data.data() + getOffsets[proc], toGet[proc].size(), MPI_DOUBLE, + proc, 666, comm, reqs.data() + cnt1); + ASSERT0(ret == MPI_SUCCESS); + cnt1++; + } + int cnt = 0; + for (size_t proc = 0; proc < toGet.size(); ++proc) { + if (toSend[proc].empty()) { + continue; + } + const void* start = sendBuffer.data() + cnt; + for (auto i : toSend[proc]) { + sendBuffer[cnt++] = f[Ind3D(i)]; + } + auto ret = MPI_Send(start, toSend[proc].size(), MPI_DOUBLE, proc, 666, comm); + ASSERT0(ret == MPI_SUCCESS); + } + for (int j = 0; j < cnt1; ++j) { + int ind{0}; + auto ret = MPI_Waitany(cnt1, reqs.data(), &ind, MPI_STATUS_IGNORE); + ASSERT0(ret == MPI_SUCCESS); + ASSERT3(ind != MPI_UNDEFINED); + ASSERT3(ind >= 0); + ASSERT3(ind < cnt1); + } + return data; +} diff --git a/src/mesh/parallel/fci_comm.hxx b/src/mesh/parallel/fci_comm.hxx new file mode 100644 index 0000000000..34512b18c4 --- /dev/null +++ b/src/mesh/parallel/fci_comm.hxx @@ -0,0 +1,187 @@ +/************************************************************************** + * Communication for Flux-coordinate Independent interpolation + * + ************************************************************************** + * Copyright 2025 BOUT++ contributors + * + * Contact: Ben Dudson, dudson2@llnl.gov + * + * This file is part of BOUT++. + * + * BOUT++ is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * BOUT++ is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with BOUT++. If not, see . + * + **************************************************************************/ + +#pragma once + +#include "bout/assert.hxx" +#include "bout/bout_types.hxx" +#include "bout/boutcomm.hxx" +#include "bout/field3d.hxx" +#include "bout/mesh.hxx" +#include "bout/region.hxx" +#include +#include +#include +#include +#include +#include + +/// GlobalField3DAccess is a class to set up the communication +/// patterns, to request abitrary data form the global +/// field. GlobalField3DAccessInstance is an instance. after +/// GlobalField3DAccess has communicated the pre-requested data. Only +/// data that has been pre-requested can be requested from +/// GlobalField3DAccessInstance after communication. +/// +/// The usage looks a bit like this: +/// +/// GlobalField3DAccess gfa; +/// // Request an abitrary number of global indices ``gi``: +/// gfa.request(gi) +/// // Communicate data +/// const auto data = gfa.communicate(f3d); +/// // Now data can be accesssed for all previously requested ``gi``s: +/// data[gi] +class GlobalField3DAccess; + +namespace fci_comm { +struct ProcLocal { + int proc; + int index; +}; + +/// Class to convert global to local indices for 1D +/// given the global index, it returns the local index and the processor. +struct GlobalToLocal1D { + GlobalToLocal1D(int mg, int npe, int localwith, bool periodic) + : mg(mg), npe(npe), localwith(localwith), local(localwith - (2 * mg)), + global(local * npe), globalwith(global + (2 * mg)), periodic(periodic) {}; + ProcLocal convert(int id) const; + int getLocalWith() const { return localwith; } + int getGlobalWith() const { return globalwith; } + int getNPE() const { return npe; } + +private: + int mg; + int npe; + int localwith; + int local; + int global; + int globalwith; + bool periodic; +}; + +/// Convert an x-y-z tupple to an Ind +template +struct XYZ2Ind { + XYZ2Ind(const int nx, const int ny, const int nz) : nx(nx), ny(ny), nz(nz) {} + ind convert(const int x, const int y, const int z) const { + return {z + ((y + x * ny) * nz), ny, nz}; + } + ind operator()(const int x, const int y, const int z) const { return convert(x, y, z); } + +private: + int nx; + int ny; + int nz; +}; +} // namespace fci_comm + +class GlobalField3DAccessInstance { +public: + const BoutReal& operator[](IndG3D ind) const; + GlobalField3DAccessInstance(const GlobalField3DAccess* gfa, + std::vector&& data) + : gfa(gfa), data(std::move(data)) {}; + +private: + const GlobalField3DAccess* gfa; + std::vector data; +}; + +class GlobalField3DAccess { +public: + friend class GlobalField3DAccessInstance; + GlobalField3DAccess(Mesh* mesh) + : mesh(mesh), + global2local_x(mesh->xstart, mesh->getNXPE(), mesh->LocalNx, mesh->periodicX), + global2local_y(mesh->ystart, mesh->getNYPE(), mesh->LocalNy, true), + global2local_z(mesh->zstart, mesh->getNZPE(), mesh->LocalNz, true), + xyzlocal(global2local_x.getLocalWith(), global2local_y.getLocalWith(), + global2local_z.getLocalWith()), + xyzglobal(global2local_x.getGlobalWith(), global2local_y.getGlobalWith(), + global2local_z.getGlobalWith()), + comm(BoutComm::get()) { +#ifdef _OPENMP + openmp_ids.resize(omp_get_max_threads()); +#endif +#if CHECK >= 2 + // We could also allow false, but then we would need to ensure it + // is false everywhere. + for (int x = 0; x < mesh->LocalNx; ++x) { + ASSERT2(mesh->periodicY(x) == true); + } +#endif + }; + void request(IndG3D ind) { + ASSERT2(is_setup == false); +#ifdef _OPENMP + ASSERT2(openmp_ids.size() > static_cast(omp_get_thread_num())); + openmp_ids[omp_get_thread_num()].emplace(ind.ind); +#else + ids.emplace(ind.ind); +#endif + } + + GlobalField3DAccessInstance communicate(const Field3D& f) { + return {this, communicate_data(f)}; + } + std::unique_ptr communicate_asPtr(const Field3D& f) { + return std::make_unique(this, communicate_data(f)); + } + +private: + void setup(); + void commCommLists(); + Mesh* mesh; +#ifdef _OPENMP + // openmp thread-local variable + std::vector> openmp_ids; +#endif + std::set ids; + std::map mapping; + bool is_setup{false}; + fci_comm::GlobalToLocal1D global2local_x; + fci_comm::GlobalToLocal1D global2local_y; + fci_comm::GlobalToLocal1D global2local_z; + +public: + fci_comm::XYZ2Ind xyzlocal; + fci_comm::XYZ2Ind xyzglobal; + +private: + std::vector> toGet; + std::vector> toSend; + std::vector getOffsets; + int sendBufferSize{0}; + MPI_Comm comm; + std::vector communicate_data(const Field3D& f); +}; + +inline const BoutReal& GlobalField3DAccessInstance::operator[](IndG3D ind) const { + auto it = gfa->mapping.find(ind.ind); + ASSERT2(it != gfa->mapping.end()); + return data[it->second]; +} diff --git a/src/mesh/parallel/shiftedmetric.cxx b/src/mesh/parallel/shiftedmetric.cxx index 382052047d..f167824541 100644 --- a/src/mesh/parallel/shiftedmetric.cxx +++ b/src/mesh/parallel/shiftedmetric.cxx @@ -24,6 +24,7 @@ ShiftedMetric::ShiftedMetric(Mesh& m, CELL_LOC location_in, Field2D zShift_, ASSERT1(zShift.getLocation() == location); // check the coordinate system used for the grid data source ShiftedMetric::checkInputGrid(); + bout::fft::assertZSerial(m, "ShiftedMetric"); cachePhases(); } @@ -38,8 +39,8 @@ void ShiftedMetric::checkInputGrid() { "Should be 'shiftedmetric'."); } } // else: parallel_transform variable not found in grid input, indicates older input - // file or grid from options so must rely on the user having ensured the type is - // correct + // file or grid from options so must rely on the user having ensured the type is + // correct } void ShiftedMetric::outputVars(Options& output_options) { diff --git a/src/mesh/parallel/shiftedmetricinterp.cxx b/src/mesh/parallel/shiftedmetricinterp.cxx index 7f3637e79c..3e187f29bf 100644 --- a/src/mesh/parallel/shiftedmetricinterp.cxx +++ b/src/mesh/parallel/shiftedmetricinterp.cxx @@ -27,8 +27,13 @@ * **************************************************************************/ +#include + #include "shiftedmetricinterp.hxx" + +#include "bout/boutexception.hxx" #include "bout/constants.hxx" +#include "bout/field3d.hxx" #include "bout/parallel_boundary_region.hxx" ShiftedMetricInterp::ShiftedMetricInterp(Mesh& mesh, CELL_LOC location_in, @@ -36,12 +41,19 @@ ShiftedMetricInterp::ShiftedMetricInterp(Mesh& mesh, CELL_LOC location_in, Options* opt) : ParallelTransform(mesh, opt), location(location_in), zShift(std::move(zShift_in)), zlength(zlength_in), ydown_index(mesh.ystart) { + + if (mesh.getNZPE() > 1) { + throw BoutException("ShiftedMetricInterp only works with 1 processor in Z"); + } + // check the coordinate system used for the grid data source ShiftedMetricInterp::checkInputGrid(); // Allocate space for interpolator cache: y-guard cells in each direction parallel_slice_interpolators.resize(mesh.ystart * 2); + const BoutReal z_factor = static_cast(mesh.GlobalNzNoBoundaries) / zlength; + // Create the Interpolation objects and set whether they go up or down the // magnetic field auto& interp_options = options["zinterpolation"]; @@ -60,16 +72,16 @@ ShiftedMetricInterp::ShiftedMetricInterp(Mesh& mesh, CELL_LOC location_in, // Find the index positions where the magnetic field line intersects the x-z plane // y_offset points up - Field3D zt_prime_up(&mesh), zt_prime_down(&mesh); + Field3D zt_prime_up(&mesh); + Field3D zt_prime_down(&mesh); zt_prime_up.allocate(); zt_prime_down.allocate(); for (const auto& i : zt_prime_up.getRegion(RGN_NOY)) { // Field line moves in z by an angle zShift(i,j+1)-zShift(i,j) when going // from j to j+1, but we want the shift in index-space - zt_prime_up[i] = static_cast(i.z()) - + (zShift[i.yp(y_offset + 1)] - zShift[i]) - * static_cast(mesh.GlobalNz) / zlength; + zt_prime_up[i] = static_cast(mesh.getGlobalZIndexNoBoundaries(i.z())) + + ((zShift[i.yp(y_offset + 1)] - zShift[i]) * z_factor); } parallel_slice_interpolators[yup_index + y_offset]->calcWeights(zt_prime_up); @@ -77,9 +89,8 @@ ShiftedMetricInterp::ShiftedMetricInterp(Mesh& mesh, CELL_LOC location_in, for (const auto& i : zt_prime_down.getRegion(RGN_NOY)) { // Field line moves in z by an angle -(zShift(i,j)-zShift(i,j-1)) when going // from j to j-1, but we want the shift in index-space - zt_prime_down[i] = static_cast(i.z()) - - (zShift[i] - zShift[i.ym(y_offset + 1)]) - * static_cast(mesh.GlobalNz) / zlength; + zt_prime_down[i] = static_cast(mesh.getGlobalZIndexNoBoundaries(i.z())) + - ((zShift[i] - zShift[i.ym(y_offset + 1)]) * z_factor); } parallel_slice_interpolators[ydown_index + y_offset]->calcWeights(zt_prime_down); @@ -91,15 +102,16 @@ ShiftedMetricInterp::ShiftedMetricInterp(Mesh& mesh, CELL_LOC location_in, interp_from_aligned = ZInterpolationFactory::getInstance().create(&interp_options, 0, &mesh); - Field3D zt_prime_to(&mesh), zt_prime_from(&mesh); + Field3D zt_prime_to(&mesh); + Field3D zt_prime_from(&mesh); zt_prime_to.allocate(); zt_prime_from.allocate(); for (const auto& i : zt_prime_to) { // Field line moves in z by an angle zShift(i,j) when going // from y0 to y(j), but we want the shift in index-space - zt_prime_to[i] = static_cast(i.z()) - + zShift[i] * static_cast(mesh.GlobalNz) / zlength; + zt_prime_to[i] = static_cast(mesh.getGlobalZIndexNoBoundaries(i.z())) + + (zShift[i] * z_factor); } interp_to_aligned->calcWeights(zt_prime_to); @@ -108,17 +120,16 @@ ShiftedMetricInterp::ShiftedMetricInterp(Mesh& mesh, CELL_LOC location_in, // Field line moves in z by an angle zShift(i,j) when going // from y0 to y(j), but we want the shift in index-space. // Here we reverse the shift, so subtract zShift - zt_prime_from[i] = static_cast(i.z()) - - zShift[i] * static_cast(mesh.GlobalNz) / zlength; + zt_prime_from[i] = static_cast(mesh.getGlobalZIndexNoBoundaries(i.z())) + - (zShift[i] * z_factor); } interp_from_aligned->calcWeights(zt_prime_from); - int yvalid = mesh.LocalNy - 2 * mesh.ystart; - // avoid overflow - no stencil need more than 5 points - if (yvalid > 20) { - yvalid = 20; - } + // avoid overflow - no stencil needs more than 5 points + const auto yvalid = + static_cast(std::min(mesh.LocalNy - (2 * mesh.ystart), 20)); + // Create regions for parallel boundary conditions Field2D dy; mesh.get(dy, "dy", 1.); @@ -128,10 +139,10 @@ ShiftedMetricInterp::ShiftedMetricInterp(Mesh& mesh, CELL_LOC location_in, for (int z = mesh.zstart; z <= mesh.zend; z++) { forward_boundary_xin->add_point( it.ind, mesh.yend, z, - mesh.GlobalX(it.ind), // x - 2. * PI * mesh.GlobalY(mesh.yend + 0.5), // y - zlength * BoutReal(z) / BoutReal(mesh.GlobalNz) // z - + 0.5 * (zShift(it.ind, mesh.yend + 1) - zShift(it.ind, mesh.yend)), + mesh.GlobalX(it.ind), // x + 2. * PI * mesh.GlobalY(mesh.yend + 0.5), // y + (zlength * mesh.GlobalZ(z)) // z + + (0.5 * (zShift(it.ind, mesh.yend + 1) - zShift(it.ind, mesh.yend))), 0.25 * (1 // dy/2 + dy(it.ind, mesh.yend + 1) / dy(it.ind, mesh.yend)), // length @@ -144,10 +155,10 @@ ShiftedMetricInterp::ShiftedMetricInterp(Mesh& mesh, CELL_LOC location_in, for (int z = mesh.zstart; z <= mesh.zend; z++) { backward_boundary_xin->add_point( it.ind, mesh.ystart, z, - mesh.GlobalX(it.ind), // x - 2. * PI * mesh.GlobalY(mesh.ystart - 0.5), // y - zlength * BoutReal(z) / BoutReal(mesh.GlobalNz) // z - + 0.5 * (zShift(it.ind, mesh.ystart) - zShift(it.ind, mesh.ystart - 1)), + mesh.GlobalX(it.ind), // x + 2. * PI * mesh.GlobalY(mesh.ystart - 0.5), // y + (zlength * mesh.GlobalZ(z)) // z + + (0.5 * (zShift(it.ind, mesh.ystart) - zShift(it.ind, mesh.ystart - 1))), 0.25 * (1 // dy/2 + dy(it.ind, mesh.ystart - 1) / dy(it.ind, mesh.ystart)), @@ -161,10 +172,10 @@ ShiftedMetricInterp::ShiftedMetricInterp(Mesh& mesh, CELL_LOC location_in, for (int z = mesh.zstart; z <= mesh.zend; z++) { forward_boundary_xout->add_point( it.ind, mesh.yend, z, - mesh.GlobalX(it.ind), // x - 2. * PI * mesh.GlobalY(mesh.yend + 0.5), // y - zlength * BoutReal(z) / BoutReal(mesh.GlobalNz) // z - + 0.5 * (zShift(it.ind, mesh.yend + 1) - zShift(it.ind, mesh.yend)), + mesh.GlobalX(it.ind), // x + 2. * PI * mesh.GlobalY(mesh.yend + 0.5), // y + (zlength * mesh.GlobalZ(z)) // z + + (0.5 * (zShift(it.ind, mesh.yend + 1) - zShift(it.ind, mesh.yend))), 0.25 * (1 // dy/2 + dy(it.ind, mesh.yend + 1) / dy(it.ind, mesh.yend)), @@ -177,10 +188,10 @@ ShiftedMetricInterp::ShiftedMetricInterp(Mesh& mesh, CELL_LOC location_in, for (int z = mesh.zstart; z <= mesh.zend; z++) { backward_boundary_xout->add_point( it.ind, mesh.ystart, z, - mesh.GlobalX(it.ind), // x - 2. * PI * mesh.GlobalY(mesh.ystart - 0.5), // y - zlength * BoutReal(z) / BoutReal(mesh.GlobalNz) // z - + 0.5 * (zShift(it.ind, mesh.ystart) - zShift(it.ind, mesh.ystart - 1)), + mesh.GlobalX(it.ind), // x + 2. * PI * mesh.GlobalY(mesh.ystart - 0.5), // y + (zlength * mesh.GlobalZ(z)) // z + + (0.5 * (zShift(it.ind, mesh.ystart) - zShift(it.ind, mesh.ystart - 1))), 0.25 * (dy(it.ind, mesh.ystart - 1) / dy(it.ind, mesh.ystart) // dy/2 + 1), @@ -204,14 +215,13 @@ void ShiftedMetricInterp::checkInputGrid() { "Should be 'orthogonal'."); } } // else: coordinate_system variable not found in grid input, indicates older input - // file so must rely on the user having ensured the type is correct + // file so must rely on the user having ensured the type is correct } /*! * Calculate the Y up and down fields */ void ShiftedMetricInterp::calcParallelSlices(Field3D& f) { - AUTO_TRACE(); // Ensure that yup and ydown are different fields f.splitParallelSlices(); diff --git a/src/physics/physicsmodel.cxx b/src/physics/physicsmodel.cxx index f4be3bde2a..6cdd78e351 100644 --- a/src/physics/physicsmodel.cxx +++ b/src/physics/physicsmodel.cxx @@ -34,12 +34,14 @@ #include "bout/version.hxx" #include +#include #include #include #include #include +#include #include using namespace std::literals; @@ -67,9 +69,10 @@ PhysicsModel::PhysicsModel() .withDefault(true)), restart_enabled(Options::root()["restart_files"]["enabled"] .doc("Write restart files") - .withDefault(true)) - -{ + .withDefault(true)), + flush_frequency(Options::root()["output"]["flush_frequency"] + .doc("How often to flush to disk") + .withDefault(1)) { if (output_enabled) { output_file = bout::OptionsIOFactory::getInstance().createOutput(); } @@ -164,7 +167,6 @@ void PhysicsModel::bout_solve(Vector3D& var, const char* name, } int PhysicsModel::postInit(bool restarting) { - TRACE("PhysicsModel::postInit"); if (restarting) { solver->readEvolvingVariablesFromOptions(restart_options); @@ -189,7 +191,7 @@ int PhysicsModel::postInit(bool restarting) { } void PhysicsModel::outputVars(Options& options) { - Timer time("io"); + const Timer time("io"); for (const auto& item : dump.getData()) { bout::utils::visit(bout::OptionsConversionVisitor{options, item.name}, item.value); if (item.repeat) { @@ -199,7 +201,7 @@ void PhysicsModel::outputVars(Options& options) { } void PhysicsModel::restartVars(Options& options) { - Timer time("io"); + const Timer time("io"); for (const auto& item : restart.getData()) { bout::utils::visit(bout::OptionsConversionVisitor{options, item.name}, item.value); if (item.repeat) { @@ -217,9 +219,7 @@ void PhysicsModel::writeRestartFile() { void PhysicsModel::writeOutputFile() { writeOutputFile(output_options); } void PhysicsModel::writeOutputFile(const Options& options) { - if (output_enabled) { - output_file->write(options, "t"); - } + writeOutputFile(options, "t"); } void PhysicsModel::writeOutputFile(const Options& options, @@ -230,13 +230,19 @@ void PhysicsModel::writeOutputFile(const Options& options, } void PhysicsModel::finishOutputTimestep() const { - if (output_enabled) { + const Timer timer("io"); + + if (output_enabled and (flush_counter % flush_frequency == 0)) { + output_file->flush(); output_file->verifyTimesteps(); } } int PhysicsModel::PhysicsModelMonitor::call(Solver* solver, BoutReal simtime, int iteration, int nout) { + + model->setFlushCounter(static_cast(iteration)); + // Restart file variables solver->outputVars(model->restart_options, false); model->restartVars(model->restart_options); @@ -255,5 +261,8 @@ int PhysicsModel::PhysicsModelMonitor::call(Solver* solver, BoutReal simtime, model->outputVars(model->output_options); model->writeOutputFile(); + // Reset output options, this avoids rewriting time-independent data + model->output_options = Options{}; + return monitor_result; } diff --git a/src/physics/smoothing.cxx b/src/physics/smoothing.cxx index 0a1391907f..1b437b4352 100644 --- a/src/physics/smoothing.cxx +++ b/src/physics/smoothing.cxx @@ -37,7 +37,6 @@ #include #include #include -#include #include #include @@ -46,7 +45,6 @@ // Smooth using simple 1-2-1 filter const Field3D smooth_x(const Field3D& f) { - TRACE("smooth_x"); Mesh* mesh = f.getMesh(); Field3D result{emptyFrom(f)}; @@ -76,7 +74,6 @@ const Field3D smooth_x(const Field3D& f) { } const Field3D smooth_y(const Field3D& f) { - TRACE("smooth_y"); Mesh* mesh = f.getMesh(); Field3D result{emptyFrom(f)}; @@ -117,7 +114,6 @@ const Field3D smooth_y(const Field3D& f) { so no processor/branch cuts in X */ const Field2D averageX(const Field2D& f) { - TRACE("averageX(Field2D)"); Mesh* mesh = f.getMesh(); int ngx = mesh->LocalNx; @@ -176,8 +172,6 @@ const Field2D averageX(const Field2D& f) { */ const Field3D averageX(const Field3D& f) { - TRACE("averageX(Field3D)"); - Mesh* mesh = f.getMesh(); int ngx = mesh->LocalNx; @@ -230,8 +224,6 @@ const Field3D averageX(const Field3D& f) { } const Field2D averageY(const Field2D& f) { - TRACE("averageY(Field2D)"); - Mesh* mesh = f.getMesh(); int ngx = mesh->LocalNx; int ngy = mesh->LocalNy; @@ -275,8 +267,6 @@ const Field2D averageY(const Field2D& f) { } const Field3D averageY(const Field3D& f) { - TRACE("averageY(Field3D)"); - Mesh* mesh = f.getMesh(); int ngx = mesh->LocalNx; @@ -350,9 +340,9 @@ BoutReal Average_XY(const Field2D& var) { return Vol_Glb; } -BoutReal Vol_Integral(const Field2D& var) { +BoutReal Vol_Integral([[maybe_unused]] const Field2D& var) { #if BOUT_USE_METRIC_3D - AUTO_TRACE(); + throw BoutException("Vol_Intregral currently incompatible with 3D metrics"); #else Mesh* mesh = var.getMesh(); @@ -431,7 +421,7 @@ void nl_filter(rvec& f, BoutReal w) { } const Field3D nl_filter_x(const Field3D& f, BoutReal w) { - TRACE("nl_filter_x( Field3D )"); + Mesh* mesh = f.getMesh(); Field3D result{emptyFrom(f)}; @@ -453,7 +443,6 @@ const Field3D nl_filter_x(const Field3D& f, BoutReal w) { } const Field3D nl_filter_y(const Field3D& f, BoutReal w) { - TRACE("nl_filter_x( Field3D )"); Mesh* mesh = f.getMesh(); @@ -481,7 +470,6 @@ const Field3D nl_filter_y(const Field3D& f, BoutReal w) { } const Field3D nl_filter_z(const Field3D& fs, BoutReal w) { - TRACE("nl_filter_z( Field3D )"); Mesh* mesh = fs.getMesh(); Field3D result{emptyFrom(fs)}; diff --git a/src/physics/sourcex.cxx b/src/physics/sourcex.cxx index 1cfc2fdc6e..659abbf409 100644 --- a/src/physics/sourcex.cxx +++ b/src/physics/sourcex.cxx @@ -2,15 +2,13 @@ * radial source and mask operators **************************************************************/ -#include -#include - #include +#include #include -#include #include +#include -#include "bout/unused.hxx" +#include BoutReal TanH(BoutReal a) { BoutReal temp = exp(a); @@ -77,7 +75,6 @@ const Field3D sink_tanhx(const Field2D& UNUSED(f0), const Field3D& f, BoutReal s // create radial buffer zones to set jpar zero near radial boundaries const Field3D mask_x(const Field3D& f, bool UNUSED(BoutRealspace)) { - TRACE("mask_x"); Mesh* localmesh = f.getMesh(); @@ -101,7 +98,6 @@ const Field3D mask_x(const Field3D& f, bool UNUSED(BoutRealspace)) { // create radial buffer zones to set jpar zero near radial boundaries const Field3D sink_tanhxl(const Field2D& UNUSED(f0), const Field3D& f, BoutReal swidth, BoutReal slength, bool UNUSED(BoutRealspace)) { - TRACE("sink_tanhx"); Mesh* localmesh = f.getMesh(); @@ -123,7 +119,6 @@ const Field3D sink_tanhxl(const Field2D& UNUSED(f0), const Field3D& f, BoutReal // create radial buffer zones to set jpar zero near radial boundaries const Field3D sink_tanhxr(const Field2D& UNUSED(f0), const Field3D& f, BoutReal swidth, BoutReal slength, bool UNUSED(BoutRealspace)) { - TRACE("sink_tanhxr"); Mesh* localmesh = f.getMesh(); @@ -144,7 +139,6 @@ const Field3D sink_tanhxr(const Field2D& UNUSED(f0), const Field3D& f, BoutReal // create radial buffer zones to damp Psi to zero near radial boundaries const Field3D buff_x(const Field3D& f, bool UNUSED(BoutRealspace)) { - TRACE("buff_x"); Mesh* localmesh = f.getMesh(); diff --git a/src/solver/impls/adams_bashforth/adams_bashforth.cxx b/src/solver/impls/adams_bashforth/adams_bashforth.cxx index 79161fcdbf..51fff4fe6a 100644 --- a/src/solver/impls/adams_bashforth/adams_bashforth.cxx +++ b/src/solver/impls/adams_bashforth/adams_bashforth.cxx @@ -4,17 +4,11 @@ #include #include -#include -#include - #include -#include - namespace { BoutReal lagrange_at_position_denominator(const std::deque& grid, const int position, const int order) { - AUTO_TRACE(); const auto xj = grid[position]; @@ -28,7 +22,7 @@ BoutReal lagrange_at_position_denominator(const std::deque& grid, BoutReal lagrange_at_position_numerator(const BoutReal varX, const std::deque& grid, const int position, const int order) { - AUTO_TRACE(); + BoutReal result = 1.0; for (int i = 0; i < order; i++) { result *= (i != position) ? (varX - grid[i]) : 1.0; @@ -55,7 +49,7 @@ BoutReal lagrange_interpolate(BoutReal start, BoutReal end, BoutReal integrate_lagrange_curve_nc9(const BoutReal start, const BoutReal end, const std::deque& points, const int position) { - AUTO_TRACE(); + constexpr std::size_t size = 9; constexpr BoutReal fac = 4.0 / 14175.0; constexpr std::array facs{989.0 * fac, 5888.0 * fac, -928.0 * fac, @@ -68,7 +62,7 @@ BoutReal integrate_lagrange_curve_nc9(const BoutReal start, const BoutReal end, BoutReal integrate_lagrange_curve_nc8(const BoutReal start, const BoutReal end, const std::deque& points, const int position) { - AUTO_TRACE(); + constexpr std::size_t size = 8; constexpr BoutReal fac = 7.0 / 17280.0; constexpr std::array facs{751.0 * fac, 3577.0 * fac, 1323.0 * fac, @@ -81,7 +75,7 @@ BoutReal integrate_lagrange_curve_nc8(const BoutReal start, const BoutReal end, BoutReal integrate_lagrange_curve_nc7(const BoutReal start, const BoutReal end, const std::deque& points, const int position) { - AUTO_TRACE(); + constexpr std::size_t size = 7; constexpr BoutReal fac = 1.0 / 140.0; constexpr std::array facs{41.0 * fac, 216.0 * fac, 27.0 * fac, @@ -94,7 +88,7 @@ BoutReal integrate_lagrange_curve_nc7(const BoutReal start, const BoutReal end, BoutReal integrate_lagrange_curve_nc6(const BoutReal start, const BoutReal end, const std::deque& points, const int position) { - AUTO_TRACE(); + constexpr std::size_t size = 6; constexpr BoutReal fac = 5.0 / 288.0; constexpr std::array facs{19.0 * fac, 75.0 * fac, 50.0 * fac, @@ -106,7 +100,7 @@ BoutReal integrate_lagrange_curve_nc6(const BoutReal start, const BoutReal end, BoutReal integrate_lagrange_curve_nc5(const BoutReal start, const BoutReal end, const std::deque& points, const int position) { - AUTO_TRACE(); + constexpr std::size_t size = 5; constexpr BoutReal fac = 2.0 / 45.0; constexpr std::array facs{7.0 * fac, 32.0 * fac, 12.0 * fac, 32.0 * fac, @@ -118,7 +112,7 @@ BoutReal integrate_lagrange_curve_nc5(const BoutReal start, const BoutReal end, BoutReal integrate_lagrange_curve_nc4(const BoutReal start, const BoutReal end, const std::deque& points, const int position) { - AUTO_TRACE(); + constexpr std::size_t size = 4; constexpr BoutReal fac = 3.0 / 8.0; constexpr std::array facs{1.0 * fac, 3.0 * fac, 3.0 * fac, 1.0 * fac}; @@ -129,7 +123,7 @@ BoutReal integrate_lagrange_curve_nc4(const BoutReal start, const BoutReal end, BoutReal integrate_lagrange_curve_nc3(const BoutReal start, const BoutReal end, const std::deque& points, const int position) { - AUTO_TRACE(); + constexpr std::size_t size = 3; constexpr BoutReal fac = 1.0 / 3.0; constexpr std::array facs{1.0 * fac, 4.0 * fac, 1.0 * fac}; @@ -140,7 +134,7 @@ BoutReal integrate_lagrange_curve_nc3(const BoutReal start, const BoutReal end, BoutReal integrate_lagrange_curve_nc2(const BoutReal start, const BoutReal end, const std::deque& points, const int position) { - AUTO_TRACE(); + constexpr std::size_t size = 2; constexpr BoutReal fac = 1.0 / 2.0; constexpr std::array facs{1.0 * fac, 1.0 * fac}; @@ -151,7 +145,6 @@ BoutReal integrate_lagrange_curve_nc2(const BoutReal start, const BoutReal end, BoutReal integrate_lagrange_curve(const BoutReal start, const BoutReal end, const std::deque& points, const int position, const int order) { - AUTO_TRACE(); switch (order) { case 1: @@ -179,7 +172,7 @@ BoutReal integrate_lagrange_curve(const BoutReal start, const BoutReal end, std::vector get_adams_bashforth_coefficients(const BoutReal nextPoint, const std::deque& points, const int order) { - AUTO_TRACE(); + ASSERT2(static_cast(order) <= points.size()); std::vector result; @@ -235,7 +228,7 @@ BoutReal get_timestep_limit(const BoutReal error, const BoutReal tolerance, /// over all processors. BoutReal get_error(const Array& stateApprox, const Array& stateAccurate) { - AUTO_TRACE(); + BoutReal local_result = 0.0; BoutReal err = 0.0; @@ -287,12 +280,12 @@ AdamsBashforthSolver::AdamsBashforthSolver(Options* options) .withDefault(getOutputTimestep())), timestep( (*options)["timestep"].doc("Starting timestep").withDefault(max_timestep)) { - AUTO_TRACE(); + canReset = true; } void AdamsBashforthSolver::setMaxTimestep(BoutReal dt) { - AUTO_TRACE(); + if (dt > timestep) { return; // Already less than this } @@ -306,8 +299,6 @@ void AdamsBashforthSolver::setMaxTimestep(BoutReal dt) { int AdamsBashforthSolver::init() { - TRACE("Initialising AdamsBashforth solver"); - Solver::init(); output << "\n\tAdams-Bashforth (explicit) multistep solver\n"; @@ -345,7 +336,6 @@ int AdamsBashforthSolver::init() { } void AdamsBashforthSolver::resetInternalFields() { - AUTO_TRACE(); // History and times history.clear(); @@ -360,7 +350,6 @@ void AdamsBashforthSolver::resetInternalFields() { } int AdamsBashforthSolver::run() { - AUTO_TRACE(); // Just for developer diagnostics int nwasted = 0; @@ -563,7 +552,6 @@ int AdamsBashforthSolver::run() { BoutReal AdamsBashforthSolver::take_step(const BoutReal timeIn, const BoutReal dt, const int order, Array& current, Array& result) { - AUTO_TRACE(); Array full_update = AB_integrate(nlocal, timeIn + dt, times, history, order); diff --git a/src/solver/impls/arkode/arkode.cxx b/src/solver/impls/arkode/arkode.cxx index 0151f90167..804e67a803 100644 --- a/src/solver/impls/arkode/arkode.cxx +++ b/src/solver/impls/arkode/arkode.cxx @@ -188,7 +188,6 @@ ArkodeSolver::~ArkodeSolver() { **************************************************************************/ int ArkodeSolver::init() { - TRACE("Initialising ARKODE solver"); Solver::init(); @@ -507,8 +506,6 @@ int ArkodeSolver::init() { **************************************************************************/ int ArkodeSolver::run() { - TRACE("ArkodeSolver::run()"); - if (!initialised) { throw BoutException("ArkodeSolver not initialised\n"); } diff --git a/src/solver/impls/cvode/cvode.cxx b/src/solver/impls/cvode/cvode.cxx index 8b81a66f6d..4f72e65933 100644 --- a/src/solver/impls/cvode/cvode.cxx +++ b/src/solver/impls/cvode/cvode.cxx @@ -177,7 +177,6 @@ CvodeSolver::~CvodeSolver() { **************************************************************************/ int CvodeSolver::init() { - TRACE("Initialising CVODE solver"); Solver::init(); @@ -346,7 +345,6 @@ int CvodeSolver::init() { } } else { output_info.write("\tUsing Newton iteration\n"); - TRACE("Setting preconditioner"); const auto prectype = use_precon ? (rightprec ? SUN_PREC_RIGHT : SUN_PREC_LEFT) : SUN_PREC_NONE; @@ -486,7 +484,6 @@ CvodeSolver::create_constraints(const std::vector>& fields) { **************************************************************************/ int CvodeSolver::run() { - TRACE("CvodeSolver::run()"); if (!cvode_initialised) { throw BoutException("CvodeSolver not initialised\n"); @@ -814,7 +811,7 @@ void CvodeSolver::loop_vector_option_values_op(Ind2D UNUSED(i2d), BoutReal* opti } void CvodeSolver::resetInternalFields() { - TRACE("CvodeSolver::resetInternalFields"); + save_vars(N_VGetArrayPointer(uvec)); if (CVodeReInit(cvode_mem, simtime, uvec) != CV_SUCCESS) { diff --git a/src/solver/impls/euler/euler.cxx b/src/solver/impls/euler/euler.cxx index 3976f4402c..9754a68826 100644 --- a/src/solver/impls/euler/euler.cxx +++ b/src/solver/impls/euler/euler.cxx @@ -1,16 +1,16 @@ - #include "euler.hxx" #include #include -#include +#include #include -#include - -#include - +#include #include +#include +#include +#include + EulerSolver::EulerSolver(Options* options) : Solver(options), mxstep((*options)["mxstep"] .doc("Maximum number of steps between outputs") @@ -20,7 +20,10 @@ EulerSolver::EulerSolver(Options* options) .withDefault(2.)), timestep((*options)["timestep"] .doc("Internal timestep (defaults to output timestep)") - .withDefault(getOutputTimestep())) {} + .withDefault(getOutputTimestep())), + dump_at_time((*options)["dump_at_time"] + .doc("Dump debug info about the simulation") + .withDefault(-1)) {} void EulerSolver::setMaxTimestep(BoutReal dt) { if (dt >= cfl_factor * timestep) { @@ -33,7 +36,6 @@ void EulerSolver::setMaxTimestep(BoutReal dt) { } int EulerSolver::init() { - TRACE("Initialising Euler solver"); Solver::init(); @@ -63,7 +65,6 @@ int EulerSolver::init() { } int EulerSolver::run() { - TRACE("EulerSolver::run()"); for (int s = 0; s < getNumberOutputSteps(); s++) { BoutReal target = simtime + getOutputTimestep(); @@ -141,7 +142,37 @@ void EulerSolver::take_step(BoutReal curtime, BoutReal dt, Array& star Array& result) { load_vars(std::begin(start)); + const bool dump_now = + (dump_at_time >= 0 && std::abs(dump_at_time - curtime) < dt) || dump_at_time < -3; + std::shared_ptr debug_ptr; + if (dump_now) { + debug_ptr = std::make_shared(); + Options& debug = *debug_ptr; + for (auto& f : f3d) { + f.F_var->enableTracking(fmt::format("ddt_{:s}", f.name), debug_ptr); + debug[fmt::format("pre_{:s}", f.name)] = *f.var; + f.var->allocate(); + } + } + run_rhs(curtime); + if (dump_now) { + Options& debug = *debug_ptr; + Mesh* mesh{nullptr}; + for (auto& f : f3d) { + saveParallel(debug, f.name, *f.var); + mesh = f.var->getMesh(); + } + + const std::string outnumber = + dump_at_time < -3 ? fmt::format(".{}", debug_counter++) : ""; + const std::string outname = fmt::format("BOUT.debug{}", outnumber); + bout::OptionsIO::write(outname, debug, mesh); + MPI_Barrier(BoutComm::get()); + for (auto& f : f3d) { + f.F_var->disableTracking(); + } + } save_derivs(std::begin(result)); BOUT_OMP_PERF(parallel for) diff --git a/src/solver/impls/euler/euler.hxx b/src/solver/impls/euler/euler.hxx index 0ee81a3d33..fc9b7f53bb 100644 --- a/src/solver/impls/euler/euler.hxx +++ b/src/solver/impls/euler/euler.hxx @@ -64,6 +64,9 @@ private: /// Take a single step to calculate f1 void take_step(BoutReal curtime, BoutReal dt, Array& start, Array& result); + + BoutReal dump_at_time{-1.}; + int debug_counter{0}; }; #endif // BOUT_KARNIADAKIS_SOLVER_H diff --git a/src/solver/impls/ida/ida.cxx b/src/solver/impls/ida/ida.cxx index e0354e8b57..84a0bd4abd 100644 --- a/src/solver/impls/ida/ida.cxx +++ b/src/solver/impls/ida/ida.cxx @@ -100,8 +100,6 @@ IdaSolver::~IdaSolver() { int IdaSolver::init() { - TRACE("Initialising IDA solver"); - Solver::init(); output.write("Initialising IDA solver\n"); @@ -233,7 +231,6 @@ int IdaSolver::init() { **************************************************************************/ int IdaSolver::run() { - TRACE("IDA IdaSolver::run()"); if (!initialised) { throw BoutException("IdaSolver not initialised\n"); diff --git a/src/solver/impls/imex-bdf2/imex-bdf2.cxx b/src/solver/impls/imex-bdf2/imex-bdf2.cxx index 07188110ab..4ccf27981b 100644 --- a/src/solver/impls/imex-bdf2/imex-bdf2.cxx +++ b/src/solver/impls/imex-bdf2/imex-bdf2.cxx @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -136,8 +135,6 @@ static PetscErrorCode imexbdf2PCapply(PC pc, Vec x, Vec y) { */ int IMEXBDF2::init() { - TRACE("Initialising IMEX-BDF2 solver"); - Solver::init(); output << "\n\tIMEX-BDF2 time-integration solver\n"; @@ -331,7 +328,7 @@ void IMEXBDF2::constructSNES(SNES* snesIn) { if (mesh->firstX()) { // Lower X boundary for (int y = mesh->ystart; y <= mesh->yend; y++) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { int localIndex = ROUND(index(mesh->xstart, y, z)); ASSERT2((localIndex >= 0) && (localIndex < localN)); if (z == 0) { @@ -350,7 +347,7 @@ void IMEXBDF2::constructSNES(SNES* snesIn) { } else { // On another processor for (int y = mesh->ystart; y <= mesh->yend; y++) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { int localIndex = ROUND(index(mesh->xstart, y, z)); ASSERT2((localIndex >= 0) && (localIndex < localN)); if (z == 0) { @@ -373,7 +370,7 @@ void IMEXBDF2::constructSNES(SNES* snesIn) { if (mesh->lastX()) { // Upper X boundary for (int y = mesh->ystart; y <= mesh->yend; y++) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { int localIndex = ROUND(index(mesh->xend, y, z)); ASSERT2((localIndex >= 0) && (localIndex < localN)); if (z == 0) { @@ -392,7 +389,7 @@ void IMEXBDF2::constructSNES(SNES* snesIn) { } else { // On another processor for (int y = mesh->ystart; y <= mesh->yend; y++) { - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { int localIndex = ROUND(index(mesh->xend, y, z)); ASSERT2((localIndex >= 0) && (localIndex < localN)); if (z == 0) { @@ -559,7 +556,7 @@ void IMEXBDF2::constructSNES(SNES* snesIn) { } // 3D fields - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { int ind = ROUND(index(x, y, z)); @@ -741,7 +738,6 @@ void IMEXBDF2::constructSNES(SNES* snesIn) { } int IMEXBDF2::run() { - TRACE("IMEXBDF2::run()"); // Multi-step scheme, so first steps are different int order = 1; @@ -1420,7 +1416,7 @@ void IMEXBDF2::loopVars(BoutReal* u) { if (mesh->firstX() && !mesh->periodicX) { for (int jx = 0; jx < mesh->xstart; ++jx) { for (int jy = mesh->ystart; jy <= mesh->yend; ++jy) { - for (int jz = 0; jz < mesh->LocalNz; ++jz) { + for (int jz = mesh->zstart; jz <= mesh->zend; ++jz) { op.run(jx, jy, jz, u); ++u; } @@ -1432,7 +1428,7 @@ void IMEXBDF2::loopVars(BoutReal* u) { if (mesh->lastX() && !mesh->periodicX) { for (int jx = mesh->xend + 1; jx < mesh->LocalNx; ++jx) { for (int jy = mesh->ystart; jy <= mesh->yend; ++jy) { - for (int jz = 0; jz < mesh->LocalNz; ++jz) { + for (int jz = mesh->zstart; jz <= mesh->zend; ++jz) { op.run(jx, jy, jz, u); ++u; } @@ -1442,7 +1438,7 @@ void IMEXBDF2::loopVars(BoutReal* u) { // Lower Y for (RangeIterator xi = mesh->iterateBndryLowerY(); !xi.isDone(); ++xi) { for (int jy = 0; jy < mesh->ystart; ++jy) { - for (int jz = 0; jz < mesh->LocalNz; ++jz) { + for (int jz = mesh->zstart; jz <= mesh->zend; ++jz) { op.run(*xi, jy, jz, u); ++u; } @@ -1452,7 +1448,7 @@ void IMEXBDF2::loopVars(BoutReal* u) { // Upper Y for (RangeIterator xi = mesh->iterateBndryUpperY(); !xi.isDone(); ++xi) { for (int jy = mesh->yend + 1; jy < mesh->LocalNy; ++jy) { - for (int jz = 0; jz < mesh->LocalNz; ++jz) { + for (int jz = mesh->zstart; jz <= mesh->zend; ++jz) { op.run(*xi, jy, jz, u); ++u; } @@ -1463,7 +1459,7 @@ void IMEXBDF2::loopVars(BoutReal* u) { // Bulk of points for (int jx = mesh->xstart; jx <= mesh->xend; ++jx) { for (int jy = mesh->ystart; jy <= mesh->yend; ++jy) { - for (int jz = 0; jz < mesh->LocalNz; ++jz) { + for (int jz = mesh->zstart; jz <= mesh->zend; ++jz) { op.run(jx, jy, jz, u); ++u; } diff --git a/src/solver/impls/petsc/petsc.cxx b/src/solver/impls/petsc/petsc.cxx index ed48d87312..cc884c240c 100644 --- a/src/solver/impls/petsc/petsc.cxx +++ b/src/solver/impls/petsc/petsc.cxx @@ -2,7 +2,7 @@ * Interface to PETSc solver * ************************************************************************** - * Copyright 2010 - 2025 BOUT++ contributors + * Copyright 2010 - 2026 BOUT++ contributors * * Contact: Ben Dudson, dudson2@llnl.gov * @@ -64,10 +64,10 @@ class ColoringStencil { private: - bool static isInSquare(int const i, int const j, int const n_square) { + bool static isInSquare(const int i, const int j, const int n_square) { return std::abs(i) <= n_square && std::abs(j) <= n_square; } - bool static isInCross(int const i, int const j, int const n_cross) { + bool static isInCross(const int i, const int j, const int n_cross) { if (i == 0) { return std::abs(j) <= n_cross; } @@ -76,7 +76,7 @@ class ColoringStencil { } return false; } - bool static isInTaxi(int const i, int const j, int const n_taxi) { + bool static isInTaxi(const int i, const int j, const int n_taxi) { return std::abs(i) + std::abs(j) <= n_taxi; } @@ -165,6 +165,39 @@ PetscErrorCode PetscMonitor(TS ts, PetscInt UNUSED(step), PetscReal t, Vec X, vo PetscFunctionBegin; auto* s = static_cast(ctx); + + if (s->diagnose) { + // Print diagnostic information. + // Using the same format at the SNES solver + + SNESConvergedReason reason; + SNESGetConvergedReason(s->snes, &reason); + + PetscReal timestep; + TSGetTimeStep(s->ts, ×tep); + + // SNES failure counter is only reset when TSSolve is called + static PetscInt prev_snes_failures = 0; + PetscInt snes_failures; + TSGetSNESFailures(s->ts, &snes_failures); + snes_failures -= prev_snes_failures; + prev_snes_failures += snes_failures; + + // Get number of iterations + int nl_its; + SNESGetIterationNumber(s->snes, &nl_its); + int lin_its; + SNESGetLinearSolveIterations(s->snes, &lin_its); + + output.print("\r"); // Carriage return for printing to screen + output.write("Time: {}, timestep: {}, nl iter: {}, lin iter: {}, reason: {}", t, + timestep, nl_its, lin_its, static_cast(reason)); + if (snes_failures > 0) { + output.write(", SNES failures: {}", snes_failures); + } + output.write("\n"); + } + if (t < s->next_output) { // Not reached output time yet => return PetscFunctionReturn(0); @@ -292,8 +325,6 @@ PetscSolver::~PetscSolver() { int PetscSolver::init() { - TRACE("Initialising PETSc-dev solver"); - Solver::init(); int nlocal = getLocalN(); // Number of evolving variables on this processor @@ -583,7 +614,7 @@ int PetscSolver::init() { n_taxi = 2; } - auto const xy_offsets = ColoringStencil::getOffsets(n_square, n_taxi, n_cross); + const auto xy_offsets = ColoringStencil::getOffsets(n_square, n_taxi, n_cross); { // This is ugly but can't think of a better and robust way to // count the non-zeros for some arbitrary stencil @@ -627,7 +658,7 @@ int PetscSolver::init() { } } // 3D fields - for (int z = 0; z < mesh->LocalNz; z++) { + for (int z = mesh->zstart; z <= mesh->zend; z++) { const int ind = ROUND(index(x, y, z)) - Istart; for (int i = 0; i < n3d; i++) { @@ -703,7 +734,7 @@ int PetscSolver::init() { // Mark non-zero entries output_progress.write("Marking non-zero Jacobian entries\n"); - PetscScalar const val = 1.0; + const PetscScalar val = 1.0; for (int x = mesh->xstart; x <= mesh->xend; x++) { for (int y = mesh->ystart; y <= mesh->yend; y++) { @@ -722,21 +753,21 @@ int PetscSolver::init() { continue; } - int const ind2 = ROUND(index(xi, yi, 0)); + const int ind2 = ROUND(index(xi, yi, 0)); if (ind2 < 0) { continue; // A boundary point } // Depends on all variables on this cell for (int j = 0; j < n2d; j++) { - PetscInt const col = ind2 + j; + const PetscInt col = ind2 + j; PetscCall(MatSetValues(Jfd, 1, &row, 1, &col, &val, INSERT_VALUES)); } } } // 3D fields - for (int z = 0; z < mesh->LocalNz; z++) { - int const ind = ROUND(index(x, y, z)); + for (int z = mesh->zstart; z <= mesh->zend; z++) { + const int ind = ROUND(index(x, y, z)); for (int i = 0; i < n3d; i++) { PetscInt row = ind + i; @@ -746,7 +777,7 @@ int PetscSolver::init() { // Depends on 2D fields for (int j = 0; j < n2d; j++) { - PetscInt const col = ind0 + j; + const PetscInt col = ind0 + j; PetscCall(MatSetValues(Jfd, 1, &row, 1, &col, &val, INSERT_VALUES)); } @@ -759,7 +790,7 @@ int PetscSolver::init() { || (yi >= mesh->LocalNy)) { continue; } - for (int zi = 0; zi < mesh->LocalNz; ++zi) { + for (int zi = mesh->zstart; zi <= mesh->zend; ++zi) { int ind2 = ROUND(index(xi, yi, zi)); if (ind2 < 0) { continue; // Boundary point @@ -771,12 +802,12 @@ int PetscSolver::init() { // 3D fields on this cell for (int j = 0; j < n3d; j++) { - PetscInt const col = ind2 + j; + const PetscInt col = ind2 + j; int ierr = MatSetValues(Jfd, 1, &row, 1, &col, &val, INSERT_VALUES); if (ierr != 0) { output.write("ERROR: {} {} : ({}, {}) -> ({}, {}) : {} -> {}\n", - row, x, y, xi, yi, ind2, ind2 + n3d - 1); + row, col, x, y, xi, yi, ind2, ind2 + n3d - 1); } CHKERRQ(ierr); } @@ -903,8 +934,17 @@ PetscErrorCode PetscSolver::rhs(BoutReal t, Vec udata, Vec dudata, bool linear) load_vars(const_cast(udata_array)); VecRestoreArrayRead(udata, &udata_array); - // Call RHS function - run_rhs(t, linear); + try { + // Call RHS function + run_rhs(t, linear); + } catch (BoutException& e) { + // Simulation might fail, e.g. negative densities + // if timestep too large + output_error.write("BoutException thrown: {}\n", e.what()); + // There is no way to recover and synchronise MPI ranks + // unless they all threw an exception at the same point. + BoutComm::abort(1); + } // Save derivatives to PETSc BoutReal* dudata_array; @@ -926,7 +966,6 @@ PetscErrorCode PetscSolver::formFunction(Vec U, Vec F) { // Matrix-free preconditioner function PetscErrorCode PetscSolver::pre(Vec x, Vec y) { - TRACE("PetscSolver::pre()"); BoutReal* data; diff --git a/src/solver/impls/power/power.cxx b/src/solver/impls/power/power.cxx index 888338b94c..5dfccfedda 100644 --- a/src/solver/impls/power/power.cxx +++ b/src/solver/impls/power/power.cxx @@ -3,19 +3,16 @@ #include "power.hxx" #include -#include +#include #include #include -#include - PowerSolver::PowerSolver(Options* opts) : Solver(opts), curtime((*options)["curtime"].doc("Simulation time (fixed)").withDefault(0.0)) {} int PowerSolver::init() { - TRACE("Initialising Power solver"); Solver::init(); output << "\n\tPower eigenvalue solver\n"; @@ -44,7 +41,6 @@ int PowerSolver::init() { } int PowerSolver::run() { - TRACE("PowerSolver::run()"); // Make sure that f0 has a norm of 1 divide(f0, norm(f0)); diff --git a/src/solver/impls/pvode/pvode.cxx b/src/solver/impls/pvode/pvode.cxx index 63d3c11753..38901a2a57 100644 --- a/src/solver/impls/pvode/pvode.cxx +++ b/src/solver/impls/pvode/pvode.cxx @@ -31,23 +31,58 @@ #include #include +#include +#include #include #include +#include +#include #include #include -#include "bout/unused.hxx" - +#include #include // use CVSPGMR linear solver each internal step #include // contains the enum for types of preconditioning #include // band preconditioner function prototypes +#include "fmt/format.h" + +#include +#include +#include + using namespace pvode; void solver_f(integer N, BoutReal t, N_Vector u, N_Vector udot, void* f_data); void solver_gloc(integer N, BoutReal t, BoutReal* u, BoutReal* udot, void* f_data); void solver_cfn(integer N, BoutReal t, N_Vector u, void* f_data); +namespace { +// local only +void pvode_load_data_f3d(const std::vector& evolve_bndrys, + std::vector& ffs, const BoutReal* udata) { + int p = 0; + const Mesh* mesh = ffs[0].getMesh(); + const int nz = mesh->LocalNz; + for (const auto& bndry : {true, false}) { + for (const auto& i2d : mesh->getRegion2D(bndry ? "RGN_BNDRY" : "RGN_NOBNDRY")) { + for (int jz = 0; jz < nz; jz++) { + // Loop over 3D variables + auto evolve_bndry = evolve_bndrys.cbegin(); + for (auto ff = ffs.begin(); ff != ffs.end(); ++ff) { + if (bndry && !*evolve_bndry) { + continue; + } + (*ff)[mesh->ind2Dto3D(i2d, jz)] = udata[p]; + p++; + } + ++evolve_bndry; + } + } + } +} +} // namespace + const BoutReal ZERO = 0.0; long int iopt[OPT_SIZE]; @@ -90,7 +125,6 @@ PvodeSolver::~PvodeSolver() { **************************************************************************/ int PvodeSolver::init() { - TRACE("Initialising PVODE solver"); int mudq, mldq, mukeep, mlkeep; boole optIn; @@ -185,10 +219,47 @@ int PvodeSolver::init() { for (i = 0; i < OPT_SIZE; i++) { ropt[i] = ZERO; } + /* iopt[MXSTEP] : maximum number of internal steps to be taken by * + * the solver in its attempt to reach tout. * + * Optional input. (Default = 500). */ iopt[MXSTEP] = pvode_mxstep; - cvode_mem = CVodeMalloc(neq, solver_f, simtime, u, BDF, NEWTON, SS, &reltol, &abstol, - this, nullptr, optIn, iopt, ropt, machEnv); + { + /* ropt[H0] : initial step size. Optional input. */ + + /* ropt[HMAX] : maximum absolute value of step size allowed. * + * Optional input. (Default is infinity). */ + const BoutReal hmax( + (*options)["max_timestep"].doc("Maximum internal timestep").withDefault(-1.)); + if (hmax > 0) { + ropt[HMAX] = hmax; + } + /* ropt[HMIN] : minimum absolute value of step size allowed. * + * Optional input. (Default is 0.0). */ + const BoutReal hmin( + (*options)["min_timestep"].doc("Minimum internal timestep").withDefault(-1.)); + if (hmin > 0) { + ropt[HMIN] = hmin; + } + /* iopt[MAXORD] : maximum lmm order to be used by the solver. * + * Optional input. (Default = 12 for ADAMS, 5 for * + * BDF). */ + const int maxOrder((*options)["max_order"].doc("Maximum order").withDefault(-1)); + if (maxOrder > 0) { + iopt[MAXORD] = maxOrder; + } + } + const bool use_adam((*options)["adams_moulton"] + .doc("Use Adams Moulton solver instead of BDF") + .withDefault(false)); + + debug_on_failure = + (*options)["debug_on_failure"] + .doc("Run an aditional rhs if the solver fails with extra tracking") + .withDefault(false); + + cvode_mem = CVodeMalloc(neq, solver_f, simtime, u, use_adam ? ADAMS : BDF, NEWTON, SS, + &reltol, &abstol, this, nullptr, optIn, iopt, ropt, machEnv); if (cvode_mem == nullptr) { throw BoutException("\tError: CVodeMalloc failed.\n"); @@ -218,7 +289,6 @@ int PvodeSolver::init() { **************************************************************************/ int PvodeSolver::run() { - TRACE("PvodeSolver::run()"); if (!pvode_initialised) { throw BoutException("PvodeSolver not initialised\n"); @@ -293,7 +363,43 @@ BoutReal PvodeSolver::run(BoutReal tout) { // Check return flag if (flag != SUCCESS) { output_error.write("ERROR CVODE step failed, flag = {:d}\n", flag); - return (-1.0); + if (debug_on_failure) { + const CVodeMemRec* cv_mem = static_cast(cvode_mem); + if (!(f2d.empty() and v2d.empty() and v3d.empty())) { + output_warn.write("debug_on_failure is currently only supported for Field3Ds"); + return -1.0; + } + auto debug_ptr = std::make_shared(); + Options& debug = *debug_ptr; + using namespace std::string_literals; + Mesh* mesh{}; + for (const auto& prefix : {"pre_"s, "residuum_"s}) { + std::vector list_of_fields{}; + std::vector evolve_bndrys{}; + for (const auto& f : f3d) { + mesh = f.var->getMesh(); + Field3D to_load{0., mesh}; + to_load.setLocation(f.location); + debug[fmt::format("{:s}{:s}", prefix, f.name)] = to_load; + list_of_fields.push_back(to_load); + evolve_bndrys.push_back(f.evolve_bndry); + } + pvode_load_data_f3d(evolve_bndrys, list_of_fields, + prefix == "pre_"s ? udata : N_VDATA(cv_mem->cv_acor)); + } + + for (auto& f : f3d) { + f.F_var->enableTracking(fmt::format("ddt_{:s}", f.name), debug_ptr); + } + run_rhs(simtime); + + for (auto& f : f3d) { + saveParallel(debug, f.name, *f.var); + } + bout::OptionsIO::write("BOUT.debug", debug, mesh); + MPI_Barrier(BoutComm::get()); + } + return -1.0; } return simtime; @@ -303,7 +409,8 @@ BoutReal PvodeSolver::run(BoutReal tout) { * RHS function **************************************************************************/ -void PvodeSolver::rhs(int UNUSED(N), BoutReal t, BoutReal* udata, BoutReal* dudata) { +void PvodeSolver::rhs([[maybe_unused]] int N, BoutReal t, BoutReal* udata, + BoutReal* dudata) { TRACE("Running RHS: PvodeSolver::rhs({})", t); // Get current timestep @@ -319,7 +426,8 @@ void PvodeSolver::rhs(int UNUSED(N), BoutReal t, BoutReal* udata, BoutReal* duda save_derivs(dudata); } -void PvodeSolver::gloc(int UNUSED(N), BoutReal t, BoutReal* udata, BoutReal* dudata) { +void PvodeSolver::gloc([[maybe_unused]] int N, BoutReal t, BoutReal* udata, + BoutReal* dudata) { TRACE("Running RHS: PvodeSolver::gloc({})", t); Timer timer("rhs"); @@ -360,8 +468,8 @@ void solver_gloc(integer N, BoutReal t, BoutReal* u, BoutReal* udot, void* f_dat } // Preconditioner communication function -void solver_cfn(integer UNUSED(N), BoutReal UNUSED(t), N_Vector UNUSED(u), - void* UNUSED(f_data)) { +void solver_cfn([[maybe_unused]] integer N, [[maybe_unused]] BoutReal t, + [[maybe_unused]] N_Vector u, [[maybe_unused]] void* f_data) { // doesn't do anything at the moment } diff --git a/src/solver/impls/pvode/pvode.hxx b/src/solver/impls/pvode/pvode.hxx index 6425fc1868..cf68444b6c 100644 --- a/src/solver/impls/pvode/pvode.hxx +++ b/src/solver/impls/pvode/pvode.hxx @@ -75,10 +75,15 @@ private: pvode::machEnvType machEnv{nullptr}; void* cvode_mem{nullptr}; - BoutReal abstol, reltol; // addresses passed in init must be preserved + BoutReal abstol, reltol; + // addresses passed in init must be preserved pvode::PVBBDData pdata{nullptr}; - bool pvode_initialised = false; + /// is pvode already initialised? + bool pvode_initialised{false}; + + /// Add debugging data if solver fails + bool debug_on_failure{false}; }; #endif // BOUT_PVODE_SOLVER_H diff --git a/src/solver/impls/rk3-ssp/rk3-ssp.cxx b/src/solver/impls/rk3-ssp/rk3-ssp.cxx index e13d996c00..7c00d44b34 100644 --- a/src/solver/impls/rk3-ssp/rk3-ssp.cxx +++ b/src/solver/impls/rk3-ssp/rk3-ssp.cxx @@ -3,10 +3,8 @@ #include #include -#include #include #include -#include #include @@ -28,7 +26,6 @@ void RK3SSP::setMaxTimestep(BoutReal dt) { } int RK3SSP::init() { - TRACE("Initialising RK3 SSP solver"); Solver::init(); output << "\n\tRunge-Kutta 3rd-order SSP solver\n"; @@ -63,7 +60,6 @@ int RK3SSP::init() { } int RK3SSP::run() { - TRACE("RK3SSP::run()"); for (int s = 0; s < getNumberOutputSteps(); s++) { BoutReal target = simtime + getOutputTimestep(); diff --git a/src/solver/impls/rk4/rk4.cxx b/src/solver/impls/rk4/rk4.cxx index 0e7a942a45..35b3cc2268 100644 --- a/src/solver/impls/rk4/rk4.cxx +++ b/src/solver/impls/rk4/rk4.cxx @@ -3,14 +3,12 @@ #include #include -#include #include +#include #include #include -#include - RK4Solver::RK4Solver(Options* opts) : Solver(opts), atol((*options)["atol"].doc("Absolute tolerance").withDefault(1.e-5)), rtol((*options)["rtol"].doc("Relative tolerance").withDefault(1.e-3)), @@ -39,8 +37,6 @@ void RK4Solver::setMaxTimestep(BoutReal dt) { int RK4Solver::init() { - TRACE("Initialising RK4 solver"); - Solver::init(); output << "\n\tRunge-Kutta 4th-order solver\n"; @@ -77,8 +73,6 @@ int RK4Solver::init() { } int RK4Solver::run() { - TRACE("RK4Solver::run()"); - for (int s = 0; s < getNumberOutputSteps(); s++) { BoutReal target = simtime + getOutputTimestep(); diff --git a/src/solver/impls/rkgeneric/rkgeneric.cxx b/src/solver/impls/rkgeneric/rkgeneric.cxx index 1c332d26de..edf8ed50e3 100644 --- a/src/solver/impls/rkgeneric/rkgeneric.cxx +++ b/src/solver/impls/rkgeneric/rkgeneric.cxx @@ -4,12 +4,8 @@ #include #include -#include -#include - -#include - #include +#include RKGenericSolver::RKGenericSolver(Options* opts) : Solver(opts), atol((*options)["atol"].doc("Absolute tolerance").withDefault(1.e-5)), @@ -40,8 +36,6 @@ void RKGenericSolver::setMaxTimestep(BoutReal dt) { int RKGenericSolver::init() { - TRACE("Initialising RKGeneric solver"); - Solver::init(); output << "\n\tRunge-Kutta generic solver with scheme type " << scheme->getType() << "\n"; @@ -86,8 +80,6 @@ void RKGenericSolver::resetInternalFields() { } int RKGenericSolver::run() { - TRACE("RKGenericSolver::run()"); - for (int s = 0; s < getNumberOutputSteps(); s++) { BoutReal target = simtime + getOutputTimestep(); diff --git a/src/solver/impls/slepc/slepc.cxx b/src/solver/impls/slepc/slepc.cxx index 13657a4553..2c52cb3b38 100644 --- a/src/solver/impls/slepc/slepc.cxx +++ b/src/solver/impls/slepc/slepc.cxx @@ -31,7 +31,6 @@ #include #include #include -#include #include #include @@ -241,8 +240,6 @@ SlepcSolver::~SlepcSolver() { int SlepcSolver::init() { - TRACE("Initialising SLEPc solver"); - // Report initialisation output.write("Initialising SLEPc solver\n"); if (selfSolve) { diff --git a/src/solver/impls/snes/snes.cxx b/src/solver/impls/snes/snes.cxx index c10095af53..31877f8aac 100644 --- a/src/solver/impls/snes/snes.cxx +++ b/src/solver/impls/snes/snes.cxx @@ -1,4 +1,4 @@ -#include "bout/build_config.hxx" +#include "bout/build_defines.hxx" #if BOUT_HAS_PETSC @@ -7,16 +7,24 @@ #include #include #include -#include +#include +#include #include #include +#include #include +#include +#include #include -#include - +#include "petscerror.h" +#include "petscmat.h" +#include "petscpc.h" #include "petscsnes.h" +#include "petscsys.h" +#include "petscsystypes.h" +#include "petscvec.h" class ColoringStencil { private: @@ -83,29 +91,410 @@ PetscErrorCode FormFunctionForDifferencing(void* ctx, Vec x, Vec f) { * * This can be a linearised and simplified form of FormFunction */ -PetscErrorCode FormFunctionForColoring(void* UNUSED(snes), Vec x, Vec f, - void* ctx) { +PetscErrorCode FormFunctionForColoring(void* UNUSED(snes), Vec x, Vec f, void* ctx) { return static_cast(ctx)->snes_function(x, f, true); } PetscErrorCode snesPCapply(PC pc, Vec x, Vec y) { // Get the context SNESSolver* s; - int ierr = PCShellGetContext(pc, reinterpret_cast(&s)); - CHKERRQ(ierr); + PetscCall(PCShellGetContext(pc, reinterpret_cast(&s))); PetscFunctionReturn(s->precon(x, y)); } +} // namespace + +PetscErrorCode SNESSolver::FDJinitialise() { + if (use_coloring) { + // Use matrix coloring. + // This greatly reduces the number of times the rhs() function + // needs to be evaluated when calculating the Jacobian. + + // Use global mesh for now + Mesh* mesh = bout::globals::mesh; + + ////////////////////////////////////////////////// + // Get the local indices by starting at 0 + Field3D index = globalIndex(0); + + ////////////////////////////////////////////////// + // Pre-allocate PETSc storage + + output_progress.write("Setting Jacobian matrix sizes\n"); + + const int n2d = f2d.size(); + const int n3d = f3d.size(); + + // Set size of Matrix on each processor to nlocal x nlocal + MatCreate(BoutComm::get(), &Jfd); + MatSetOption(Jfd, MAT_KEEP_NONZERO_PATTERN, PETSC_TRUE); + MatSetSizes(Jfd, nlocal, nlocal, PETSC_DETERMINE, PETSC_DETERMINE); + MatSetFromOptions(Jfd); + // Determine which row/columns of the matrix are locally owned + int Istart, Iend; + MatGetOwnershipRange(Jfd, &Istart, &Iend); + // Convert local into global indices + // Note: Not in the boundary cells, to keep -1 values + for (const auto& i : mesh->getRegion3D("RGN_NOBNDRY")) { + index[i] += Istart; + } + // Now communicate to fill guard cells + mesh->communicate(index); + + // Non-zero elements on this processor + std::vector d_nnz; + std::vector o_nnz; + auto n_square = (*options)["stencil:square"] + .doc("Extent of stencil (square)") + .withDefault(0); + auto n_cross = + (*options)["stencil:cross"].doc("Extent of stencil (cross)").withDefault(0); + // Set n_taxi 2 if nothing else is set + auto n_taxi = (*options)["stencil:taxi"] + .doc("Extent of stencil (taxi-cab norm)") + .withDefault((n_square == 0 && n_cross == 0) ? 2 : 0); + + auto const xy_offsets = ColoringStencil::getOffsets(n_square, n_taxi, n_cross); + { + // This is ugly but can't think of a better and robust way to + // count the non-zeros for some arbitrary stencil + // effectively the same loop as the one that sets the non-zeros below + std::vector> d_nnz_map2d(nlocal); + std::vector> o_nnz_map2d(nlocal); + std::vector> d_nnz_map3d(nlocal); + std::vector> o_nnz_map3d(nlocal); + // Loop over every element in 2D to count the *unique* non-zeros + for (int x = mesh->xstart; x <= mesh->xend; x++) { + for (int y = mesh->ystart; y <= mesh->yend; y++) { + + const int ind0 = ROUND(index(x, y, 0)) - Istart; + + // 2D fields + for (int i = 0; i < n2d; i++) { + const PetscInt row = ind0 + i; + // Loop through each point in the stencil + for (const auto& [x_off, y_off] : xy_offsets) { + const int xi = x + x_off; + const int yi = y + y_off; + if ((xi < 0) || (yi < 0) || (xi >= mesh->LocalNx) + || (yi >= mesh->LocalNy)) { + continue; + } + + const int ind2 = ROUND(index(xi, yi, 0)); + if (ind2 < 0) { + continue; // A boundary point + } + + // Depends on all variables on this cell + for (int j = 0; j < n2d; j++) { + const PetscInt col = ind2 + j; + if (col >= Istart && col < Iend) { + d_nnz_map2d[row].insert(col); + } else { + o_nnz_map2d[row].insert(col); + } + } + } + } + // 3D fields + for (int z = mesh->zstart; z <= mesh->zend; z++) { + const int ind = ROUND(index(x, y, z)) - Istart; + + for (int i = 0; i < n3d; i++) { + PetscInt row = ind + i; + if (z == 0) { + row += n2d; + } + + // Depends on 2D fields + for (int j = 0; j < n2d; j++) { + const PetscInt col = ind0 + j; + if (col >= Istart && col < Iend) { + d_nnz_map2d[row].insert(col); + } else { + o_nnz_map2d[row].insert(col); + } + } + + // Star pattern + for (const auto& [x_off, y_off] : xy_offsets) { + const int xi = x + x_off; + const int yi = y + y_off; + + if ((xi < 0) || (yi < 0) || (xi >= mesh->LocalNx) + || (yi >= mesh->LocalNy)) { + continue; + } + + int ind2 = ROUND(index(xi, yi, 0)); + if (ind2 < 0) { + continue; // Boundary point + } + + if (z == 0) { + ind2 += n2d; + } + + // 3D fields on this cell + for (int j = 0; j < n3d; j++) { + const PetscInt col = ind2 + j; + if (col >= Istart && col < Iend) { + d_nnz_map3d[row].insert(col); + } else { + o_nnz_map3d[row].insert(col); + } + } + } + } + } + } + } + + d_nnz.reserve(nlocal); + d_nnz.reserve(nlocal); + + for (int i = 0; i < nlocal; ++i) { + // Assume all elements in the z direction are potentially coupled + d_nnz.emplace_back((d_nnz_map3d[i].size() * mesh->LocalNz) + + d_nnz_map2d[i].size()); + o_nnz.emplace_back((o_nnz_map3d[i].size() * mesh->LocalNz) + + o_nnz_map2d[i].size()); + } + } + + output_progress.write("Pre-allocating Jacobian\n"); + // Pre-allocate + MatMPIAIJSetPreallocation(Jfd, 0, d_nnz.data(), 0, o_nnz.data()); + MatSeqAIJSetPreallocation(Jfd, 0, d_nnz.data()); + MatSetUp(Jfd); + MatSetOption(Jfd, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE); + + ////////////////////////////////////////////////// + // Mark non-zero entries + + output_progress.write("Marking non-zero Jacobian entries\n"); + const PetscScalar val = 1.0; + for (int x = mesh->xstart; x <= mesh->xend; x++) { + for (int y = mesh->ystart; y <= mesh->yend; y++) { + + const int ind0 = ROUND(index(x, y, 0)); + + // 2D fields + for (int i = 0; i < n2d; i++) { + const PetscInt row = ind0 + i; + + // Loop through each point in the stencil + for (const auto& [x_off, y_off] : xy_offsets) { + const int xi = x + x_off; + const int yi = y + y_off; + if ((xi < 0) || (yi < 0) || (xi >= mesh->LocalNx) || (yi >= mesh->LocalNy)) { + continue; + } + + const int ind2 = ROUND(index(xi, yi, 0)); + if (ind2 < 0) { + continue; // A boundary point + } + + // Depends on all variables on this cell + for (int j = 0; j < n2d; j++) { + const PetscInt col = ind2 + j; + PetscCall(MatSetValues(Jfd, 1, &row, 1, &col, &val, INSERT_VALUES)); + } + } + } + // 3D fields + for (int z = mesh->zstart; z <= mesh->zend; z++) { + int ind = ROUND(index(x, y, z)); + + for (int i = 0; i < n3d; i++) { + PetscInt row = ind + i; + if (z == 0) { + row += n2d; + } + + // Depends on 2D fields + for (int j = 0; j < n2d; j++) { + const PetscInt col = ind0 + j; + PetscCall(MatSetValues(Jfd, 1, &row, 1, &col, &val, INSERT_VALUES)); + } + + // Star pattern + for (const auto& [x_off, y_off] : xy_offsets) { + int xi = x + x_off; + int yi = y + y_off; + + if ((xi < 0) || (yi < 0) || (xi >= mesh->LocalNx) + || (yi >= mesh->LocalNy)) { + continue; + } + for (int zi = mesh->zstart; zi <= mesh->zend; ++zi) { + int ind2 = ROUND(index(xi, yi, zi)); + if (ind2 < 0) { + continue; // Boundary point + } + + if (z == 0) { + ind2 += n2d; + } + + // 3D fields on this cell + for (int j = 0; j < n3d; j++) { + const PetscInt col = ind2 + j; + PetscErrorCode ierr = + MatSetValues(Jfd, 1, &row, 1, &col, &val, INSERT_VALUES); + + if (ierr != PETSC_SUCCESS) { + output.write("ERROR: {} {} : ({}, {}) -> ({}, {}) : {} -> {}\n", row, + col, x, y, xi, yi, ind2, ind2 + n3d - 1); + } + CHKERRQ(ierr); + } + } + } + } + } + } + } + + // Finished marking non-zero entries + + output_progress.write("Assembling Jacobian matrix\n"); + + // Assemble Matrix + MatAssemblyBegin(Jfd, MAT_FINAL_ASSEMBLY); + MatAssemblyEnd(Jfd, MAT_FINAL_ASSEMBLY); + + { + // Test if the matrix is symmetric + // Values are 0 or 1 so tolerance (1e-5) shouldn't matter + PetscBool symmetric; + PetscCall(MatIsSymmetric(Jfd, 1e-5, &symmetric)); + if (!static_cast(symmetric)) { + output_warn.write("Jacobian pattern is not symmetric\n"); + } + } + + // The above can miss entries around the X-point branch cut: + // The diagonal terms are complicated because moving in X then Y + // is different from moving in Y then X at the X-point. + // Making sure the colouring matrix is symmetric does not + // necessarily give the correct stencil but may help. + if ((*options)["force_symmetric_coloring"] + .doc("Modifies coloring matrix to force it to be symmetric") + .withDefault(false)) { + Mat Jfd_T; + MatCreateTranspose(Jfd, &Jfd_T); + MatAXPY(Jfd, 1, Jfd_T, DIFFERENT_NONZERO_PATTERN); + } + + output_progress.write("Creating Jacobian coloring\n"); + updateColoring(); + + if (prune_jacobian) { + // Will remove small elements from the Jacobian. + // Save a copy to recover from over-pruning + PetscCall(MatDuplicate(Jfd, MAT_SHARE_NONZERO_PATTERN, &Jfd_original)); + } + } else { + // Brute force calculation + // There is usually no reason to use this, except as a check of + // the coloring calculation. + + MatCreateAIJ( + BoutComm::get(), nlocal, nlocal, // Local sizes + PETSC_DETERMINE, PETSC_DETERMINE, // Global sizes + 3, // Number of nonzero entries in diagonal portion of local submatrix + nullptr, + 0, // Number of nonzeros per row in off-diagonal portion of local submatrix + nullptr, &Jfd); + + if (matrix_free_operator) { + SNESSetJacobian(snes, Jmf, Jfd, SNESComputeJacobianDefault, this); + } else { + SNESSetJacobian(snes, Jfd, Jfd, SNESComputeJacobianDefault, this); + } + + MatSetOption(Jfd, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE); + } + return PETSC_SUCCESS; +} + +PetscErrorCode SNESSolver::FDJpruneJacobian() { +#if PETSC_VERSION_GE(3, 20, 0) + + // Remove small elements from the Jacobian and recompute the coloring + // Only do this if there are a significant number of small elements. + int small_elements = 0; + int total_elements = 0; + + // Get index of rows owned by this processor + int rstart, rend; + MatGetOwnershipRange(Jfd, &rstart, &rend); + + PetscInt ncols; + const PetscScalar* vals; + for (int row = rstart; row < rend; row++) { + MatGetRow(Jfd, row, &ncols, nullptr, &vals); + for (int col = 0; col < ncols; col++) { + if (std::abs(vals[col]) < prune_abstol) { + ++small_elements; + } + ++total_elements; + } + MatRestoreRow(Jfd, row, &ncols, nullptr, &vals); + } + + if (small_elements > prune_fraction * total_elements) { + if (diagnose) { + output.write("\nPruning Jacobian elements: {} / {}\n", small_elements, + total_elements); + } + + // Prune Jacobian, keeping diagonal elements + PetscCall(MatFilter(Jfd, prune_abstol, PETSC_TRUE, PETSC_TRUE)); + + // Update the coloring from Jfd matrix + updateColoring(); + + // Mark the Jacobian as pruned. This is so that it is only restored if pruned. + jacobian_pruned = true; + } +#endif // PETSC_VERSION_GE(3,20,0) + return PETSC_SUCCESS; +} + +PetscErrorCode SNESSolver::FDJrestoreFromPruning() { + // Restore pruned non-zero elements + PetscCall(MatCopy(Jfd_original, Jfd, DIFFERENT_NONZERO_PATTERN)); + // The non-zero pattern has changed, so update coloring + updateColoring(); + jacobian_pruned = false; // Reset flag. Will be set after pruning. + return PETSC_SUCCESS; } SNESSolver::SNESSolver(Options* opts) : Solver(opts), + output_trigger( + (*options)["output_trigger"] + .doc("Decides when to save outputs. fixed_time_interval, residual_ratio") + .withDefault(BoutSnesOutput::fixed_time_interval)), + output_residual_ratio( + (*options)["output_residual_ratio"] + .doc("Trigger an output when residual falls by this ratio") + .withDefault(0.5)), timestep( (*options)["timestep"].doc("Initial backward Euler timestep").withDefault(1.0)), dt_min_reset((*options)["dt_min_reset"] .doc("If dt falls below this, reset to starting dt") .withDefault(1e-6)), max_timestep((*options)["max_timestep"].doc("Maximum timestep").withDefault(1e37)), + equation_form( + (*options)["equation_form"] + .doc("Form of equation to solve: rearranged_backward_euler (default);" + " pseudo_transient; backward_euler; direct_newton") + .withDefault(BoutSnesEquationForm::rearranged_backward_euler)), snes_type((*options)["snes_type"] .doc("PETSc nonlinear solver method to use") .withDefault("anderson")), @@ -127,6 +516,9 @@ SNESSolver::SNESSolver(Options* opts) upper_its((*options)["upper_its"] .doc("Iterations above which the next timestep is reduced") .withDefault(static_cast(maxits * 0.8))), + max_snes_failures((*options)["max_snes_failures"] + .doc("Abort after this number of consecutive failures") + .withDefault(10)), timestep_factor_on_failure((*options)["timestep_factor_on_failure"] .doc("Multiply timestep on convergence failure") .withDefault(0.5)), @@ -138,22 +530,44 @@ SNESSolver::SNESSolver(Options* opts) (*options)["timestep_factor_on_lower_its"] .doc("Multiply timestep if iterations are below lower_its") .withDefault(1.4)), - pid_controller( - (*options)["pid_controller"].doc("Use PID controller?").withDefault(false)), + pseudo_strategy((*options)["pseudo_strategy"] + .doc("PTC strategy to use when setting timesteps") + .withDefault(BoutPTCStrategy::inverse_residual)), + pseudo_alpha((*options)["pseudo_alpha"] + .doc("Sets timestep using dt = alpha / residual") + .withDefault(100. * atol * timestep)), + pseudo_growth_factor((*options)["pseudo_growth_factor"] + .doc("PTC growth factor on success") + .withDefault(1.1)), + pseudo_reduction_factor((*options)["pseudo_reduction_factor"] + .doc("PTC reduction factor on failure") + .withDefault(0.5)), + pseudo_max_ratio((*options)["pseudo_max_ratio"] + .doc("PTC maximum timestep ratio between neighbors") + .withDefault(2.)), + timestep_control((*options)["timestep_control"] + .doc("Timestep control method") + .withDefault(BoutSnesTimestep::pid_nonlinear_its)), + timestep_factor((*options)["timestep_factor"] + .doc("When timestep_control=residual_ratio, multiply timestep " + "by this factor each step") + .withDefault(1.1)), target_its((*options)["target_its"].doc("Target snes iterations").withDefault(7)), kP((*options)["kP"].doc("Proportional PID parameter").withDefault(0.7)), kI((*options)["kI"].doc("Integral PID parameter").withDefault(0.3)), kD((*options)["kD"].doc("Derivative PID parameter").withDefault(0.2)), + pid_consider_failures( + (*options)["pid_consider_failures"] + .doc("Reduce timestep increases if recent solves have failed") + .withDefault(false)), + last_failure_weight((*options)["last_failure_weight"] + .doc("Weighting of last timestep in recent failure rate") + .withDefault(0.1)), diagnose( (*options)["diagnose"].doc("Print additional diagnostics").withDefault(false)), diagnose_failures((*options)["diagnose_failures"] .doc("Print more diagnostics when SNES fails") .withDefault(false)), - equation_form( - (*options)["equation_form"] - .doc("Form of equation to solve: rearranged_backward_euler (default);" - " pseudo_transient; backward_euler; direct_newton") - .withDefault(BoutSnesEquationForm::rearranged_backward_euler)), predictor((*options)["predictor"].doc("Use linear predictor?").withDefault(true)), use_precon((*options)["use_precon"] .doc("Use user-supplied preconditioner?") @@ -185,6 +599,10 @@ SNESSolver::SNESSolver(Options* opts) lag_jacobian((*options)["lag_jacobian"] .doc("Re-use the Jacobian this number of SNES iterations") .withDefault(50)), + jacobian_persists( + (*options)["jacobian_persists"] + .doc("Re-use Jacobian and preconditioner across nonlinear solves") + .withDefault(false)), use_coloring((*options)["use_coloring"] .doc("Use matrix coloring to calculate Jacobian?") .withDefault(true)), @@ -203,12 +621,12 @@ SNESSolver::SNESSolver(Options* opts) .withDefault(false)), scale_vars((*options)["scale_vars"] .doc("Scale variables (Jacobian column scaling)?") + .withDefault(false)), + asinh_vars((*options)["asinh_vars"] + .doc("Apply asinh() to all variables?") .withDefault(false)) {} int SNESSolver::init() { - - TRACE("Initialising SNES solver"); - Solver::init(); output << "\n\tSNES steady state solver\n"; @@ -218,7 +636,8 @@ int SNESSolver::init() { // Get total problem size int ntmp; if (bout::globals::mpi->MPI_Allreduce(&nlocal, &ntmp, 1, MPI_INT, MPI_SUM, - BoutComm::get())) { + BoutComm::get()) + != 0) { throw BoutException("MPI_Allreduce failed!"); } neq = ntmp; @@ -227,55 +646,54 @@ int SNESSolver::init() { n3Dvars(), n2Dvars(), neq, nlocal); // Initialise PETSc components - int ierr; // Vectors output_info.write("Creating vector\n"); - ierr = VecCreate(BoutComm::get(), &snes_x); - CHKERRQ(ierr); - ierr = VecSetSizes(snes_x, nlocal, PETSC_DECIDE); - CHKERRQ(ierr); - ierr = VecSetFromOptions(snes_x); - CHKERRQ(ierr); - - VecDuplicate(snes_x, &snes_f); - VecDuplicate(snes_x, &x0); - - if (equation_form == BoutSnesEquationForm::rearranged_backward_euler) { - // Need an intermediate vector for rearranged Backward Euler - ierr = VecDuplicate(snes_x, &delta_x); - CHKERRQ(ierr); + PetscCall(VecCreate(BoutComm::get(), &snes_x)); + PetscCall(VecSetSizes(snes_x, nlocal, PETSC_DECIDE)); + PetscCall(VecSetFromOptions(snes_x)); + + PetscCall(VecDuplicate(snes_x, &snes_f)); + PetscCall(VecDuplicate(snes_x, &x0)); + + if ((equation_form == BoutSnesEquationForm::rearranged_backward_euler) + || (equation_form == BoutSnesEquationForm::pseudo_transient)) { + // Need an intermediate vector for rearranged Backward Euler or Pseudo-Transient Continuation + PetscCall(VecDuplicate(snes_x, &delta_x)); } if (predictor) { // Storage for previous solution - ierr = VecDuplicate(snes_x, &x1); - CHKERRQ(ierr); + PetscCall(VecDuplicate(snes_x, &x1)); } if (scale_rhs) { // Storage for rhs factors, one per evolving variable - ierr = VecDuplicate(snes_x, &rhs_scaling_factors); - CHKERRQ(ierr); + PetscCall(VecDuplicate(snes_x, &rhs_scaling_factors)); // Set all factors to 1 to start with - ierr = VecSet(rhs_scaling_factors, 1.0); - CHKERRQ(ierr); + PetscCall(VecSet(rhs_scaling_factors, 1.0)); // Array to store inverse Jacobian row norms - ierr = VecDuplicate(snes_x, &jac_row_inv_norms); - CHKERRQ(ierr); + PetscCall(VecDuplicate(snes_x, &jac_row_inv_norms)); } if (scale_vars) { // Storage for var factors, one per evolving variable - ierr = VecDuplicate(snes_x, &var_scaling_factors); - CHKERRQ(ierr); + PetscCall(VecDuplicate(snes_x, &var_scaling_factors)); // Set all factors to 1 to start with - ierr = VecSet(var_scaling_factors, 1.0); - CHKERRQ(ierr); + PetscCall(VecSet(var_scaling_factors, 1.0)); // Storage for scaled 'x' state vectors - ierr = VecDuplicate(snes_x, &scaled_x); - CHKERRQ(ierr); + PetscCall(VecDuplicate(snes_x, &scaled_x)); + } else if (asinh_vars) { + PetscCall(VecDuplicate(snes_x, &scaled_x)); + } + + if (equation_form == BoutSnesEquationForm::pseudo_transient) { + PetscCall(initPseudoTimestepping()); } + // Per-cell residuals + local_residual = 0.0; + local_residual_2d = 0.0; + global_residual = 0.0; // Nonlinear solver interface (SNES) output_info.write("Create SNES\n"); @@ -326,348 +744,22 @@ int SNESSolver::init() { // Create a vector to store interpolated output solution // Used so that the timestep does not have to be adjusted, // because that would require updating the preconditioner. - ierr = VecDuplicate(snes_x, &output_x); - CHKERRQ(ierr); - - if (use_coloring) { - // Use matrix coloring. - // This greatly reduces the number of times the rhs() function - // needs to be evaluated when calculating the Jacobian. - - // Use global mesh for now - Mesh* mesh = bout::globals::mesh; - - ////////////////////////////////////////////////// - // Get the local indices by starting at 0 - Field3D index = globalIndex(0); - - ////////////////////////////////////////////////// - // Pre-allocate PETSc storage - - output_progress.write("Setting Jacobian matrix sizes\n"); - - const int n2d = f2d.size(); - const int n3d = f3d.size(); - - // Set size of Matrix on each processor to nlocal x nlocal - MatCreate(BoutComm::get(), &Jfd); - MatSetOption(Jfd, MAT_KEEP_NONZERO_PATTERN, PETSC_TRUE); - MatSetSizes(Jfd, nlocal, nlocal, PETSC_DETERMINE, PETSC_DETERMINE); - MatSetFromOptions(Jfd); - // Determine which row/columns of the matrix are locally owned - int Istart, Iend; - MatGetOwnershipRange(Jfd, &Istart, &Iend); - // Convert local into global indices - // Note: Not in the boundary cells, to keep -1 values - for (const auto& i : mesh->getRegion3D("RGN_NOBNDRY")) { - index[i] += Istart; - } - // Now communicate to fill guard cells - mesh->communicate(index); - - // Non-zero elements on this processor - std::vector d_nnz; - std::vector o_nnz; - auto n_square = (*options)["stencil:square"] - .doc("Extent of stencil (square)") - .withDefault(0); - auto n_taxi = (*options)["stencil:taxi"] - .doc("Extent of stencil (taxi-cab norm)") - .withDefault(0); - auto n_cross = (*options)["stencil:cross"] - .doc("Extent of stencil (cross)") - .withDefault(0); - // Set n_taxi 2 if nothing else is set - // Probably a better way to do this - if (n_square == 0 && n_taxi == 0 && n_cross == 0) { - output_info.write("Setting solver:stencil:taxi = 2\n"); - n_taxi = 2; - } - - auto const xy_offsets = ColoringStencil::getOffsets(n_square, n_taxi, n_cross); - { - // This is ugly but can't think of a better and robust way to - // count the non-zeros for some arbitrary stencil - // effectively the same loop as the one that sets the non-zeros below - std::vector> d_nnz_map2d(nlocal); - std::vector> o_nnz_map2d(nlocal); - std::vector> d_nnz_map3d(nlocal); - std::vector> o_nnz_map3d(nlocal); - // Loop over every element in 2D to count the *unique* non-zeros - for (int x = mesh->xstart; x <= mesh->xend; x++) { - for (int y = mesh->ystart; y <= mesh->yend; y++) { - - const int ind0 = ROUND(index(x, y, 0)) - Istart; - - // 2D fields - for (int i = 0; i < n2d; i++) { - const PetscInt row = ind0 + i; - // Loop through each point in the stencil - for (const auto& [x_off, y_off] : xy_offsets) { - const int xi = x + x_off; - const int yi = y + y_off; - if ((xi < 0) || (yi < 0) || (xi >= mesh->LocalNx) - || (yi >= mesh->LocalNy)) { - continue; - } - - const int ind2 = ROUND(index(xi, yi, 0)); - if (ind2 < 0) { - continue; // A boundary point - } - - // Depends on all variables on this cell - for (int j = 0; j < n2d; j++) { - const PetscInt col = ind2 + j; - if (col >= Istart && col < Iend) { - d_nnz_map2d[row].insert(col); - } else { - o_nnz_map2d[row].insert(col); - } - } - } - } - // 3D fields - for (int z = 0; z < mesh->LocalNz; z++) { - const int ind = ROUND(index(x, y, z)) - Istart; - - for (int i = 0; i < n3d; i++) { - PetscInt row = ind + i; - if (z == 0) { - row += n2d; - } - - // Depends on 2D fields - for (int j = 0; j < n2d; j++) { - const PetscInt col = ind0 + j; - if (col >= Istart && col < Iend) { - d_nnz_map2d[row].insert(col); - } else { - o_nnz_map2d[row].insert(col); - } - } - - // Star pattern - for (const auto& [x_off, y_off] : xy_offsets) { - const int xi = x + x_off; - const int yi = y + y_off; - - if ((xi < 0) || (yi < 0) || (xi >= mesh->LocalNx) - || (yi >= mesh->LocalNy)) { - continue; - } - - int ind2 = ROUND(index(xi, yi, 0)); - if (ind2 < 0) { - continue; // Boundary point - } - - if (z == 0) { - ind2 += n2d; - } - - // 3D fields on this cell - for (int j = 0; j < n3d; j++) { - const PetscInt col = ind2 + j; - if (col >= Istart && col < Iend) { - d_nnz_map3d[row].insert(col); - } else { - o_nnz_map3d[row].insert(col); - } - } - } - } - } - } - } - - d_nnz.reserve(nlocal); - d_nnz.reserve(nlocal); - - for (int i = 0; i < nlocal; ++i) { - // Assume all elements in the z direction are potentially coupled - d_nnz.emplace_back(d_nnz_map3d[i].size() * mesh->LocalNz - + d_nnz_map2d[i].size()); - o_nnz.emplace_back(o_nnz_map3d[i].size() * mesh->LocalNz - + o_nnz_map2d[i].size()); - } - } - - output_progress.write("Pre-allocating Jacobian\n"); - // Pre-allocate - MatMPIAIJSetPreallocation(Jfd, 0, d_nnz.data(), 0, o_nnz.data()); - MatSeqAIJSetPreallocation(Jfd, 0, d_nnz.data()); - MatSetUp(Jfd); - MatSetOption(Jfd, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE); - - ////////////////////////////////////////////////// - // Mark non-zero entries - - output_progress.write("Marking non-zero Jacobian entries\n"); - PetscScalar val = 1.0; - for (int x = mesh->xstart; x <= mesh->xend; x++) { - for (int y = mesh->ystart; y <= mesh->yend; y++) { - - const int ind0 = ROUND(index(x, y, 0)); - - // 2D fields - for (int i = 0; i < n2d; i++) { - const PetscInt row = ind0 + i; - - // Loop through each point in the stencil - for (const auto& [x_off, y_off] : xy_offsets) { - const int xi = x + x_off; - const int yi = y + y_off; - if ((xi < 0) || (yi < 0) || (xi >= mesh->LocalNx) - || (yi >= mesh->LocalNy)) { - continue; - } - - int ind2 = ROUND(index(xi, yi, 0)); - if (ind2 < 0) { - continue; // A boundary point - } - - // Depends on all variables on this cell - for (int j = 0; j < n2d; j++) { - PetscInt col = ind2 + j; - ierr = MatSetValues(Jfd, 1, &row, 1, &col, &val, INSERT_VALUES); - CHKERRQ(ierr); - } - } - } - // 3D fields - for (int z = 0; z < mesh->LocalNz; z++) { - int ind = ROUND(index(x, y, z)); - - for (int i = 0; i < n3d; i++) { - PetscInt row = ind + i; - if (z == 0) { - row += n2d; - } - - // Depends on 2D fields - for (int j = 0; j < n2d; j++) { - PetscInt col = ind0 + j; - ierr = MatSetValues(Jfd, 1, &row, 1, &col, &val, INSERT_VALUES); - CHKERRQ(ierr); - } - - // Star pattern - for (const auto& [x_off, y_off] : xy_offsets) { - int xi = x + x_off; - int yi = y + y_off; - - if ((xi < 0) || (yi < 0) || (xi >= mesh->LocalNx) - || (yi >= mesh->LocalNy)) { - continue; - } - for (int zi = 0; zi < mesh->LocalNz; ++zi) { - int ind2 = ROUND(index(xi, yi, zi)); - if (ind2 < 0) { - continue; // Boundary point - } - - if (z == 0) { - ind2 += n2d; - } - - // 3D fields on this cell - for (int j = 0; j < n3d; j++) { - PetscInt col = ind2 + j; - ierr = MatSetValues(Jfd, 1, &row, 1, &col, &val, INSERT_VALUES); - - if (ierr != 0) { - output.write("ERROR: {} {} : ({}, {}) -> ({}, {}) : {} -> {}\n", - row, x, y, xi, yi, ind2, ind2 + n3d - 1); - } - CHKERRQ(ierr); - } - } - } - } - } - } - } - - // Finished marking non-zero entries - - output_progress.write("Assembling Jacobian matrix\n"); + PetscCall(VecDuplicate(snes_x, &output_x)); - // Assemble Matrix - MatAssemblyBegin(Jfd, MAT_FINAL_ASSEMBLY); - MatAssemblyEnd(Jfd, MAT_FINAL_ASSEMBLY); - - { - // Test if the matrix is symmetric - // Values are 0 or 1 so tolerance (1e-5) shouldn't matter - PetscBool symmetric; - ierr = MatIsSymmetric(Jfd, 1e-5, &symmetric); - CHKERRQ(ierr); - if (!symmetric) { - output_warn.write("Jacobian pattern is not symmetric\n"); - } - } - - // The above can miss entries around the X-point branch cut: - // The diagonal terms are complicated because moving in X then Y - // is different from moving in Y then X at the X-point. - // Making sure the colouring matrix is symmetric does not - // necessarily give the correct stencil but may help. - if ((*options)["force_symmetric_coloring"] - .doc("Modifies coloring matrix to force it to be symmetric") - .withDefault(false)) { - Mat Jfd_T; - MatCreateTranspose(Jfd, &Jfd_T); - MatAXPY(Jfd, 1, Jfd_T, DIFFERENT_NONZERO_PATTERN); - } - - output_progress.write("Creating Jacobian coloring\n"); - updateColoring(); - - if (prune_jacobian) { - // Will remove small elements from the Jacobian. - // Save a copy to recover from over-pruning - ierr = MatDuplicate(Jfd, MAT_SHARE_NONZERO_PATTERN, &Jfd_original); - CHKERRQ(ierr); - } - } else { - // Brute force calculation - // There is usually no reason to use this, except as a check of - // the coloring calculation. - - MatCreateAIJ( - BoutComm::get(), nlocal, nlocal, // Local sizes - PETSC_DETERMINE, PETSC_DETERMINE, // Global sizes - 3, // Number of nonzero entries in diagonal portion of local submatrix - nullptr, - 0, // Number of nonzeros per row in off-diagonal portion of local submatrix - nullptr, &Jfd); - - if (matrix_free_operator) { - SNESSetJacobian(snes, Jmf, Jfd, SNESComputeJacobianDefault, this); - } else { - SNESSetJacobian(snes, Jfd, Jfd, SNESComputeJacobianDefault, this); - } - - MatSetOption(Jfd, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE); - } + // Initialize the Finite Difference Jacobian + PetscCall(FDJinitialise()); // Re-use Jacobian // Note: If the 'Amat' Jacobian is matrix free, SNESComputeJacobian // always updates its reference 'u' vector every nonlinear iteration SNESSetLagJacobian(snes, lag_jacobian); - if (pid_controller) { - nl_its_prev = target_its; - nl_its_prev2 = target_its; - SNESSetLagJacobianPersists(snes, PETSC_FALSE); - SNESSetLagPreconditionerPersists(snes, PETSC_FALSE); - } else { - // Set Jacobian and preconditioner to persist across time steps - SNESSetLagJacobianPersists(snes, PETSC_TRUE); - SNESSetLagPreconditionerPersists(snes, PETSC_TRUE); - } - SNESSetLagPreconditioner(snes, 1); // Rebuild when Jacobian is rebuilt + nl_its_prev = target_its; + nl_its_prev2 = target_its; + PetscCall( + SNESSetLagJacobianPersists(snes, static_cast(jacobian_persists))); + PetscCall(SNESSetLagPreconditionerPersists( + snes, static_cast(jacobian_persists))); + PetscCall(SNESSetLagPreconditioner(snes, 1)); // Rebuild when Jacobian is rebuilt } // Set tolerances @@ -743,10 +835,10 @@ int SNESSolver::init() { SNESType snestype; SNESGetType(snes, &snestype); output_info.write("SNES Type : {}\n", snestype); - if (ksptype) { + if (ksptype != nullptr) { output_info.write("KSP Type : {}\n", ksptype); } - if (pctype) { + if (pctype != nullptr) { output_info.write("PC Type : {}\n", pctype); } } @@ -755,30 +847,51 @@ int SNESSolver::init() { } int SNESSolver::run() { - TRACE("SNESSolver::run()"); - int ierr; // Set initial guess at the solution from variables { BoutReal* xdata = nullptr; - int ierr = VecGetArray(snes_x, &xdata); - CHKERRQ(ierr); + PetscCall(VecGetArray(snes_x, &xdata)); save_vars(xdata); - ierr = VecRestoreArray(snes_x, &xdata); - CHKERRQ(ierr); + + if (asinh_vars) { + // Evolving asinh(vars) + PetscInt size; + PetscCall(VecGetLocalSize(snes_x, &size)); + for (PetscInt i = 0; i != size; ++i) { + xdata[i] = std::asinh(xdata[i] / asinh_scale); + } + } + + PetscCall(VecRestoreArray(snes_x, &xdata)); + } + + // Initialise residuals + local_residual = 0.0; + local_residual_2d = 0.0; + global_residual = 0.0; + PetscCall(updateResiduals(snes_x)); + if (diagnose) { + output.write("\n Residual: {}\n", global_residual); } BoutReal target = simtime; + recent_failure_rate = 0.0; for (int s = 0; s < getNumberOutputSteps(); s++) { target += getOutputTimestep(); bool looping = true; int snes_failures = 0; // Count SNES convergence failures - int steps_since_snes_failure = 0; int saved_jacobian_lag = 0; int loop_count = 0; + + BoutReal start_global_residual = global_residual; do { - if (simtime >= target) + if ((output_trigger == BoutSnesOutput::fixed_time_interval && (simtime >= target)) + || (output_trigger == BoutSnesOutput::residual_ratio + && (global_residual <= start_global_residual * output_residual_ratio))) { break; // Could happen if step over multiple outputs + } + if (scale_vars) { // Individual variable scaling // Note: If variables are rescaled then the Jacobian columns @@ -792,14 +905,11 @@ int SNESSolver::run() { // Take ownership of snes_x and var_scaling_factors data PetscScalar* snes_x_data = nullptr; - ierr = VecGetArray(snes_x, &snes_x_data); - CHKERRQ(ierr); + PetscCall(VecGetArray(snes_x, &snes_x_data)); PetscScalar* x1_data; - ierr = VecGetArray(x1, &x1_data); - CHKERRQ(ierr); + PetscCall(VecGetArray(x1, &x1_data)); PetscScalar* var_scaling_factors_data; - ierr = VecGetArray(var_scaling_factors, &var_scaling_factors_data); - CHKERRQ(ierr); + PetscCall(VecGetArray(var_scaling_factors, &var_scaling_factors_data)); // Normalise each value in the state // Limit normalisation so scaling factor is never smaller than rtol @@ -812,12 +922,9 @@ int SNESSolver::run() { } // Restore vector underlying data - ierr = VecRestoreArray(var_scaling_factors, &var_scaling_factors_data); - CHKERRQ(ierr); - ierr = VecRestoreArray(x1, &x1_data); - CHKERRQ(ierr); - ierr = VecRestoreArray(snes_x, &snes_x_data); - CHKERRQ(ierr); + PetscCall(VecRestoreArray(var_scaling_factors, &var_scaling_factors_data)); + PetscCall(VecRestoreArray(x1, &x1_data)); + PetscCall(VecRestoreArray(snes_x, &snes_x_data)); if (diagnose) { // Print maximum and minimum scaling factors @@ -837,45 +944,59 @@ int SNESSolver::run() { // Copy the state (snes_x) into initial values (x0) VecCopy(snes_x, x0); - if (timestep < dt_min_reset) { - // Hit the minimum timestep, probably through repeated failures + if (equation_form == BoutSnesEquationForm::pseudo_transient) { + // Pseudo-Transient Continuation + // Each evolving quantity may have its own timestep + // Set timestep and dt scalars to the minimum of dt_vec - if (saved_jacobian_lag != 0) { - // Already tried this and it didn't work - throw BoutException("Solver failed after many attempts"); + PetscCall(VecMin(dt_vec, nullptr, ×tep)); + dt = timestep; + + if (output_trigger == BoutSnesOutput::fixed_time_interval + && simtime + timestep >= target) { + looping = false; } + } else { + // Timestepping - // Try resetting the preconditioner, turn off predictor, and use a large timestep - SNESGetLagJacobian(snes, &saved_jacobian_lag); - SNESSetLagJacobian(snes, 1); - timestep = getOutputTimestep(); - predictor = false; // Predictor can cause problems in near steady-state. - } + if (timestep < dt_min_reset) { + // Hit the minimum timestep, probably through repeated failures + + if (saved_jacobian_lag != 0) { + // Already tried this and it didn't work + throw BoutException("Solver failed after many attempts"); + } - // Set the timestep - dt = timestep; - looping = true; - if (simtime + dt >= target) { - // Note: When the timestep is changed the preconditioner needs to be updated - // => Step over the output time and interpolate if not matrix free - - if (matrix_free) { - // Ensure that the timestep goes to the next output time and then stops. - // This avoids the need to interpolate - dt = target - simtime; + // Try resetting the preconditioner, turn off predictor, and use a large timestep + SNESGetLagJacobian(snes, &saved_jacobian_lag); + SNESSetLagJacobian(snes, 1); + timestep = getOutputTimestep(); + predictor = false; // Predictor can cause problems in near steady-state. } - looping = false; - } - if (predictor and (time1 > 0.0)) { - // Use (time1, x1) and (simtime, x0) to make prediction - // snes_x <- x0 + (dt / (simtime - time1)) * (x0 - x1) - // snes_x <- -β * x1 + (1 + β) * snes_x - BoutReal beta = dt / (simtime - time1); - VecAXPBY(snes_x, -beta, (1. + beta), x1); - } + // Set the timestep + dt = timestep; + if (output_trigger == BoutSnesOutput::fixed_time_interval + && simtime + dt >= target) { + // Note: When the timestep is changed the preconditioner needs to be updated + // => Step over the output time and interpolate if not matrix free + + if (matrix_free) { + // Ensure that the timestep goes to the next output time and then stops. + // This avoids the need to interpolate + dt = target - simtime; + } + looping = false; + } + + if (predictor and (time1 > 0.0)) { + // Use (time1, x1) and (simtime, x0) to make prediction + // snes_x <- x0 + (dt / (simtime - time1)) * (x0 - x1) + // snes_x <- -β * x1 + (1 + β) * snes_x + const BoutReal beta = dt / (simtime - time1); + VecAXPBY(snes_x, -beta, (1. + beta), x1); + } - if (pid_controller) { SNESSetLagJacobian(snes, lag_jacobian); } @@ -892,10 +1013,17 @@ int SNESSolver::run() { int lin_its; SNESGetLinearSolveIterations(snes, &lin_its); - if ((ierr != 0) or (reason < 0)) { + // Rolling average of recent failures + recent_failure_rate *= 1. - last_failure_weight; + + if ((ierr != PETSC_SUCCESS) or (reason < 0)) { // Diverged or SNES failed - if (diagnose_failures) { + recent_failure_rate += last_failure_weight; + + ++snes_failures; + + if (diagnose_failures or (snes_failures == max_snes_failures)) { // Print diagnostics to help identify source of the problem output.write("\n======== SNES failed =========\n"); @@ -914,11 +1042,29 @@ int SNESSolver::run() { } } - ++snes_failures; - steps_since_snes_failure = 0; + if (snes_failures == max_snes_failures) { + output.write("Too many SNES failures ({}). Aborting.", snes_failures); + return 1; + } - // Try a smaller timestep - timestep *= timestep_factor_on_failure; + if (equation_form == BoutSnesEquationForm::pseudo_transient) { + if (snes_failures == max_snes_failures - 1) { + // Last chance. Set to uniform smallest timestep + PetscCall(VecSet(dt_vec, dt_min_reset)); + + } else if (snes_failures == 5) { + // Set uniform timestep + PetscCall(VecSet(dt_vec, timestep)); + } else { + // Global scaling of timesteps + // Note: A better strategy might be to reduce timesteps + // in problematic cells. + PetscCall(VecScale(dt_vec, timestep_factor_on_failure)); + } + } else { + // Try a smaller timestep + timestep *= timestep_factor_on_failure; + } // Restore state VecCopy(x0, snes_x); @@ -926,15 +1072,10 @@ int SNESSolver::run() { if (jacobian_pruned and (snes_failures > 2) and (4 * lin_its > 3 * maxl)) { // Taking 3/4 of maximum linear iterations on average per linear step // May indicate a preconditioner problem. - // Restore pruned non-zero elements if (diagnose) { output.write("\nRestoring Jacobian\n"); } - ierr = MatCopy(Jfd_original, Jfd, DIFFERENT_NONZERO_PATTERN); - CHKERRQ(ierr); - // The non-zero pattern has changed, so update coloring - updateColoring(); - jacobian_pruned = false; // Reset flag. Will be set after pruning. + PetscCall(FDJrestoreFromPruning()); } if (saved_jacobian_lag == 0) { @@ -972,37 +1113,40 @@ int SNESSolver::run() { if (nl_its == 0) { // This can occur even with SNESSetForceIteration - // Results in simulation state freezing and rapidly going to the end + // Results in simulation state freezing and rapidly going + // to the end if (scale_vars) { // scaled_x <- snes_x * var_scaling_factors - ierr = VecPointwiseMult(scaled_x, snes_x, var_scaling_factors); - CHKERRQ(ierr); + PetscCall(VecPointwiseMult(scaled_x, snes_x, var_scaling_factors)); const BoutReal* xdata = nullptr; - ierr = VecGetArrayRead(scaled_x, &xdata); - CHKERRQ(ierr); + PetscCall(VecGetArrayRead(scaled_x, &xdata)); load_vars(const_cast(xdata)); - ierr = VecRestoreArrayRead(scaled_x, &xdata); - CHKERRQ(ierr); + PetscCall(VecRestoreArrayRead(scaled_x, &xdata)); } else { const BoutReal* xdata = nullptr; - ierr = VecGetArrayRead(snes_x, &xdata); - CHKERRQ(ierr); + PetscCall(VecGetArrayRead(snes_x, &xdata)); load_vars(const_cast(xdata)); - ierr = VecRestoreArrayRead(snes_x, &xdata); - CHKERRQ(ierr); + PetscCall(VecRestoreArrayRead(snes_x, &xdata)); + } + + try { + run_rhs(simtime); + } catch (BoutException& e) { + output_error.write("ERROR: BoutException thrown: {}\n", e.what()); + // Abort simulation. There is no way to recover and + // synchronise unless all processors throw exceptions + // together + BoutComm::abort(1); } - run_rhs(simtime); // Copy derivatives back { BoutReal* fdata = nullptr; - ierr = VecGetArray(snes_f, &fdata); - CHKERRQ(ierr); + PetscCall(VecGetArray(snes_f, &fdata)); save_derivs(fdata); - ierr = VecRestoreArray(snes_f, &fdata); - CHKERRQ(ierr); + PetscCall(VecRestoreArray(snes_f, &fdata)); } // Forward Euler @@ -1010,7 +1154,9 @@ int SNESSolver::run() { } simtime += dt; - ++steps_since_snes_failure; + + // Update local and global residuals + PetscCall(updateResiduals(snes_x)); if (diagnose) { // Gather and print diagnostic information @@ -1022,112 +1168,44 @@ int SNESSolver::run() { output.write(", SNES failures: {}", snes_failures); } output.write("\n"); + // Print residual on separate line, so post-processing isn't affected + output.write(" Residual: {}\n", global_residual); } -#if PETSC_VERSION_GE(3, 20, 0) // MatFilter and MatEliminateZeros(Mat, bool) require PETSc >= 3.20 if (jacobian_recalculated and prune_jacobian) { jacobian_recalculated = false; // Reset flag - // Remove small elements from the Jacobian and recompute the coloring - // Only do this if there are a significant number of small elements. - int small_elements = 0; - int total_elements = 0; - - // Get index of rows owned by this processor - int rstart, rend; - MatGetOwnershipRange(Jfd, &rstart, &rend); - - PetscInt ncols; - const PetscScalar* vals; - for (int row = rstart; row < rend; row++) { - MatGetRow(Jfd, row, &ncols, nullptr, &vals); - for (int col = 0; col < ncols; col++) { - if (std::abs(vals[col]) < prune_abstol) { - ++small_elements; - } - ++total_elements; - } - MatRestoreRow(Jfd, row, &ncols, nullptr, &vals); - } - - if (small_elements > prune_fraction * total_elements) { - if (diagnose) { - output.write("\nPruning Jacobian elements: {} / {}\n", small_elements, - total_elements); - } + FDJpruneJacobian(); + } - // Prune Jacobian, keeping diagonal elements - ierr = MatFilter(Jfd, prune_abstol, PETSC_TRUE, PETSC_TRUE); + if (equation_form == BoutSnesEquationForm::pseudo_transient) { + // Adjust pseudo_alpha to globally scale timesteps + pseudo_alpha = updateGlobalTimestep(pseudo_alpha, nl_its, recent_failure_rate, + max_timestep * atol * 100); - // Update the coloring from Jfd matrix - updateColoring(); + // Adjust local timesteps + PetscCall(updatePseudoTimestepping()); - // Mark the Jacobian as pruned. This is so that it is only restored if pruned. - jacobian_pruned = true; - } + } else { + // Adjust timestep + timestep = + updateGlobalTimestep(timestep, nl_its, recent_failure_rate, max_timestep); } -#endif // PETSC_VERSION_GE(3,20,0) - if (pid_controller) { - // Changing the timestep. - // Note: The preconditioner depends on the timestep, - // so we recalculate the jacobian and the preconditioner - // every time the timestep changes - - timestep = pid(timestep, nl_its); - - // NOTE(malamast): Do we really need this? - // Recompute Jacobian (for now) + if (static_cast(lin_its) / nl_its > 0.5 * maxl) { + // Recompute Jacobian if number of linear iterations is too high if (saved_jacobian_lag == 0) { SNESGetLagJacobian(snes, &saved_jacobian_lag); SNESSetLagJacobian(snes, 1); } - - } else { - - // Consider changing the timestep. - // Note: The preconditioner depends on the timestep, - // so if it is not recalculated the it will be less - // effective. - if ((nl_its <= lower_its) && (timestep < max_timestep) - && (steps_since_snes_failure > 2)) { - // Increase timestep slightly - timestep *= timestep_factor_on_lower_its; - - timestep = std::min(timestep, max_timestep); - - // Note: Setting the SNESJacobianFn to NULL retains - // previously set evaluation function. - // - // The SNES Jacobian is a combination of the RHS Jacobian - // and a factor involving the timestep. - // Depends on equation_form - // -> Probably call SNESSetJacobian(snes, Jfd, Jfd, NULL, fdcoloring); - - if (static_cast(lin_its) / nl_its > 4) { - // Recompute Jacobian (for now) - if (saved_jacobian_lag == 0) { - SNESGetLagJacobian(snes, &saved_jacobian_lag); - SNESSetLagJacobian(snes, 1); - } - } - - } else if (nl_its >= upper_its) { - // Reduce timestep slightly - timestep *= timestep_factor_on_upper_its; - - // Recompute Jacobian - if (saved_jacobian_lag == 0) { - SNESGetLagJacobian(snes, &saved_jacobian_lag); - SNESSetLagJacobian(snes, 1); - } - } } + snes_failures = 0; } while (looping); - if (!matrix_free) { + BoutReal output_time = simtime; + if (output_trigger == BoutSnesOutput::fixed_time_interval && !matrix_free) { ASSERT2(simtime >= target); ASSERT2(simtime - dt <= target); // Stepped over output timestep => Interpolate @@ -1142,6 +1220,7 @@ int SNESSolver::run() { // output_x <- alpha * x0 + (1 - alpha) * output_x VecAXPBY(output_x, alpha, 1. - alpha, x0); + output_time = target; } else { // Timestep was adjusted to hit target output time @@ -1151,26 +1230,40 @@ int SNESSolver::run() { // Put the result into variables if (scale_vars) { // scaled_x <- output_x * var_scaling_factors - int ierr = VecPointwiseMult(scaled_x, output_x, var_scaling_factors); - CHKERRQ(ierr); - - const BoutReal* xdata = nullptr; - ierr = VecGetArrayRead(scaled_x, &xdata); - CHKERRQ(ierr); - load_vars(const_cast(xdata)); - ierr = VecRestoreArrayRead(scaled_x, &xdata); - CHKERRQ(ierr); + PetscCall(VecPointwiseMult(scaled_x, output_x, var_scaling_factors)); + } else if (asinh_vars) { + PetscCall(VecCopy(output_x, scaled_x)); } else { - const BoutReal* xdata = nullptr; - int ierr = VecGetArrayRead(output_x, &xdata); - CHKERRQ(ierr); - load_vars(const_cast(xdata)); - ierr = VecRestoreArrayRead(output_x, &xdata); - CHKERRQ(ierr); + scaled_x = output_x; + } + + if (asinh_vars) { + PetscInt size; + PetscCall(VecGetLocalSize(scaled_x, &size)); + + BoutReal* scaled_data = nullptr; + PetscCall(VecGetArray(scaled_x, &scaled_data)); + for (PetscInt i = 0; i != size; ++i) { + scaled_data[i] = asinh_scale * std::sinh(scaled_data[i]); + } + PetscCall(VecRestoreArray(scaled_x, &scaled_data)); + } + + const BoutReal* xdata = nullptr; + PetscCall(VecGetArrayRead(scaled_x, &xdata)); + load_vars(const_cast(xdata)); + PetscCall(VecRestoreArrayRead(scaled_x, &xdata)); + + try { + run_rhs(output_time); // Run RHS to calculate auxilliary variables + } catch (BoutException& e) { + output_error.write("ERROR: BoutException thrown: {}\n", e.what()); + // Abort simulation. There is no way to recover unless + // all processors throw an exception at the same point. + BoutComm::abort(1); } - run_rhs(target); // Run RHS to calculate auxilliary variables - if (call_monitors(target, s, getNumberOutputSteps()) != 0) { + if (call_monitors(output_time, s, getNumberOutputSteps()) != 0) { break; // User signalled to quit } } @@ -1178,31 +1271,358 @@ int SNESSolver::run() { return 0; } -// f = rhs -PetscErrorCode SNESSolver::snes_function(Vec x, Vec f, bool linear) { +BoutReal SNESSolver::updateGlobalTimestep(BoutReal timestep, int nl_its, + BoutReal recent_failure_rate, BoutReal max_dt) { + // Note: The preconditioner depends on the timestep, + // so if it is not recalculated the it will be less + // effective. + + switch (timestep_control) { + case BoutSnesTimestep::pid_nonlinear_its: + // Changing the timestep using a PID controller. + return pid(timestep, nl_its, max_dt); + + case BoutSnesTimestep::threshold_nonlinear_its: + // Consider changing the timestep, based on thresholds in NL iterations + if ((nl_its <= lower_its) && (timestep < max_timestep) + && (recent_failure_rate < 0.5)) { + // Increase timestep slightly + timestep *= timestep_factor_on_lower_its; + return std::min(timestep, max_timestep); + } + + if (nl_its >= upper_its) { + // Reduce timestep slightly + return timestep * timestep_factor_on_upper_its; + } + return timestep; // No change + + case BoutSnesTimestep::residual_ratio: + // Use ratio of previous and current global residual + // Intended to be the same as https://petsc.org/release/manualpages/TS/TSPSEUDO/ + // (Note the PETSc manual has the expression for dt_n upside down) + + return std::min({timestep_factor * timestep * global_residual_prev / global_residual, + max_timestep}); + + case BoutSnesTimestep::fixed: + break; + } + return timestep; // No change +} + +PetscErrorCode SNESSolver::updateResiduals(Vec x) { + // Push residuals to previous residuals + // Copying so that data is not shared + local_residual_prev = copy(local_residual); + local_residual_2d_prev = copy(local_residual_2d); + global_residual_prev = global_residual; + + // Call RHS function to get time derivatives + PetscCall(rhs_function(x, snes_f, false)); + + // Reading the residual vectors + const BoutReal* current_residual = nullptr; + PetscCall(VecGetArrayRead(snes_f, ¤t_residual)); + + // Note: The ordering of quantities in the PETSc vectors + // depends on the Solver::loop_vars function + Mesh* mesh = bout::globals::mesh; + int idx = 0; // Index into PETSc Vecs + + // Boundary cells + for (const auto& i2d : mesh->getRegion2D("RGN_BNDRY")) { + // Field2D quantities evolved together + BoutReal residual = 0.0; + int count = 0; + + for (const auto& f : f2d) { + if (!f.evolve_bndry) { + continue; // Not evolving boundary => Skip + } + residual += SQ(current_residual[idx++]); + ++count; + } + if (count > 0) { + local_residual_2d[i2d] = sqrt(residual / count); + } + + // Field3D quantities evolved together within a cell + for (int jz = mesh->zstart; jz <= mesh->zend; jz++) { + count = 0; + residual = 0.0; + for (const auto& f : f3d) { + if (!f.evolve_bndry) { + continue; // Not evolving boundary => Skip + } + residual += SQ(current_residual[idx++]); + ++count; + } + if (count > 0) { + auto i3d = mesh->ind2Dto3D(i2d, jz); + local_residual[i3d] = sqrt(residual / count); + } + } + } + + // Bulk of domain. + // These loops don't check the boundary flags + for (const auto& i2d : mesh->getRegion2D("RGN_NOBNDRY")) { + // Field2D quantities evolved together + if (!f2d.empty()) { + BoutReal residual = 0.0; + for (std::size_t i = 0; i != f2d.size(); ++i) { + residual += SQ(current_residual[idx++]); + } + local_residual_2d[i2d] = sqrt(residual / static_cast(f2d.size())); + } + + // Field3D quantities evolved together within a cell + if (!f3d.empty()) { + for (int jz = mesh->zstart; jz <= mesh->zend; jz++) { + auto i3d = mesh->ind2Dto3D(i2d, jz); + + BoutReal residual = 0.0; + for (std::size_t i = 0; i != f3d.size(); ++i) { + residual += SQ(current_residual[idx++]); + } + local_residual[i3d] = sqrt(residual / static_cast(f3d.size())); + } + } + } + + // Restore Vec data arrays + PetscCall(VecRestoreArrayRead(snes_f, ¤t_residual)); + + // Global residual metric (RMS) + global_residual = std::sqrt(mean(SQ(local_residual), true)); + + return PETSC_SUCCESS; +} + +PetscErrorCode SNESSolver::initPseudoTimestepping() { + // Storage for per-variable timestep + PetscCall(VecDuplicate(snes_x, &dt_vec)); + // Starting timestep + PetscCall(VecSet(dt_vec, timestep)); + + // Diagnostic outputs + pseudo_timestep = timestep; + + return PETSC_SUCCESS; +} + +PetscErrorCode SNESSolver::updatePseudoTimestepping() { + // Use a per-cell timestep so that e.g density and pressure + // evolve in a way consistent with the equation of state. + + // Modifying the dt_vec values + BoutReal* dt_data = nullptr; + PetscCall(VecGetArray(dt_vec, &dt_data)); + + // Note: The ordering of quantities in the PETSc vectors + // depends on the Solver::loop_vars function + Mesh* mesh = bout::globals::mesh; + int idx = 0; // Index into PETSc Vecs + + // Boundary cells + for (const auto& i2d : mesh->getRegion2D("RGN_BNDRY")) { + // Field2D quantities evolved together + int count = 0; + for (const auto& f : f2d) { + if (!f.evolve_bndry) { + continue; // Not evolving boundary => Skip + } + ++count; + } + if (count > 0) { + // Adjust timestep for these quantities + BoutReal new_timestep = updatePseudoTimestep( + dt_data[idx], local_residual_2d_prev[i2d], local_residual_2d[i2d]); + for (int i = 0; i != count; ++i) { + dt_data[idx++] = new_timestep; + } + } + + // Field3D quantities evolved together within a cell + for (int jz = mesh->zstart; jz <= mesh->zend; jz++) { + count = 0; + for (const auto& f : f3d) { + if (!f.evolve_bndry) { + continue; // Not evolving boundary => Skip + } + ++count; + } + if (count > 0) { + auto i3d = mesh->ind2Dto3D(i2d, jz); + const BoutReal new_timestep = updatePseudoTimestep( + dt_data[idx], local_residual_prev[i3d], local_residual[i3d]); + + for (int i = 0; i != count; ++i) { + dt_data[idx++] = new_timestep; + } + } + } + } + + // Bulk of domain. + // These loops don't check the boundary flags + for (const auto& i2d : mesh->getRegion2D("RGN_NOBNDRY")) { + // Field2D quantities evolved together + if (!f2d.empty()) { + // Adjust timestep for these quantities + const BoutReal new_timestep = updatePseudoTimestep( + dt_data[idx], local_residual_2d_prev[i2d], local_residual_2d[i2d]); + for (std::size_t i = 0; i != f2d.size(); ++i) { + dt_data[idx++] = new_timestep; + } + } + + // Field3D quantities evolved together within a cell + if (!f3d.empty()) { + for (int jz = mesh->zstart; jz <= mesh->zend; jz++) { + auto i3d = mesh->ind2Dto3D(i2d, jz); + + BoutReal new_timestep = updatePseudoTimestep( + dt_data[idx], local_residual_prev[i3d], local_residual[i3d]); + + // Compare to neighbors + BoutReal min_neighboring_dt = max_timestep; + if (i3d.x() != 0) { + min_neighboring_dt = std::min(min_neighboring_dt, pseudo_timestep[i3d.xm()]); + } + if (i3d.x() != mesh->LocalNx - 1) { + min_neighboring_dt = std::min(min_neighboring_dt, pseudo_timestep[i3d.xp()]); + } + if (i3d.y() != 0) { + min_neighboring_dt = std::min(min_neighboring_dt, pseudo_timestep[i3d.ym()]); + } + if (i3d.x() != mesh->LocalNy - 1) { + min_neighboring_dt = std::min(min_neighboring_dt, pseudo_timestep[i3d.yp()]); + } + + // Limit ratio of timestep between neighboring cells + new_timestep = std::min(new_timestep, pseudo_max_ratio * min_neighboring_dt); + + pseudo_timestep[i3d] = new_timestep; + + for (std::size_t i = 0; i != f3d.size(); ++i) { + dt_data[idx++] = new_timestep; + } + } + } + } + + // Restore Vec data arrays + PetscCall(VecRestoreArray(dt_vec, &dt_data)); + + // Need timesteps on neighboring processors + // to limit ratio + mesh->communicate(pseudo_timestep); + // Neumann boundary so timestep isn't pinned at boundaries + pseudo_timestep.applyBoundary("neumann"); + + return PETSC_SUCCESS; +} + +/// Timestep inversely proportional to the residual, clipped to avoid +/// rapid changes in timestep. +BoutReal SNESSolver::updatePseudoTimestep_inverse_residual(BoutReal previous_timestep, + BoutReal current_residual) { + return std::min( + {std::max({pseudo_alpha / current_residual, previous_timestep / 1.5, dt_min_reset}), + 1.5 * previous_timestep, max_timestep}); +} + +// Strategy based on history of residuals +BoutReal SNESSolver::updatePseudoTimestep_history_based(BoutReal previous_timestep, + BoutReal previous_residual, + BoutReal current_residual) { + const BoutReal converged_threshold = 10 * atol; + const BoutReal transition_threshold = 100 * atol; + + if (current_residual < converged_threshold) { + return max_timestep; + } + + // Smoothly transition from SER updates to frozen timestep + if (current_residual < transition_threshold) { + const BoutReal reduction_ratio = current_residual / previous_residual; + + if (reduction_ratio < 0.8) { + return std::min(0.5 * (pseudo_growth_factor + 1.) * previous_timestep, + max_timestep); + } else if (reduction_ratio > 1.2) { + return std::max(0.5 * (pseudo_reduction_factor + 1) * previous_timestep, + dt_min_reset); + } + // Leave unchanged if residual changes are small + return previous_timestep; + } + + if (current_residual <= previous_residual) { + // Success + return std::min(pseudo_growth_factor * previous_timestep, max_timestep); + } + // Failed + // Consider rejecting the step + return std::max(pseudo_reduction_factor * previous_timestep, dt_min_reset); +} + +/// Switched Evolution Relaxation (SER) +BoutReal SNESSolver::updatePseudoTimestep(BoutReal previous_timestep, + BoutReal previous_residual, + BoutReal current_residual) { + switch (pseudo_strategy) { + case BoutPTCStrategy::inverse_residual: + return updatePseudoTimestep_inverse_residual(previous_timestep, current_residual); + + case BoutPTCStrategy::history_based: + return updatePseudoTimestep_history_based(previous_timestep, previous_residual, + current_residual); + + case BoutPTCStrategy::hybrid: + // A hybrid strategy may be most effective, in which the timestep is + // inversely proportional to residual initially, or when residuals are large, + // and then the method transitions to being history-based + if (current_residual > 1000. * atol) { + return updatePseudoTimestep_inverse_residual(previous_timestep, current_residual); + } + return updatePseudoTimestep_history_based(previous_timestep, previous_residual, + current_residual); + }; + throw BoutException("SNESSolver::updatePseudoTimestep invalid BoutPTCStrategy"); +} + +PetscErrorCode SNESSolver::rhs_function(Vec x, Vec f, bool linear) { // Get data from PETSc into BOUT++ fields if (scale_vars) { // scaled_x <- x * var_scaling_factors - int ierr = VecPointwiseMult(scaled_x, x, var_scaling_factors); - CHKERRQ(ierr); - - const BoutReal* xdata = nullptr; - ierr = VecGetArrayRead(scaled_x, &xdata); - CHKERRQ(ierr); - load_vars(const_cast( - xdata)); // const_cast needed due to load_vars API. Not writing to xdata. - ierr = VecRestoreArrayRead(scaled_x, &xdata); - CHKERRQ(ierr); + PetscCall(VecPointwiseMult(scaled_x, x, var_scaling_factors)); + } else if (asinh_vars) { + PetscCall(VecCopy(x, scaled_x)); } else { - const BoutReal* xdata = nullptr; - int ierr = VecGetArrayRead(x, &xdata); - CHKERRQ(ierr); - load_vars(const_cast( - xdata)); // const_cast needed due to load_vars API. Not writing to xdata. - ierr = VecRestoreArrayRead(x, &xdata); - CHKERRQ(ierr); + scaled_x = x; + } + + if (asinh_vars) { + PetscInt size; + PetscCall(VecGetLocalSize(scaled_x, &size)); + + BoutReal* scaled_data = nullptr; + PetscCall(VecGetArray(scaled_x, &scaled_data)); + for (PetscInt i = 0; i != size; ++i) { + scaled_data[i] = asinh_scale * std::sinh(scaled_data[i]); + } + PetscCall(VecRestoreArray(scaled_x, &scaled_data)); } + const BoutReal* xdata = nullptr; + PetscCall(VecGetArrayRead(scaled_x, &xdata)); + // const_cast needed due to load_vars API. Not writing to xdata. + load_vars(const_cast(xdata)); + PetscCall(VecRestoreArrayRead(scaled_x, &xdata)); + try { // Call RHS function run_rhs(simtime + dt, linear); @@ -1210,28 +1630,49 @@ PetscErrorCode SNESSolver::snes_function(Vec x, Vec f, bool linear) { // Simulation might fail, e.g. negative densities // if timestep too large output_warn.write("WARNING: BoutException thrown: {}\n", e.what()); + // Abort simulation. Unless all processors threw an exception + // at the same point, there is no way to synchronise again. + BoutComm::abort(2); + } + + // Copy derivatives back + BoutReal* fdata = nullptr; + PetscCall(VecGetArray(f, &fdata)); + save_derivs(fdata); + + if (asinh_vars) { + // Modify time-derivatives for asinh(var) using chain rule + // Evolving u = asinh(var / scale) + // + // du/dt = dvar/dt * du/dvar + // + // du/var = 1 / sqrt(var^2 + scale^2) + PetscInt size; + PetscCall(VecGetLocalSize(f, &size)); + const BoutReal* scaled_data = nullptr; + PetscCall(VecGetArrayRead(scaled_x, &scaled_data)); + for (PetscInt i = 0; i != size; ++i) { + fdata[i] /= std::sqrt(SQ(scaled_data[i]) + SQ(asinh_scale)); + } + PetscCall(VecRestoreArrayRead(scaled_x, &scaled_data)); + } + + PetscCall(VecRestoreArray(f, &fdata)); + return PETSC_SUCCESS; +} +// Result in f depends on equation_form +PetscErrorCode SNESSolver::snes_function(Vec x, Vec f, bool linear) { + + // Call the RHS function + if (rhs_function(x, f, linear) != PETSC_SUCCESS) { // Tell SNES that the input was out of domain SNESSetFunctionDomainError(snes); // Note: Returning non-zero error here leaves vectors in locked state return 0; } - // Copy derivatives back - BoutReal* fdata = nullptr; - int ierr = VecGetArray(f, &fdata); - CHKERRQ(ierr); - save_derivs(fdata); - ierr = VecRestoreArray(f, &fdata); - CHKERRQ(ierr); - switch (equation_form) { - case BoutSnesEquationForm::pseudo_transient: { - // Pseudo-transient timestepping (as in UEDGE) - // f <- f - x/Δt - VecAXPY(f, -1. / dt, x); - break; - } case BoutSnesEquationForm::rearranged_backward_euler: { // Rearranged Backward Euler // f = (x0 - x)/Δt + f @@ -1240,6 +1681,15 @@ PetscErrorCode SNESSolver::snes_function(Vec x, Vec f, bool linear) { VecAXPY(f, -1. / dt, delta_x); // f <- f - delta_x / dt break; } + case BoutSnesEquationForm::pseudo_transient: { + // Pseudo-transient timestepping. Same as Rearranged Backward Euler + // except that Δt is a vector + // f = (x0 - x)/Δt + f + VecWAXPY(delta_x, -1.0, x0, x); + VecPointwiseDivide(delta_x, delta_x, dt_vec); // delta_x /= dt + VecAXPY(f, -1., delta_x); // f <- f - delta_x + break; + } case BoutSnesEquationForm::backward_euler: { // Backward Euler // Set f = x - x0 - Δt*f @@ -1258,11 +1708,10 @@ PetscErrorCode SNESSolver::snes_function(Vec x, Vec f, bool linear) { if (scale_rhs) { // f <- f * rhs_scaling_factors - ierr = VecPointwiseMult(f, f, rhs_scaling_factors); - CHKERRQ(ierr); + PetscCall(VecPointwiseMult(f, f, rhs_scaling_factors)); } - return 0; + return PETSC_SUCCESS; } /* @@ -1274,36 +1723,28 @@ PetscErrorCode SNESSolver::precon(Vec x, Vec f) { throw BoutException("No user preconditioner"); } - int ierr; - // Get data from PETSc into BOUT++ fields Vec solution; - SNESGetSolution(snes, &solution); + PetscCall(SNESGetSolution(snes, &solution)); BoutReal* soldata; - ierr = VecGetArray(solution, &soldata); - CHKERRQ(ierr); + PetscCall(VecGetArray(solution, &soldata)); load_vars(soldata); - ierr = VecRestoreArray(solution, &soldata); - CHKERRQ(ierr); + PetscCall(VecRestoreArray(solution, &soldata)); // Load vector to be inverted into ddt() variables const BoutReal* xdata; - ierr = VecGetArrayRead(x, &xdata); - CHKERRQ(ierr); + PetscCall(VecGetArrayRead(x, &xdata)); load_derivs(const_cast(xdata)); // Note: load_derivs does not modify data - ierr = VecRestoreArrayRead(x, &xdata); - CHKERRQ(ierr); + PetscCall(VecRestoreArrayRead(x, &xdata)); // Run the preconditioner runPreconditioner(simtime + dt, dt, 0.0); // Save the solution from F_vars BoutReal* fdata; - ierr = VecGetArray(f, &fdata); - CHKERRQ(ierr); + PetscCall(VecGetArray(f, &fdata)); save_derivs(fdata); - ierr = VecRestoreArray(f, &fdata); - CHKERRQ(ierr); + PetscCall(VecRestoreArray(f, &fdata)); return 0; } @@ -1312,11 +1753,9 @@ PetscErrorCode SNESSolver::scaleJacobian(Mat Jac_new) { jacobian_recalculated = true; if (!scale_rhs) { - return 0; // Not scaling the RHS values + return PETSC_SUCCESS; // Not scaling the RHS values } - int ierr; - // Get index of rows owned by this processor int rstart, rend; MatGetOwnershipRange(Jac_new, &rstart, &rend); @@ -1331,8 +1770,7 @@ PetscErrorCode SNESSolver::scaleJacobian(Mat Jac_new) { // Calculate the norm of each row of the Jacobian PetscScalar* row_inv_norm_data; - ierr = VecGetArray(jac_row_inv_norms, &row_inv_norm_data); - CHKERRQ(ierr); + PetscCall(VecGetArray(jac_row_inv_norms, &row_inv_norm_data)); PetscInt ncols; const PetscScalar* vals; @@ -1356,12 +1794,11 @@ PetscErrorCode SNESSolver::scaleJacobian(Mat Jac_new) { MatRestoreRow(Jac_new, row, &ncols, nullptr, &vals); } - ierr = VecRestoreArray(jac_row_inv_norms, &row_inv_norm_data); - CHKERRQ(ierr); + PetscCall(VecRestoreArray(jac_row_inv_norms, &row_inv_norm_data)); // Modify the RHS scaling: factor = factor / norm - ierr = VecPointwiseMult(rhs_scaling_factors, rhs_scaling_factors, jac_row_inv_norms); - CHKERRQ(ierr); + PetscCall( + VecPointwiseMult(rhs_scaling_factors, rhs_scaling_factors, jac_row_inv_norms)); if (diagnose) { // Print maximum and minimum scaling factors @@ -1372,10 +1809,9 @@ PetscErrorCode SNESSolver::scaleJacobian(Mat Jac_new) { } // Scale the Jacobian rows by multiplying on the left by 1/norm - ierr = MatDiagonalScale(Jac_new, jac_row_inv_norms, nullptr); - CHKERRQ(ierr); + PetscCall(MatDiagonalScale(Jac_new, jac_row_inv_norms, nullptr)); - return 0; + return PETSC_SUCCESS; } /// @@ -1440,7 +1876,7 @@ void SNESSolver::updateColoring() { } } -BoutReal SNESSolver::pid(BoutReal timestep, int nl_its) { +BoutReal SNESSolver::pid(BoutReal timestep, int nl_its, BoutReal max_dt) { /* ---------- multiplicative PID factors ---------- */ const BoutReal facP = std::pow(double(target_its) / double(nl_its), kP); @@ -1449,11 +1885,16 @@ BoutReal SNESSolver::pid(BoutReal timestep, int nl_its) { / double(nl_its) / double(nl_its_prev2), kD); - // clamp groth factor to avoid huge changes - const BoutReal fac = std::clamp(facP * facI * facD, 0.2, 5.0); + // clamp growth factor to avoid huge changes + BoutReal fac = std::clamp(facP * facI * facD, 0.2, 5.0); + + if (pid_consider_failures && (fac > 1.0)) { + // Reduce aggressiveness if recent steps have failed often + fac = pow(fac, std::max(0.3, 1.0 - 2.0 * recent_failure_rate)); + } /* ---------- update timestep and history ---------- */ - const BoutReal dt_new = std::min(timestep * fac, max_timestep); + const BoutReal dt_new = std::min(timestep * fac, max_dt); nl_its_prev2 = nl_its_prev; nl_its_prev = nl_its; @@ -1461,4 +1902,25 @@ BoutReal SNESSolver::pid(BoutReal timestep, int nl_its) { return dt_new; } +void SNESSolver::outputVars(Options& output_options, bool save_repeat) { + // Call base class function + Solver::outputVars(output_options, save_repeat); + + if (!save_repeat) { + return; // Don't save diagnostics to restart files + } + + output_options["snes_local_residual"].assignRepeat(local_residual, "t", save_repeat, + "SNESSolver"); + output_options["snes_global_residual"].assignRepeat(global_residual, "t", save_repeat, + "SNESSolver"); + + if (equation_form == BoutSnesEquationForm::pseudo_transient) { + output_options["snes_pseudo_alpha"].assignRepeat(pseudo_alpha, "t", save_repeat, + "SNESSolver"); + output_options["snes_pseudo_timestep"].assignRepeat(pseudo_timestep, "t", save_repeat, + "SNESSolver"); + } +} + #endif // BOUT_HAS_PETSC diff --git a/src/solver/impls/snes/snes.hxx b/src/solver/impls/snes/snes.hxx index 31deae6f06..10ac4783e7 100644 --- a/src/solver/impls/snes/snes.hxx +++ b/src/solver/impls/snes/snes.hxx @@ -4,7 +4,7 @@ * using PETSc for the SNES interface * ************************************************************************** - * Copyright 2015-2025 BOUT++ contributors + * Copyright 2015-2026 BOUT++ contributors * * Contact: Ben Dudson, dudson2@llnl.gov * @@ -39,6 +39,8 @@ class SNESSolver; #include #include +#include +#include #include #include @@ -52,6 +54,21 @@ RegisterSolver registersolverbeuler("beuler"); BOUT_ENUM_CLASS(BoutSnesEquationForm, pseudo_transient, rearranged_backward_euler, backward_euler, direct_newton); +BOUT_ENUM_CLASS(BoutPTCStrategy, + inverse_residual, ///< dt = pseudo_alpha / residual + history_based, ///< Grow/shrink dt based on residual decrease/increase + hybrid); ///< Combine inverse_residual and history_based strategies + +BOUT_ENUM_CLASS(BoutSnesTimestep, + pid_nonlinear_its, ///< PID controller on nonlinear iterations + threshold_nonlinear_its, ///< Use thresholds on nonlinear iterations + residual_ratio, ///< Use ratio of previous and current residual + fixed); ///< Fixed timestep (no adaptation) + +BOUT_ENUM_CLASS(BoutSnesOutput, + fixed_time_interval, ///< Output at fixed time intervals + residual_ratio); ///< When the residual is reduced by a given ratio + /// Uses PETSc's SNES interface to find a steady state solution to a /// nonlinear ODE by integrating in time with Backward Euler class SNESSolver : public Solver { @@ -68,6 +85,8 @@ public: /// /// f = (x - gamma*G(x)) - rhs /// + /// The form depends on equation_form + /// /// /// @param[in] x The state vector /// @param[out] f The vector for the result f(x) @@ -88,12 +107,33 @@ public: /// finite difference approximated Jacobian. PetscErrorCode scaleJacobian(Mat B); + /// Save diagnostics to output + void outputVars(Options& output_options, bool save_repeat = true) override; + private: + PetscErrorCode FDJinitialise(); ///< Finite Difference Jacobian initialise + PetscErrorCode FDJpruneJacobian(); ///< Remove small elements from the Jacobian + PetscErrorCode FDJrestoreFromPruning(); ///< Restore Jacobian to original pattern + + /// Call the physics model RHS function + /// + /// @param[in] x The state vector. Will be scaled if scale_vars=true + /// @param[out] f The vector for the result f(x) + /// @param[in] linear Specifies that the SNES solver is in a linear (KSP) inner loop + PetscErrorCode rhs_function(Vec x, Vec f, bool linear); + + BoutSnesOutput output_trigger; ///< Sets when outputs are written + + BoutReal output_residual_ratio; ///< Trigger an output when residual falls by this ratio + BoutReal timestep; ///< Internal timestep - BoutReal dt; ///< Current timestep used in snes_function + BoutReal dt; ///< Current timestep used in snes_function. BoutReal dt_min_reset; ///< If dt falls below this, reset solve BoutReal max_timestep; ///< Maximum timestep + /// Form of the equation to solve + BoutSnesEquationForm equation_form; + std::string snes_type; BoutReal atol; ///< Absolute tolerance BoutReal rtol; ///< Relative tolerance @@ -103,22 +143,71 @@ private: int maxits; ///< Maximum nonlinear iterations int lower_its, upper_its; ///< Limits on iterations for timestep adjustment + int max_snes_failures; ///< Maximum number of consecutive SNES failures before abort. + BoutReal timestep_factor_on_failure; BoutReal timestep_factor_on_upper_its; BoutReal timestep_factor_on_lower_its; - ///< PID controller parameters - bool pid_controller; ///< Use PID controller? - int target_its; ///< Target number of nonlinear iterations for the PID controller. - ///< Use with caution! Not tested values. + // Pseudo-Transient Continuation (PTC) variables + // These are used if equation_form = pseudo_transient + BoutPTCStrategy pseudo_strategy; ///< Strategy to use when setting timesteps + BoutReal pseudo_alpha; ///< dt = alpha / residual + BoutReal pseudo_growth_factor; ///< Timestep increase 1.1 - 1.2 + BoutReal pseudo_reduction_factor; ///< Timestep decrease 0.5 + BoutReal pseudo_max_ratio; ///< Maximum timestep ratio between neighboring cells + Vec dt_vec; ///< Each quantity can have its own timestep + + /// Adjust the global timestep + BoutReal updateGlobalTimestep(BoutReal timestep, int nl_its, + BoutReal recent_failure_rate, BoutReal max_dt); + + /// Calculate per-cell and global residuals + /// given an input system state `x` + PetscErrorCode updateResiduals(Vec x); + Field3D local_residual; ///< Residual of Field3D quantities in each cell + Field2D local_residual_2d; ///< Residual of Field2D quantities in each cell + BoutReal global_residual; ///< Global residual measure + Field3D local_residual_prev; ///< Previous Field3D local residuals + Field2D local_residual_2d_prev; ///< Previous Field2D local residuals + BoutReal global_residual_prev; ///< Previous global residual + + /// Initialize the Pseudo-Transient Continuation method + PetscErrorCode initPseudoTimestepping(); + /// Update dt_vec based on residuals + PetscErrorCode updatePseudoTimestepping(); + /// Decide the next pseudo-timestep. Called by updatePseudoTimestepping + BoutReal updatePseudoTimestep(BoutReal previous_timestep, BoutReal previous_residual, + BoutReal current_residual); + BoutReal updatePseudoTimestep_inverse_residual(BoutReal previous_timestep, + BoutReal current_residual); + BoutReal updatePseudoTimestep_history_based(BoutReal previous_timestep, + BoutReal previous_residual, + BoutReal current_residual); + + Field3D pseudo_timestep; + + /// Timestep controller method + BoutSnesTimestep timestep_control; + + /// When using BoutSnesTimestep::residual_ratio + BoutReal timestep_factor; ///< Multiply timestep by this each step + + /// Target number of nonlinear iterations for the PID controller. + /// This can be non-integer to push timestep more aggressively + BoutReal target_its; + BoutReal kP; ///< (0.6 - 0.8) Proportional parameter (main response to current step) BoutReal kI; ///< (0.2 - 0.4) Integral parameter (smooths history of changes) BoutReal kD; ///< (0.1 - 0.3) Derivative (dampens oscillation - optional) + bool pid_consider_failures; ///< Reduce timestep increases if recent solves have failed + BoutReal recent_failure_rate; ///< Rolling average of recent failure rate + BoutReal last_failure_weight; ///< 1 / number of recent solves - int nl_its_prev; - int nl_its_prev2; + BoutReal nl_its_prev; + BoutReal nl_its_prev2; - BoutReal pid(BoutReal timestep, int nl_its); ///< Updates the timestep + BoutReal pid(BoutReal timestep, int nl_its, BoutReal max_dt); ///< Updates the timestep bool diagnose; ///< Output additional diagnostics bool diagnose_failures; ///< Print diagnostics on SNES failures @@ -126,9 +215,6 @@ private: int nlocal; ///< Number of variables on local processor int neq; ///< Number of variables in total - /// Form of the equation to solve - BoutSnesEquationForm equation_form; - PetscLib lib; ///< Handles initialising, finalising PETSc Vec snes_f; ///< Used by SNES to store function Vec snes_x; ///< Result of SNES @@ -157,7 +243,8 @@ private: bool matrix_free; ///< Use matrix free Jacobian bool matrix_free_operator; ///< Use matrix free Jacobian in the operator? int lag_jacobian; ///< Re-use Jacobian - bool use_coloring; ///< Use matrix coloring + bool jacobian_persists; ///< Re-use Jacobian and preconditioner across nonlinear solves + bool use_coloring; ///< Use matrix coloring bool jacobian_recalculated; ///< Flag set when Jacobian is recalculated bool prune_jacobian; ///< Remove small elements in the Jacobian? @@ -174,6 +261,9 @@ private: bool scale_vars; ///< Scale individual variables? Vec var_scaling_factors; ///< Factors to multiply variables when passing to user Vec scaled_x; ///< The values passed to the user RHS + + bool asinh_vars; ///< Evolve asinh(vars) to compress magnitudes while preserving signs + const BoutReal asinh_scale = 1e-5; // Scale below which asinh response becomes ~linear }; #else diff --git a/src/solver/impls/split-rk/split-rk.cxx b/src/solver/impls/split-rk/split-rk.cxx index cd6bd1718c..ac4d62ebe2 100644 --- a/src/solver/impls/split-rk/split-rk.cxx +++ b/src/solver/impls/split-rk/split-rk.cxx @@ -1,5 +1,21 @@ #include "split-rk.hxx" +#include "bout/array.hxx" +#include "bout/assert.hxx" +#include "bout/bout_types.hxx" +#include "bout/boutexception.hxx" +#include "bout/field2d.hxx" +#include "bout/globals.hxx" +#include "bout/openmpwrap.hxx" +#include "bout/options.hxx" +#include "bout/output.hxx" +#include "bout/solver.hxx" +#include "bout/sys/gettext.hxx" +#include "bout/utils.hxx" + +#include +#include + SplitRK::SplitRK(Options* opts) : Solver(opts), nstages((*options)["nstages"] .doc("Number of stages in RKL step. Must be > 1") @@ -35,7 +51,6 @@ SplitRK::SplitRK(Options* opts) } int SplitRK::init() { - AUTO_TRACE(); Solver::init(); output.write(_("\n\tSplit Runge-Kutta-Legendre and SSP-RK3 solver\n")); @@ -73,13 +88,12 @@ int SplitRK::init() { ASSERT0(ninternal_steps > 0); timestep = getOutputTimestep() / ninternal_steps; - output.write(_("\tUsing a timestep {:e}\n"), timestep); + output.write(_f("\tUsing a timestep {:e}\n"), timestep); return 0; } int SplitRK::run() { - AUTO_TRACE(); for (int step = 0; step < getNumberOutputSteps(); step++) { // Take an output step diff --git a/src/solver/solver.cxx b/src/solver/solver.cxx index a3f6a874f6..c6e9b464fe 100644 --- a/src/solver/solver.cxx +++ b/src/solver/solver.cxx @@ -24,23 +24,38 @@ #include "bout/array.hxx" #include "bout/assert.hxx" +#include "bout/bout_types.hxx" #include "bout/boutcomm.hxx" #include "bout/boutexception.hxx" +#include "bout/field2d.hxx" +#include "bout/field3d.hxx" #include "bout/field_factory.hxx" +#include "bout/globals.hxx" #include "bout/initialprofiles.hxx" #include "bout/interpolation.hxx" +#include "bout/monitor.hxx" #include "bout/msg_stack.hxx" +#include "bout/options.hxx" #include "bout/output.hxx" #include "bout/region.hxx" #include "bout/solver.hxx" +#include "bout/sys/gettext.hxx" #include "bout/sys/timer.hxx" #include "bout/sys/uuid.h" +#include "bout/unused.hxx" +#include "bout/utils.hxx" +#include "bout/vector2d.hxx" +#include "bout/vector3d.hxx" + +#include #include -#include #include +#include #include #include +#include +#include // Implementations: #include "impls/adams_bashforth/adams_bashforth.hxx" @@ -507,11 +522,11 @@ int Solver::solve(int nout, BoutReal timestep) { finaliseMonitorPeriods(nout, timestep); output_progress.write( - _("Solver running for {:d} outputs with output timestep of {:e}\n"), nout, + _f("Solver running for {:d} outputs with output timestep of {:e}\n"), nout, timestep); if (default_monitor_period > 1) { output_progress.write( - _("Solver running for {:d} outputs with monitor timestep of {:e}\n"), + _f("Solver running for {:d} outputs with monitor timestep of {:e}\n"), nout / default_monitor_period, timestep * default_monitor_period); } @@ -537,7 +552,7 @@ int Solver::solve(int nout, BoutReal timestep) { } time_t start_time = time(nullptr); - output_progress.write(_("\nRun started at : {:s}\n"), toString(start_time)); + output_progress.write(_f("\nRun started at : {:s}\n"), toString(start_time)); Timer timer("run"); // Start timer @@ -583,7 +598,7 @@ int Solver::solve(int nout, BoutReal timestep) { status = run(); time_t end_time = time(nullptr); - output_progress.write(_("\nRun finished at : {:s}\n"), toString(end_time)); + output_progress.write(_f("\nRun finished at : {:s}\n"), toString(end_time)); output_progress.write(_("Run time : ")); int dt = end_time - start_time; @@ -636,7 +651,7 @@ std::string Solver::createRunID() const { } std::string Solver::getRunID() const { - AUTO_TRACE(); + if (run_id == default_run_id) { throw BoutException("run_id not set!"); } @@ -644,7 +659,7 @@ std::string Solver::getRunID() const { } std::string Solver::getRunRestartFrom() const { - AUTO_TRACE(); + // Check against run_id, because this might not be a restarted run if (run_id == default_run_id) { throw BoutException("run_restart_from not set!"); @@ -664,9 +679,6 @@ void Solver::writeToModelOutputFile(const Options& options) { **************************************************************************/ int Solver::init() { - - TRACE("Solver::init()"); - if (initialised) { throw BoutException(_("ERROR: Solver is already initialised\n")); } @@ -769,7 +781,7 @@ BoutReal Solver::adjustMonitorPeriods(Monitor* new_monitor) { } if (!isMultiple(internal_timestep, new_monitor->timestep)) { - throw BoutException(_("Couldn't add Monitor: {:g} is not a multiple of {:g}!"), + throw BoutException(_f("Couldn't add Monitor: {:g} is not a multiple of {:g}!"), internal_timestep, new_monitor->timestep); } @@ -785,8 +797,8 @@ BoutReal Solver::adjustMonitorPeriods(Monitor* new_monitor) { if (initialised) { throw BoutException( - _("Solver::addMonitor: Cannot reduce timestep (from {:g} to {:g}) " - "after init is called!"), + _f("Solver::addMonitor: Cannot reduce timestep (from {:g} to {:g}) " + "after init is called!"), internal_timestep, new_monitor->timestep); } @@ -886,7 +898,7 @@ int Solver::call_monitors(BoutReal simtime, int iter, int NOUT) { monitor.monitor->call(this, simtime, iter / monitor.monitor->period, NOUT / monitor.monitor->period); if (ret != 0) { - throw BoutException(_("Monitor signalled to quit (return code {})"), ret); + throw BoutException(_f("Monitor signalled to quit (return code {})"), ret); } // Write the monitor's diagnostics to the main output file Options monitor_dump; @@ -908,7 +920,7 @@ int Solver::call_monitors(BoutReal simtime, int iter, int NOUT) { for (const auto& monitor : monitors) { monitor.monitor->cleanup(); } - output_error.write(_("Monitor signalled to quit (exception {})\n"), e.what()); + output_error.write(_f("Monitor signalled to quit (exception {})\n"), e.what()); throw; } @@ -1237,13 +1249,13 @@ void Solver::load_derivs(BoutReal* udata) { void Solver::save_vars(BoutReal* udata) { for (const auto& f : f2d) { if (!f.var->isAllocated()) { - throw BoutException(_("Variable '{:s}' not initialised"), f.name); + throw BoutException(_f("Variable '{:s}' not initialised"), f.name); } } for (const auto& f : f3d) { if (!f.var->isAllocated()) { - throw BoutException(_("Variable '{:s}' not initialised"), f.name); + throw BoutException(_f("Variable '{:s}' not initialised"), f.name); } } @@ -1286,8 +1298,8 @@ void Solver::save_derivs(BoutReal* dudata) { // Make sure 3D fields are at the correct cell location for (const auto& f : f3d) { if (f.var->getLocation() != (f.F_var)->getLocation()) { - throw BoutException(_("Time derivative at wrong location - Field is at {:s}, " - "derivative is at {:s} for field '{:s}'\n"), + throw BoutException(_f("Time derivative at wrong location - Field is at {:s}, " + "derivative is at {:s} for field '{:s}'\n"), toString(f.var->getLocation()), toString(f.F_var->getLocation()), f.name); } @@ -1491,7 +1503,7 @@ void Solver::post_rhs(BoutReal UNUSED(t)) { #if CHECK > 0 for (const auto& f : f3d) { if (!f.F_var->isAllocated()) { - throw BoutException(_("Time derivative for variable '{:s}' not set"), f.name); + throw BoutException(_f("Time derivative for variable '{:s}' not set"), f.name); } } #endif diff --git a/src/sys/bout_types.cxx b/src/sys/bout_types.cxx index 35d37f11be..b27fab7ae7 100644 --- a/src/sys/bout_types.cxx +++ b/src/sys/bout_types.cxx @@ -1,12 +1,12 @@ #include #include -#include + #include namespace { template const std::string& safeAt(const std::map& mymap, T t) { - AUTO_TRACE(); + auto found = mymap.find(t); if (found == mymap.end()) { throw BoutException("Did not find enum {:d}", static_cast(t)); @@ -16,7 +16,7 @@ const std::string& safeAt(const std::map& mymap, T t) { template const T& safeAt(const std::map& mymap, const std::string& s) { - AUTO_TRACE(); + auto found = mymap.find(s); if (found == mymap.end()) { throw BoutException("Did not find enum {:s}", s); @@ -26,7 +26,7 @@ const T& safeAt(const std::map& mymap, const std::string& s) { } // namespace std::string toString(CELL_LOC location) { - AUTO_TRACE(); + const static std::map CELL_LOCtoString = { ENUMSTR(CELL_DEFAULT), ENUMSTR(CELL_CENTRE), ENUMSTR(CELL_XLOW), ENUMSTR(CELL_YLOW), ENUMSTR(CELL_ZLOW), ENUMSTR(CELL_VSHIFT)}; @@ -35,7 +35,7 @@ std::string toString(CELL_LOC location) { } CELL_LOC CELL_LOCFromString(const std::string& location_string) { - AUTO_TRACE(); + const static std::map stringtoCELL_LOC = { STRENUM(CELL_DEFAULT), STRENUM(CELL_CENTRE), STRENUM(CELL_XLOW), STRENUM(CELL_YLOW), STRENUM(CELL_ZLOW), STRENUM(CELL_VSHIFT)}; @@ -44,7 +44,7 @@ CELL_LOC CELL_LOCFromString(const std::string& location_string) { } std::string toString(DIFF_METHOD location) { - AUTO_TRACE(); + const static std::map DIFF_METHODtoString = { {DIFF_DEFAULT, "DEFAULT"}, {DIFF_U1, "U1"}, {DIFF_U2, "U2"}, {DIFF_U3, "U3"}, {DIFF_C2, "C2"}, {DIFF_C4, "C4"}, {DIFF_S2, "S2"}, {DIFF_W2, "W2"}, @@ -54,7 +54,7 @@ std::string toString(DIFF_METHOD location) { } std::string toString(REGION region) { - AUTO_TRACE(); + const static std::map REGIONtoString = { ENUMSTR(RGN_ALL), ENUMSTR(RGN_NOBNDRY), ENUMSTR(RGN_NOX), ENUMSTR(RGN_NOY), ENUMSTR(RGN_NOZ)}; @@ -62,7 +62,7 @@ std::string toString(REGION region) { } std::string toString(DIRECTION direction) { - AUTO_TRACE(); + const static std::map DIRECTIONtoString = { {DIRECTION::X, "X"}, {DIRECTION::Y, "Y"}, @@ -116,7 +116,7 @@ bool areDirectionsCompatible(const DirectionTypes& d1, const DirectionTypes& d2) } std::string toString(STAGGER stagger) { - AUTO_TRACE(); + const static std::map STAGGERtoString = { {STAGGER::None, "No staggering"}, {STAGGER::C2L, "Centre to Low"}, @@ -126,7 +126,7 @@ std::string toString(STAGGER stagger) { } std::string toString(DERIV deriv) { - AUTO_TRACE(); + const static std::map DERIVtoString = { {DERIV::Standard, "Standard"}, {DERIV::StandardSecond, "Standard -- second order"}, @@ -138,7 +138,7 @@ std::string toString(DERIV deriv) { } std::string toString(YDirectionType d) { - AUTO_TRACE(); + const static std::map YDirectionTypeToString = { {YDirectionType::Standard, "Standard"}, {YDirectionType::Aligned, "Aligned"}}; @@ -146,7 +146,7 @@ std::string toString(YDirectionType d) { } YDirectionType YDirectionTypeFromString(const std::string& y_direction_string) { - AUTO_TRACE(); + const static std::map stringToYDirectionType = { {"Standard", YDirectionType::Standard}, {"Aligned", YDirectionType::Aligned}}; @@ -154,7 +154,7 @@ YDirectionType YDirectionTypeFromString(const std::string& y_direction_string) { } std::string toString(ZDirectionType d) { - AUTO_TRACE(); + const static std::map ZDirectionTypeToString = { {ZDirectionType::Standard, "Standard"}, {ZDirectionType::Average, "Average"}}; @@ -162,7 +162,7 @@ std::string toString(ZDirectionType d) { } ZDirectionType ZDirectionTypeFromString(const std::string& z_direction_string) { - AUTO_TRACE(); + const static std::map stringToZDirectionType = { {"Standard", ZDirectionType::Standard}, {"Average", ZDirectionType::Average}}; diff --git a/src/sys/boutcomm.cxx b/src/sys/boutcomm.cxx index 2d78a24076..358f2346f9 100644 --- a/src/sys/boutcomm.cxx +++ b/src/sys/boutcomm.cxx @@ -58,6 +58,8 @@ int BoutComm::size() { return NPES; } +void BoutComm::abort(int errorcode) { MPI_Abort(get(), errorcode); } + BoutComm* BoutComm::getInstance() { if (instance == nullptr) { // Create the singleton object diff --git a/src/sys/boutexception.cxx b/src/sys/boutexception.cxx index 825fb37e28..eff2de2e1f 100644 --- a/src/sys/boutexception.cxx +++ b/src/sys/boutexception.cxx @@ -1,39 +1,38 @@ -#include "bout/build_defines.hxx" - #include #include #include -#include + #include -#if BOUT_USE_BACKTRACE -#include -#include -#endif +#include // IWYU pragma: keep +#include +#include -#include #include #include #include -namespace { -const std::string header{"====== Exception thrown ======\n"}; -} +bool BoutException::show_backtrace = true; void BoutParallelThrowRhsFail(int status, const char* message) { - int allstatus; + int allstatus = 0; MPI_Allreduce(&status, &allstatus, 1, MPI_INT, MPI_LOR, BoutComm::get()); - if (allstatus) { + if (allstatus != 0) { throw BoutRhsFail(message); } } BoutException::BoutException(std::string msg) : message(std::move(msg)) { - makeBacktrace(); - if (std::getenv("BOUT_SHOW_BACKTRACE") != nullptr) { - message = getBacktrace() + "\n" + message; + const char* show_backtrace_env_var = std::getenv("BOUT_SHOW_BACKTRACE"); + const auto should_show_backtrace = + show_backtrace + and ((show_backtrace_env_var == nullptr) + or (show_backtrace_env_var != nullptr + and std::string{show_backtrace_env_var} != "0")); + if (should_show_backtrace) { + message = getBacktrace(); } } @@ -42,67 +41,36 @@ BoutException::~BoutException() { // up the msg_stack. We also won't know how many messages to pop, so // just clear everything msg_stack.clear(); -#if BOUT_USE_BACKTRACE - // Call required for memory allocated by `backtrace_symbols` - free(messages); // NOLINT -#endif } std::string BoutException::getBacktrace() const { - std::string backtrace_message; -#if BOUT_USE_BACKTRACE - backtrace_message = "====== Exception path ======\n"; - // skip first stack frame (points here) - for (int i = trace_size - 1; i > 1; --i) { - backtrace_message += fmt::format(FMT_STRING("[bt] #{:d} {:s}\n"), i - 1, messages[i]); - // find first occurence of '(' or ' ' in message[i] and assume - // everything before that is the file name. (Don't go beyond 0 though - // (string terminator) - int p = 0; // snprintf %.*s expects int - while (messages[i][p] != '(' && messages[i][p] != ' ' && messages[i][p] != 0) { - ++p; - } + using namespace cpptrace; - // If we are compiled as PIE, need to get base pointer of .so and substract - Dl_info info; - void* ptr = trace[i]; - if (dladdr(trace[i], &info)) { - // Additionally, check whether this is the default offset for an executable - if (info.dli_fbase != reinterpret_cast(0x400000)) { - ptr = reinterpret_cast(reinterpret_cast(trace[i]) - - reinterpret_cast(info.dli_fbase)); - } - } + const auto colours = isatty(stdout_fileno) || isatty(stderr_fileno) + ? formatter::color_mode::always + : formatter::color_mode::none; - // Pipe stderr to /dev/null to avoid cluttering output - // when addr2line fails or is not installed - const auto syscom = fmt::format( - FMT_STRING("addr2line {:p} -Cfpie {:.{}s} 2> /dev/null"), ptr, messages[i], p); - // last parameter is the file name of the symbol - FILE* file = popen(syscom.c_str(), "r"); - if (file != nullptr) { - std::array out{}; - char* retstr = nullptr; - std::string buf; - while ((retstr = fgets(out.data(), out.size() - 1, file)) != nullptr) { - buf += retstr; - } - int const status = pclose(file); - if (status == 0) { - backtrace_message += buf; - } - } - } -#else - backtrace_message = "Stacktrace not enabled.\n"; -#endif + auto formatter = cpptrace::formatter{} + .addresses(formatter::address_mode::none) + .break_before_filename(true) + .colors(colours) + .snippets(true) + .symbols(formatter::symbol_mode::pretty) + .filter([](const stacktrace_frame& frame) { + return ( + // Don't include our exception machinery + (frame.symbol.find("BoutException::") == std::string::npos) + // Don't include pre-main functions + and (frame.symbol.find("__libc_start") == std::string::npos) + and (frame.symbol != "_start")); + }); - return backtrace_message + msg_stack.getDump() + "\n" + header + message + "\n"; -} + std::string backtrace_message = + fmt::format("{}\n\n====== Exception thrown ======\n{}", + formatter.format(generate_trace()), message); -void BoutException::makeBacktrace() { -#if BOUT_USE_BACKTRACE - trace_size = backtrace(trace.data(), TRACE_MAX); - messages = backtrace_symbols(trace.data(), trace_size); -#endif + if (msg_stack.size() > 0) { + return fmt::format("{}\n\n{}", backtrace_message, msg_stack.getDump()); + } + return backtrace_message; } diff --git a/src/sys/derivs.cxx b/src/sys/derivs.cxx index ee9bcbcc2c..aab75b8f19 100644 --- a/src/sys/derivs.cxx +++ b/src/sys/derivs.cxx @@ -38,19 +38,15 @@ * **************************************************************************/ +#include #include #include #include #include #include -#include -#include -#include - -#include - #include #include +#include /******************************************************************************* * First central derivatives diff --git a/src/sys/expressionparser.cxx b/src/sys/expressionparser.cxx index 804f371bbe..5573846c4f 100644 --- a/src/sys/expressionparser.cxx +++ b/src/sys/expressionparser.cxx @@ -22,12 +22,29 @@ * **************************************************************************/ -#include - +#include "bout/sys/expressionparser.hxx" + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include - -#include "bout/sys/gettext.hxx" -#include "bout/utils.hxx" +#include using std::list; using std::string; @@ -183,7 +200,7 @@ FieldGeneratorPtr FieldBinary::clone(const list args) { bool toBool(BoutReal rval) { int ival = ROUND(rval); if ((fabs(rval - static_cast(ival)) > 1e-3) or (ival < 0) or (ival > 1)) { - throw BoutException(_("Boolean operator argument {:e} is not a bool"), rval); + throw BoutException(_f("Boolean operator argument {:e} is not a bool"), rval); } return ival == 1; } @@ -302,11 +319,6 @@ ExpressionParser::fuzzyFind(const std::string& name, FieldGeneratorPtr ExpressionParser::parseIdentifierExpr(LexInfo& lex) const { // Make a nice error message if we couldn't find the identifier const auto generatorNotFoundErrorMessage = [&](const std::string& name) -> std::string { - const std::string message_template = _( - R"(Couldn't find generator '{}'. BOUT++ expressions are now case-sensitive, so you -may need to change your input file. -{})"); - // Start position of the current identifier: by this point, we've either // moved one character past the token, or we're still at the start const auto start = @@ -326,13 +338,18 @@ may need to change your input file. [](const auto& match) -> bool { return match.distance == 0; }); // No matches, just point out the error + std::string error_message = fmt::format( + _f( + R"(Couldn't find generator '{}'. BOUT++ expressions are now case-sensitive, so you +may need to change your input file. +{})"), + name, problem_bit); if (possible_matches.empty()) { - return fmt::format(message_template, name, problem_bit); + return error_message; } // Give the first suggestion as a possible alternative - std::string error_message = fmt::format(message_template, name, problem_bit); - error_message += fmt::format(_("\n {1: ^{2}}{0}\n Did you mean '{0}'?"), + error_message += fmt::format(_f("\n {1: ^{2}}{0}\n Did you mean '{0}'?"), possible_matches.begin()->name, "", start); return error_message; }; diff --git a/src/sys/generator_context.cxx b/src/sys/generator_context.cxx index c34f266b76..01090daa51 100644 --- a/src/sys/generator_context.cxx +++ b/src/sys/generator_context.cxx @@ -1,5 +1,7 @@ #include "bout/sys/generator_context.hxx" + #include "bout/boundary_region.hxx" +#include "bout/bout_types.hxx" #include "bout/constants.hxx" #include "bout/mesh.hxx" @@ -15,9 +17,8 @@ Context::Context(int ix, int iy, int iz, CELL_LOC loc, Mesh* msh, BoutReal t) parameters["y"] = (loc == CELL_YLOW) ? PI * (msh->GlobalY(iy) + msh->GlobalY(iy - 1)) : TWOPI * msh->GlobalY(iy); - parameters["z"] = (loc == CELL_ZLOW) - ? TWOPI * (iz - 0.5) / static_cast(msh->LocalNz) - : TWOPI * iz / static_cast(msh->LocalNz); + parameters["z"] = (loc == CELL_ZLOW) ? PI * (msh->GlobalZ(iz) + msh->GlobalZ(iz - 1)) + : TWOPI * msh->GlobalZ(iz); parameters["t"] = t; } @@ -35,9 +36,8 @@ Context::Context(const BoundaryRegion* bndry, int iz, CELL_LOC loc, BoutReal t, ? PI * (msh->GlobalY(jy_) + msh->GlobalY(jy_ - 1)) : TWOPI * msh->GlobalY(jy_); - parameters["z"] = (loc == CELL_ZLOW) - ? TWOPI * (iz - 0.5) / static_cast(msh->LocalNz) - : TWOPI * iz / static_cast(msh->LocalNz); + parameters["z"] = (loc == CELL_ZLOW) ? PI * (msh->GlobalZ(iz) + msh->GlobalZ(iz - 1)) + : TWOPI * msh->GlobalZ(iz); parameters["t"] = t; } diff --git a/src/sys/msg_stack.cxx b/src/sys/msg_stack.cxx index 502836324c..bd14a766ed 100644 --- a/src/sys/msg_stack.cxx +++ b/src/sys/msg_stack.cxx @@ -27,7 +27,7 @@ #include "bout/openmpwrap.hxx" #include #include -#include + #include #if BOUT_USE_OPENMP @@ -59,7 +59,9 @@ void MsgStack::pop() { return; } BOUT_OMP_SAFE(single) - { --position; } + { + --position; + } } void MsgStack::pop(int id) { @@ -87,11 +89,13 @@ void MsgStack::clear() { void MsgStack::dump() { BOUT_OMP_SAFE(single) - { output << this->getDump(); } + { + output << this->getDump(); + } } std::string MsgStack::getDump() { - std::string res = "====== Back trace ======\n"; + std::string res = "=== Additional information ===\n"; for (int i = position - 1; i >= 0; i--) { if (stack[i] != "") { res += " -> "; diff --git a/src/sys/options.cxx b/src/sys/options.cxx index 077b2a6d85..8bd83c0bfb 100644 --- a/src/sys/options.cxx +++ b/src/sys/options.cxx @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -106,13 +107,11 @@ Options::Options(InitializerList values, Options* parent_instance, } Options& Options::operator[](const std::string& name) { - TRACE("Options::operator[]"); - if (isValue()) { - throw BoutException(_("Trying to index Option '{0}' with '{1}', but '{0}' is a " - "value, not a section.\n" - "This is likely the result of clashing input options, and you " - "may have to rename one of them.\n"), + throw BoutException(_f("Trying to index Option '{0}' with '{1}', but '{0}' is a " + "value, not a section.\n" + "This is likely the result of clashing input options, and you " + "may have to rename one of them.\n"), full_name, name); } @@ -145,13 +144,11 @@ Options& Options::operator[](const std::string& name) { } const Options& Options::operator[](const std::string& name) const { - TRACE("Options::operator[] const"); - if (isValue()) { - throw BoutException(_("Trying to index Option '{0}' with '{1}', but '{0}' is a " - "value, not a section.\n" - "This is likely the result of clashing input options, and you " - "may have to rename one of them.\n"), + throw BoutException(_f("Trying to index Option '{0}' with '{1}', but '{0}' is a " + "value, not a section.\n" + "This is likely the result of clashing input options, and you " + "may have to rename one of them.\n"), full_name, name); } @@ -169,7 +166,7 @@ const Options& Options::operator[](const std::string& name) const { auto child = children.find(name); if (child == children.end()) { // Doesn't exist - throw BoutException(_("Option {:s}:{:s} does not exist"), full_name, name); + throw BoutException(_f("Option {:s}:{:s} does not exist"), full_name, name); } return child->second; @@ -358,6 +355,31 @@ Options& Options::assign<>(Tensor val, std::string source) { return *this; } +void saveParallel(Options& opt, const std::string& name, const Field3D& tosave) { + opt[name] = tosave; + const size_t numberParallelSlices = + tosave.hasParallelSlices() ? 0 : tosave.getMesh()->ystart; + for (size_t i0 = 1; i0 <= numberParallelSlices; ++i0) { + for (int i : {i0, -i0}) { + Field3D tmp; + tmp.allocate(); + const auto& fpar = tosave.ynext(i); + if (fpar.isAllocated()) { + for (auto j : tmp.getRegion("RGN_NOY")) { + tmp[j] = fpar[j.yp(i)]; + } + opt[fmt::format("{}_y{:+d}", name, i)] = tmp; + } else { + if (tosave.isFci()) { // likely an error + throw BoutException( + "Tried to save parallel fields - but parallel field {} is not allocated", + i); + } + } + } + } +} + namespace { /// Use FieldFactory to evaluate expression double parseExpression(const Options::ValueType& value, const Options* options, @@ -373,7 +395,7 @@ double parseExpression(const Options::ValueType& value, const Options* options, return gen->generate({}); } catch (ParseException& error) { // Convert any exceptions to something a bit more useful - throw BoutException(_("Couldn't get {} from option {:s} = '{:s}': {}"), type, + throw BoutException(_f("Couldn't get {} from option {:s} = '{:s}': {}"), type, full_name, bout::utils::variantToString(value), error.what()); } } @@ -381,7 +403,7 @@ double parseExpression(const Options::ValueType& value, const Options* options, /// Helper function to print `key = value` with optional source template void printNameValueSourceLine(const Options& option, const T& value) { - output_info.write(_("\tOption {} = {}"), option.str(), value); + output_info.write(_f("\tOption {} = {}"), option.str(), value); if (option.hasAttribute("source")) { // Specify the source of the setting output_info.write(" ({})", @@ -394,7 +416,7 @@ void printNameValueSourceLine(const Options& option, const T& value) { template <> std::string Options::as(const std::string& UNUSED(similar_to)) const { if (is_section) { - throw BoutException(_("Option {:s} has no value"), full_name); + throw BoutException(_f("Option {:s} has no value"), full_name); } // Mark this option as used @@ -410,7 +432,7 @@ std::string Options::as(const std::string& UNUSED(similar_to)) cons template <> int Options::as(const int& UNUSED(similar_to)) const { if (is_section) { - throw BoutException(_("Option {:s} has no value"), full_name); + throw BoutException(_f("Option {:s} has no value"), full_name); } int result = 0; @@ -430,7 +452,7 @@ int Options::as(const int& UNUSED(similar_to)) const { } else { // Another type which can't be converted - throw BoutException(_("Value for option {:s} is not an integer"), full_name); + throw BoutException(_f("Value for option {:s} is not an integer"), full_name); } // Convert to int by rounding @@ -438,7 +460,7 @@ int Options::as(const int& UNUSED(similar_to)) const { // Check that the value is close to an integer if (fabs(rval - static_cast(result)) > 1e-3) { - throw BoutException(_("Value for option {:s} = {:e} is not an integer"), full_name, + throw BoutException(_f("Value for option {:s} = {:e} is not an integer"), full_name, rval); } } @@ -453,7 +475,7 @@ int Options::as(const int& UNUSED(similar_to)) const { template <> BoutReal Options::as(const BoutReal& UNUSED(similar_to)) const { if (is_section) { - throw BoutException(_("Option {:s} has no value"), full_name); + throw BoutException(_f("Option {:s} has no value"), full_name); } BoutReal result = BoutNaN; @@ -468,7 +490,7 @@ BoutReal Options::as(const BoutReal& UNUSED(similar_to)) const { result = parseExpression(value, this, "BoutReal", full_name); } else { - throw BoutException(_("Value for option {:s} cannot be converted to a BoutReal"), + throw BoutException(_f("Value for option {:s} cannot be converted to a BoutReal"), full_name); } @@ -483,7 +505,7 @@ BoutReal Options::as(const BoutReal& UNUSED(similar_to)) const { template <> bool Options::as(const bool& UNUSED(similar_to)) const { if (is_section) { - throw BoutException(_("Option {:s} has no value"), full_name); + throw BoutException(_f("Option {:s} has no value"), full_name); } bool result = false; @@ -498,12 +520,12 @@ bool Options::as(const bool& UNUSED(similar_to)) const { // Check that the result is either close to 1 (true) or close to 0 (false) const int ival = ROUND(rval); if ((fabs(rval - static_cast(ival)) > 1e-3) or (ival < 0) or (ival > 1)) { - throw BoutException(_("Value for option {:s} = {:e} is not a bool"), full_name, + throw BoutException(_f("Value for option {:s} = {:e} is not a bool"), full_name, rval); } result = ival == 1; } else { - throw BoutException(_("Value for option {:s} cannot be converted to a bool"), + throw BoutException(_f("Value for option {:s} cannot be converted to a bool"), full_name); } @@ -583,7 +605,7 @@ Field3D Options::as(const Field3D& similar_to) const { localmesh->LocalNz); } - throw BoutException(_("Value for option {:s} cannot be converted to a Field3D"), + throw BoutException(_f("Value for option {:s} cannot be converted to a Field3D"), full_name); } @@ -635,7 +657,7 @@ Field2D Options::as(const Field2D& similar_to) const { } } - throw BoutException(_("Value for option {:s} cannot be converted to a Field2D"), + throw BoutException(_f("Value for option {:s} cannot be converted to a Field2D"), full_name); } @@ -717,32 +739,51 @@ FieldPerp Options::as(const FieldPerp& similar_to) const { // to select a region from it using Mesh e.g. if this // is from the input grid file. } - throw BoutException(_("Value for option {:s} cannot be converted to a FieldPerp"), + throw BoutException(_f("Value for option {:s} cannot be converted to a FieldPerp"), full_name); } namespace { -/// Visitor to convert an int, BoutReal or Array/Matrix/Tensor to the -/// appropriate container +/// Primary declaration of ConvertContainer, for specialization below. +/// No definition needed unless it is used. template -struct ConvertContainer { +struct ConvertContainer; + +/// Visitor to convert an int, BoutReal or Array/Matrix/Tensor to the +/// appropriate container. Templated on both the container class C +/// and scalar type Scalar. +template