Merge pull request #2934 from boutproject/flush-frequency-netcdf

Add `output:flush_frequency` for `OptionsNetCDF`

Author: bendudson

.git-blame-ignore-revs

@@ -14,3 +14,4 @@ c8f38049359170a34c915e209276238ea66b9a1e
 8d5cb39e03c2644715a50684f8cd0318b4e82676
 ec69e8838be2dde140a915e50506f8e1ce3cb534
 f2bc0488a298f136294c523bc5ab4086d090059b
+1b4707187a3a85126338303dc766280b8fb2dc56

include/bout/options_io.hxx

@@ -1,3 +1,19 @@
+#pragma once
+
+#ifndef OPTIONS_IO_H
+#define OPTIONS_IO_H
+
+#include "bout/build_defines.hxx"
+#include "bout/generic_factory.hxx"
+#include "bout/options.hxx"
+
+#include <memory>
+#include <string>
+
+class Mesh;
+
+namespace bout {
+
 /// Parent class for IO to binary files and streams
 ///
 ///
@@ -36,30 +52,14 @@
 ///
 ///    if mesh is omitted, no grid information is added.
 ///
-
-#pragma once
-
-#ifndef OPTIONS_IO_H
-#define OPTIONS_IO_H
-
-#include "bout/build_defines.hxx"
-#include "bout/generic_factory.hxx"
-#include "bout/mesh.hxx"
-#include "bout/options.hxx"
-
-#include <memory>
-#include <string>
-
-namespace bout {
-
 class OptionsIO {
 public:
   /// No default constructor, as settings are required
   OptionsIO() = delete;
 
   /// Constructor specifies the kind of file, and options to control
   /// the name of file, mode of operation etc.
-  OptionsIO(Options&) {}
+  OptionsIO(Options& /*unused*/) {}
 
   virtual ~OptionsIO() = default;
 
@@ -81,6 +81,9 @@ public:
   /// ADIOS: Indicate completion of an output step.
   virtual void verifyTimesteps() const = 0;
 
+  /// NetCDF: Flush file to disk
+  virtual void flush() {}
+
   /// Create an OptionsIO for I/O to the given file.
   /// This uses the default file type and default options.
   static std::unique_ptr<OptionsIO> create(const std::string& file);

include/bout/physicsmodel.hxx

@@ -48,6 +48,7 @@ class PhysicsModel;
 #include "bout/utils.hxx"
 
 #include <chrono>
+#include <cstddef>
 #include <thread>
 #include <type_traits>
 #include <vector>
@@ -378,6 +379,9 @@ protected:
     PhysicsModel* model;
   };
 
+  /// Set timestep counter for flushing file
+  void setFlushCounter(std::size_t iteration) { flush_counter = iteration; }
+
 private:
   /// State for outputs
   Options output_options;
@@ -401,6 +405,10 @@ private:
   bool initialised{false};
   /// write restarts and pass outputMonitor method inside a Monitor subclass
   PhysicsModelMonitor modelMonitor{this};
+  /// How often to flush to disk
+  std::size_t flush_frequency{1};
+  /// Current timestep counter
+  std::size_t flush_counter{0};
 };
 
 /*!

manual/sphinx/user_docs/bout_options.rst

@@ -554,42 +554,42 @@ may be useful anyway. See :ref:`sec-output` for more details.
 Input and Output
 ----------------
 
-The output (dump) files with time-history are controlled by settings
-in a section called “output”. Restart files contain a single
-time-slice, and are controlled by a section called “restart”. The
-options available are listed in table :numref:`tab-outputopts`.
+The output (dump) files with time-history are controlled by settings in a
+section called ``"output"``. Restart files contain a single time-slice, and are
+controlled by a section called ``"restart"``. The options available are listed
+in table :numref:`tab-outputopts`.
 
 .. _tab-outputopts:
 .. table:: Output file options
 
-   +-------------+----------------------------------------------------+--------------+
-   | Option      | Description                                        | Default      |
-   |             |                                                    | value        |
-   +-------------+----------------------------------------------------+--------------+
-   | enabled     | Writing is enabled                                 | true         |
-   +-------------+----------------------------------------------------+--------------+
-   | type        | File type e.g. "netcdf" or "adios"                 | "netcdf"     |
-   +-------------+----------------------------------------------------+--------------+
-   | prefix      | File name prefix                                   | "BOUT.dmp"   |
-   +-------------+----------------------------------------------------+--------------+
-   | path        | Directory to write the file into                   | ``datadir``  |
-   +-------------+----------------------------------------------------+--------------+
-   | floats      | Write floats rather than doubles                   | false        |
-   +-------------+----------------------------------------------------+--------------+
-   | flush       | Flush the file to disk after each write            | true         |
-   +-------------+----------------------------------------------------+--------------+
-   | guards      | Output guard cells                                 | true         |
-   +-------------+----------------------------------------------------+--------------+
-   | openclose   | Re-open the file for each write, and close after   | true         |
-   +-------------+----------------------------------------------------+--------------+
+   +----------------------+-----------------------------------------+----------------+
+   | Option               | Description                             | Default value  |
+   +======================+=========================================+================+
+   | ``append``           | Append to existing file if true,        | ``false``      |
+   |                      | otherwise overwrite                     |                |
+   +----------------------+-----------------------------------------+----------------+
+   | ``enabled``          | Writing is enabled                      | ``true``       |
+   +----------------------+-----------------------------------------+----------------+
+   | ``flush_frequency``  | How many output timesteps between       | ``1``          |
+   |                      | writing output to disk (NetCDF only)    |                |
+   +----------------------+-----------------------------------------+----------------+
+   | ``prefix``           | File name prefix                        | ``"BOUT.dmp"`` |
+   +----------------------+-----------------------------------------+----------------+
+   | ``path``             | Directory to write the file into        | ``datadir``    |
+   +----------------------+-----------------------------------------+----------------+
+   | ``type``             | File type, either ``"netcdf"`` or       | ``"netcdf"``   |
+   |                      | ``"adios"``                             |                |
+   +----------------------+-----------------------------------------+----------------+
 
 |
 
-**enabled** is useful mainly for doing performance or scaling tests, where you
-want to exclude I/O from the timings. **floats** can be used to reduce the size
-of the output files: files are stored as double by default, but setting
-**floats = true** changes the output to single-precision floats.
-
+- ``enabled`` is useful mainly for doing performance or scaling tests, where you
+  want to exclude I/O from the timings.
+- If you find that IO is taking more and more time as your simulation goes on,
+  try setting ``flush_frequency`` to a larger value such as ``10``. This can
+  workaround an issue with NetCDF where subsequent writes take longer and
+  longer. However, larger values risk losing more data in the event of a crash
+  or the simulation being killed early.
 
 Implementation
 --------------

src/physics/physicsmodel.cxx

@@ -34,12 +34,14 @@
 
 #include "bout/version.hxx"
 #include <bout/mesh.hxx>
+#include <bout/options.hxx>
 #include <bout/sys/timer.hxx>
 #include <bout/vector2d.hxx>
 #include <bout/vector3d.hxx>
 
 #include <fmt/core.h>
 
+#include <cstddef>
 #include <string>
 using namespace std::literals;
 
@@ -67,9 +69,10 @@ PhysicsModel::PhysicsModel()
                                                     .withDefault(true)),
       restart_enabled(Options::root()["restart_files"]["enabled"]
                           .doc("Write restart files")
-                          .withDefault(true))
-
-{
+                          .withDefault(true)),
+      flush_frequency(Options::root()["output"]["flush_frequency"]
+                          .doc("How often to flush to disk")
+                          .withDefault<std::size_t>(1)) {
   if (output_enabled) {
     output_file = bout::OptionsIOFactory::getInstance().createOutput();
   }
@@ -188,7 +191,7 @@ int PhysicsModel::postInit(bool restarting) {
 }
 
 void PhysicsModel::outputVars(Options& options) {
-  Timer time("io");
+  const Timer time("io");
   for (const auto& item : dump.getData()) {
     bout::utils::visit(bout::OptionsConversionVisitor{options, item.name}, item.value);
     if (item.repeat) {
@@ -198,7 +201,7 @@ void PhysicsModel::outputVars(Options& options) {
 }
 
 void PhysicsModel::restartVars(Options& options) {
-  Timer time("io");
+  const Timer time("io");
   for (const auto& item : restart.getData()) {
     bout::utils::visit(bout::OptionsConversionVisitor{options, item.name}, item.value);
     if (item.repeat) {
@@ -216,9 +219,7 @@ void PhysicsModel::writeRestartFile() {
 void PhysicsModel::writeOutputFile() { writeOutputFile(output_options); }
 
 void PhysicsModel::writeOutputFile(const Options& options) {
-  if (output_enabled) {
-    output_file->write(options, "t");
-  }
+  writeOutputFile(options, "t");
 }
 
 void PhysicsModel::writeOutputFile(const Options& options,
@@ -229,13 +230,19 @@ void PhysicsModel::writeOutputFile(const Options& options,
 }
 
 void PhysicsModel::finishOutputTimestep() const {
-  if (output_enabled) {
+  const Timer timer("io");
+
+  if (output_enabled and (flush_counter % flush_frequency == 0)) {
+    output_file->flush();
     output_file->verifyTimesteps();
   }
 }
 
 int PhysicsModel::PhysicsModelMonitor::call(Solver* solver, BoutReal simtime,
                                             int iteration, int nout) {
+
+  model->setFlushCounter(static_cast<std::size_t>(iteration));
+
   // Restart file variables
   solver->outputVars(model->restart_options, false);
   model->restartVars(model->restart_options);
@@ -254,5 +261,8 @@ int PhysicsModel::PhysicsModelMonitor::call(Solver* solver, BoutReal simtime,
   model->outputVars(model->output_options);
   model->writeOutputFile();
 
+  // Reset output options, this avoids rewriting time-independent data
+  model->output_options = Options{};
+
   return monitor_result;
 }

src/sys/options/options_netcdf.cxx

@@ -791,10 +791,10 @@ void OptionsNetCDF::write(const Options& options, const std::string& time_dim) {
   }
 
   writeGroup(options, *data_file, time_dim);
-
-  data_file->sync();
 }
 
+void OptionsNetCDF::flush() { data_file->sync(); }
+
 } // namespace bout
 
 #endif // BOUT_HAS_NETCDF

src/sys/options/options_netcdf.hxx

@@ -39,7 +39,7 @@ public:
   ///  - "append"  File mode, default is false
   OptionsNetCDF(Options& options);
 
-  ~OptionsNetCDF() {}
+  ~OptionsNetCDF() override = default;
 
   OptionsNetCDF(const OptionsNetCDF&) = delete;
   OptionsNetCDF(OptionsNetCDF&&) noexcept = default;
@@ -58,6 +58,9 @@ public:
   /// any differences, otherwise is silent
   void verifyTimesteps() const override;
 
+  /// Flush file to disk
+  void flush() override;
+
 private:
   enum class FileMode {
     replace, ///< Overwrite file when writing
@@ -74,7 +77,7 @@ private:
 };
 
 namespace {
-RegisterOptionsIO<OptionsNetCDF> registeroptionsnetcdf("netcdf");
+const inline RegisterOptionsIO<OptionsNetCDF> registeroptionsnetcdf("netcdf");
 }
 
 } // namespace bout