Numpy Datatype: Enable np.dtype() type conversions

examples/10_streaming_write.py

@@ -42,7 +42,7 @@
 
         length = 10
         local_data = np.arange(i * length, (i + 1) * length,
-                               dtype=np.dtype("double"))
+                               dtype="double")
         for dim in ["x", "y", "z"]:
             pos = electronPositions[dim]
             pos.reset_dataset(io.Dataset(local_data.dtype, [length]))
@@ -63,7 +63,7 @@
         temperature_dataset = temperature
         # let's say we are in a 3x3 mesh
         temperature_dataset.reset_dataset(
-            io.Dataset(np.dtype("double"), [3, 3]))
+            io.Dataset(local_data.dtype, [3, 3]))
         # temperature is constant
         temperature_dataset.make_constant(273.15)
 

examples/12_span_write.py

@@ -1,11 +1,10 @@
-import numpy as np
 import openpmd_api as io
 
 
 def span_write(filename):
     series = io.Series(filename, io.Access_Type.create_linear)
 
-    datatype = np.dtype("double")
+    datatype = "double"
     length = 10
     extent = [length]
     dataset = io.Dataset(datatype, extent)

examples/13_write_dynamic_configuration.py

@@ -82,7 +82,7 @@ def main():
 
         length = 10
         local_data = np.arange(i * length, (i + 1) * length,
-                               dtype=np.dtype("double"))
+                               dtype="double")
         for dim in ["x", "y", "z"]:
             pos = electronPositions[dim]
             pos.reset_dataset(io.Dataset(local_data.dtype, [length]))
@@ -122,10 +122,10 @@ def main():
         # temperature has no x,y,z components, so skip the last layer:
         temperature_dataset = temperature
         # let's say we are in a 3x3 mesh
-        dataset = io.Dataset(np.dtype("double"), [3, 3], config)
+        dataset = io.Dataset(local_data.dtype, [3, 3], config)
         temperature_dataset.reset_dataset(dataset)
         # temperature is constant
-        local_data = np.arange(i * 9, (i + 1) * 9, dtype=np.dtype("double"))
+        local_data = np.arange(i * 9, (i + 1) * 9, dtype="double")
         local_data = local_data.reshape([3, 3])
         temperature_dataset[()] = local_data
 

examples/15_compression.py

@@ -46,9 +46,9 @@ def write(filename, config):
         E.axis_labels = ["x", "y"]
         for dim in ["x", "y"]:
             component = E[dim]
-            component.reset_dataset(opmd.Dataset(np.dtype("float"), [10, 10]))
+            component.reset_dataset(opmd.Dataset("float", [10, 10]))
             component[:, :] = np.reshape(
-                np.arange(i * 100, (i + 1) * 100, dtype=np.dtype("float")),
+                np.arange(i * 100, (i + 1) * 100, dtype="float"),
                 [10, 10],
             )
 
@@ -57,13 +57,13 @@ def write(filename, config):
         for dim in ["x", "y"]:
             # Do not bother with a positionOffset
             position_offset = e["positionOffset"][dim]
-            position_offset.reset_dataset(opmd.Dataset(np.dtype("int"), [100]))
+            position_offset.reset_dataset(opmd.Dataset("int", [100]))
             position_offset.make_constant(0)
 
             position = e["position"][dim]
-            position.reset_dataset(opmd.Dataset(np.dtype("float"), [100]))
+            position.reset_dataset(opmd.Dataset("float", [100]))
             position[:] = np.arange(
-                i * 100, (i + 1) * 100, dtype=np.dtype("float")
+                i * 100, (i + 1) * 100, dtype="float"
             )
 
 

examples/7_extended_write_serial.py

@@ -89,7 +89,7 @@
     electrons["displacement"]["x"].unit_SI = 1.e-6
     del electrons["displacement"]
     electrons["weighting"] \
-        .reset_dataset(Dataset(np.dtype("float32"), extent=[1])) \
+        .reset_dataset(Dataset("float32", extent=[1])) \
         .make_constant(1.e-5)
 
     mesh = cur_it.meshes["lowRez_2D_field"]
@@ -128,7 +128,7 @@
     d = Dataset(partial_particleOff.dtype, mpiDims)
     electrons["positionOffset"]["x"].reset_dataset(d)
 
-    dset = Dataset(np.dtype("uint64"), extent=[2])
+    dset = Dataset("uint64", extent=[2])
     electrons.particle_patches["numParticles"].reset_dataset(dset)
     electrons.particle_patches["numParticlesOffset"]. \
         reset_dataset(dset)

examples/9_particle_write_serial.py

@@ -37,14 +37,14 @@
     # let's set a weird user-defined record this time
     electrons["displacement"].unit_dimension = {Unit_Dimension.M: 1}
     electrons["displacement"].unit_SI = 1.e-6
-    dset = Dataset(np.dtype("float64"), extent=[n_particles])
+    dset = Dataset("float64", extent=[n_particles])
     electrons["displacement"].reset_dataset(dset)
     electrons["displacement"].make_constant(42.43)
     # don't like it anymore? remove it with:
     # del electrons["displacement"]
 
     electrons["weighting"] \
-        .reset_dataset(Dataset(np.dtype("float32"), extent=[n_particles])) \
+        .reset_dataset(Dataset("float32", extent=[n_particles])) \
         .make_constant(1.e-5)
 
     particlePos_x = np.random.rand(n_particles).astype(np.float32)

include/openPMD/binding/python/Numpy.hpp

@@ -98,6 +98,44 @@ inline Datatype dtype_from_numpy(pybind11::dtype const dt)
     }
 }
 
+inline Datatype dtype_from_numpy(pybind11::object dt)
+{
+    if (py::isinstance<pybind11::dtype>(dt))
+    {
+        return dtype_from_numpy(py::cast<pybind11::dtype>(std::move(dt)));
+    }
+    else
+    {
+        pybind11::module_ numpy;
+        try
+        {
+            numpy = pybind11::module_::import("numpy");
+        }
+        catch (std::exception const &e)
+        {
+            throw std::runtime_error(
+                std::string(
+                    "dtype_from_numpy: Cannot convert from object type to "
+                    "datatype without numpy, failed importing: ") +
+                e.what());
+        }
+        pybind11::object create_dtype = numpy.attr("dtype");
+        pybind11::object dtype_obj;
+        try
+        {
+            dtype_obj = create_dtype(std::move(dt));
+        }
+        catch (std::exception const &e)
+        {
+            throw std::runtime_error(
+                std::string("dtype_from_numpy: Failed to create dtype from: ") +
+                pybind11::str(dt).cast<std::string>() +
+                std::string(", error: ") + e.what());
+        }
+        return dtype_from_numpy(py::cast<pybind11::dtype>(dtype_obj));
+    }
+}
+
 /** Return openPMD::Datatype from py::buffer_info::format
  */
 inline Datatype dtype_from_bufferformat(std::string const &fmt)
@@ -239,4 +277,5 @@ inline pybind11::dtype dtype_to_numpy(Datatype const dt)
         throw std::runtime_error("dtype_to_numpy: Invalid Datatype!");
     }
 }
+
 } // namespace openPMD

src/binding/python/Dataset.cpp

@@ -25,6 +25,7 @@
 #include "openPMD/binding/python/auxiliary.hpp"
 
 #include <string>
+#include <utility>
 
 void init_Dataset(py::module &m)
 {
@@ -37,7 +38,7 @@ void init_Dataset(py::module &m)
                 py::arg("extent"),
                 py::arg("options") = "{}")
             .def(
-                py::init([](py::dtype dt, Extent e, std::string options) {
+                py::init([](py::object dt, Extent e, std::string options) {
                     auto const d = dtype_from_numpy(std::move(dt));
                     return new Dataset{d, std::move(e), std::move(options)};
                 }),
@@ -54,8 +55,10 @@ void init_Dataset(py::module &m)
                 py::arg("extent"),
                 py::arg("options"))
             .def(
-                py::init([](py::dtype dt, Extent e, py::object const &options) {
-                    auto const d = dtype_from_numpy(std::move(dt));
+                py::init([](py::object const &dt,
+                            Extent e,
+                            py::object const &options) {
+                    auto const d = dtype_from_numpy(dt);
                     auto resolved_options = ::auxiliary::json_dumps(options);
                     return new Dataset{
                         d, std::move(e), std::move(resolved_options)};

src/binding/python/RecordComponent.cpp

@@ -50,6 +50,7 @@
 #include <string>
 #include <tuple>
 #include <type_traits>
+#include <utility>
 #include <vector>
 
 /** Convert a py::tuple of py::slices to Offset & Extent
@@ -1011,9 +1012,10 @@ void init_RecordComponent(py::module &m)
         .def(
             "make_empty",
             [](RecordComponent &rc,
-               pybind11::dtype const &dt,
+               pybind11::object dt,
                uint8_t dimensionality) {
-                return rc.makeEmpty(dtype_from_numpy(dt), dimensionality);
+                return rc.makeEmpty(
+                    dtype_from_numpy(std::move(dt)), dimensionality);
             })
 
         // deprecated: pass-through C++ API