From 42bcb88f80ca8ad31bdec7d774e57d941cb51f97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 2 Mar 2026 17:14:26 +0100 Subject: [PATCH 1/9] Try converting from things like np.int_ to np.dtype --- include/openPMD/binding/python/Numpy.hpp | 15 +++++++++++++++ src/binding/python/Dataset.cpp | 15 +++++++++------ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/include/openPMD/binding/python/Numpy.hpp b/include/openPMD/binding/python/Numpy.hpp index 062b76f560..f29b7b1cc9 100644 --- a/include/openPMD/binding/python/Numpy.hpp +++ b/include/openPMD/binding/python/Numpy.hpp @@ -98,6 +98,20 @@ inline Datatype dtype_from_numpy(pybind11::dtype const dt) } } +inline Datatype dtype_from_numpy(pybind11::object const &dt) +{ + if (py::isinstance(dt)) + { + return dtype_from_numpy(py::cast(dt)); + } + else + { + auto numpy = pybind11::module_::import("numpy"); + auto create_dtype = numpy.attr("dtype"); + return dtype_from_numpy(py::cast(create_dtype(dt))); + } +} + /** Return openPMD::Datatype from py::buffer_info::format */ inline Datatype dtype_from_bufferformat(std::string const &fmt) @@ -239,4 +253,5 @@ inline pybind11::dtype dtype_to_numpy(Datatype const dt) throw std::runtime_error("dtype_to_numpy: Invalid Datatype!"); } } + } // namespace openPMD diff --git a/src/binding/python/Dataset.cpp b/src/binding/python/Dataset.cpp index 92ae82a441..b9a84ca78b 100644 --- a/src/binding/python/Dataset.cpp +++ b/src/binding/python/Dataset.cpp @@ -37,10 +37,11 @@ void init_Dataset(py::module &m) py::arg("extent"), py::arg("options") = "{}") .def( - py::init([](py::dtype dt, Extent e, std::string options) { - auto const d = dtype_from_numpy(std::move(dt)); - return new Dataset{d, std::move(e), std::move(options)}; - }), + py::init( + [](py::object const &dt, Extent e, std::string options) { + auto const d = dtype_from_numpy(dt); + return new Dataset{d, std::move(e), std::move(options)}; + }), py::arg("dtype"), py::arg("extent"), py::arg("options") = "{}") @@ -54,8 +55,10 @@ void init_Dataset(py::module &m) py::arg("extent"), py::arg("options")) .def( - py::init([](py::dtype dt, Extent e, py::object const &options) { - auto const d = dtype_from_numpy(std::move(dt)); + py::init([](py::object const &dt, + Extent e, + py::object const &options) { + auto const d = dtype_from_numpy(dt); auto resolved_options = ::auxiliary::json_dumps(options); return new Dataset{ d, std::move(e), std::move(resolved_options)}; From ad95286cea5b359b63354fcbed0b27395266044c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 2 Mar 2026 17:25:03 +0100 Subject: [PATCH 2/9] Error handling --- include/openPMD/binding/python/Numpy.hpp | 50 ++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/include/openPMD/binding/python/Numpy.hpp b/include/openPMD/binding/python/Numpy.hpp index f29b7b1cc9..f2c66d3d68 100644 --- a/include/openPMD/binding/python/Numpy.hpp +++ b/include/openPMD/binding/python/Numpy.hpp @@ -106,9 +106,53 @@ inline Datatype dtype_from_numpy(pybind11::object const &dt) } else { - auto numpy = pybind11::module_::import("numpy"); - auto create_dtype = numpy.attr("dtype"); - return dtype_from_numpy(py::cast(create_dtype(dt))); + pybind11::module_ numpy; + try + { + numpy = pybind11::module_::import("numpy"); + } + catch (std::exception const &e) + { + throw std::runtime_error( + std::string( + "dtype_from_numpy: Cannot convert from object type to " + "datatype without numpy, failed importing: ") + + e.what()); + } + pybind11::object create_dtype; + try + { + create_dtype = numpy.attr("dtype"); + } + catch (std::exception const &e) + { + throw std::runtime_error( + std::string( + "dtype_from_numpy: Failed to access numpy.dtype: ") + + e.what()); + } + pybind11::object dtype_obj; + try + { + dtype_obj = create_dtype(dt); + } + catch (std::exception const &e) + { + throw std::runtime_error( + std::string("dtype_from_numpy: Failed to create dtype from: ") + + pybind11::str(dt).cast() + + std::string(", error: ") + e.what()); + } + try + { + return dtype_from_numpy(py::cast(dtype_obj)); + } + catch (std::exception const &e) + { + throw std::runtime_error( + std::string("dtype_from_numpy: Failed to cast to dtype: ") + + e.what()); + } } } From be3969bbed707c47888eeb00b691bbe3cad24849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 2 Mar 2026 18:02:46 +0100 Subject: [PATCH 3/9] Same for makeEmpty --- src/binding/python/RecordComponent.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/binding/python/RecordComponent.cpp b/src/binding/python/RecordComponent.cpp index 382783a730..afc2a0e717 100644 --- a/src/binding/python/RecordComponent.cpp +++ b/src/binding/python/RecordComponent.cpp @@ -1011,7 +1011,7 @@ void init_RecordComponent(py::module &m) .def( "make_empty", [](RecordComponent &rc, - pybind11::dtype const &dt, + pybind11::object const &dt, uint8_t dimensionality) { return rc.makeEmpty(dtype_from_numpy(dt), dimensionality); }) From aafd96dc1808f4a627ab49e4e702c7487e943e4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 2 Mar 2026 18:02:57 +0100 Subject: [PATCH 4/9] Adapt tests --- test/python/unittest/API/APITest.py | 154 ++++++++++++++-------------- 1 file changed, 77 insertions(+), 77 deletions(-) diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 534de9f8a8..025f06dff9 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -98,9 +98,9 @@ def refcountingCreateData(self): for dim in ["x", "y"]: component = E[dim] component.reset_dataset( - io.Dataset(np.dtype("float"), [10, 10])) + io.Dataset("float", [10, 10])) component[:, :] = np.reshape( - np.arange(i * 100, (i + 1) * 100, dtype=np.dtype("float")), + np.arange(i * 100, (i + 1) * 100, dtype="float"), [10, 10], ) @@ -110,13 +110,13 @@ def refcountingCreateData(self): # Do not bother with a positionOffset position_offset = e["positionOffset"][dim] position_offset.reset_dataset( - io.Dataset(np.dtype("int"), [100])) + io.Dataset("int", [100])) position_offset.make_constant(0) position = e["position"][dim] - position.reset_dataset(io.Dataset(np.dtype("float"), [100])) + position.reset_dataset(io.Dataset("float", [100])) position[:] = np.arange( - i * 100, (i + 1) * 100, dtype=np.dtype("float") + i * 100, (i + 1) * 100, dtype="float" ) def testRefCounting(self): @@ -128,7 +128,7 @@ def testRefCounting(self): loaded = pos_x[:] read.flush() self.assertTrue(np.allclose( - loaded, np.arange(0, 100, dtype=np.dtype("float")))) + loaded, np.arange(0, 100, dtype="float"))) def testFieldData(self): """ Testing serial IO on a pure field dataset. """ @@ -593,39 +593,39 @@ def makeConstantRoundTrip(self, file_ending): self.assertTrue(ms["pybool"][SCALAR].constant) if found_numpy: - ms["int16"][SCALAR].reset_dataset(DS(np.dtype("int16"), extent)) + ms["int16"][SCALAR].reset_dataset(DS("int16", extent)) ms["int16"][SCALAR].make_constant(np.int16(234)) - ms["int32"][SCALAR].reset_dataset(DS(np.dtype("int32"), extent)) + ms["int32"][SCALAR].reset_dataset(DS("int32", extent)) ms["int32"][SCALAR].make_constant(np.int32(43)) - ms["int64"][SCALAR].reset_dataset(DS(np.dtype("int64"), extent)) + ms["int64"][SCALAR].reset_dataset(DS("int64", extent)) ms["int64"][SCALAR].make_constant(np.int64(987654321)) - ms["uint16"][SCALAR].reset_dataset(DS(np.dtype("uint16"), extent)) + ms["uint16"][SCALAR].reset_dataset(DS("uint16", extent)) ms["uint16"][SCALAR].make_constant(np.uint16(134)) - ms["uint32"][SCALAR].reset_dataset(DS(np.dtype("uint32"), extent)) + ms["uint32"][SCALAR].reset_dataset(DS("uint32", extent)) ms["uint32"][SCALAR].make_constant(np.uint32(32)) - ms["uint64"][SCALAR].reset_dataset(DS(np.dtype("uint64"), extent)) + ms["uint64"][SCALAR].reset_dataset(DS("uint64", extent)) ms["uint64"][SCALAR].make_constant(np.uint64(9876543210)) - ms["single"][SCALAR].reset_dataset(DS(np.dtype("single"), extent)) + ms["single"][SCALAR].reset_dataset(DS("single", extent)) ms["single"][SCALAR].make_constant(np.single(1.234)) - ms["double"][SCALAR].reset_dataset(DS(np.dtype("double"), extent)) + ms["double"][SCALAR].reset_dataset(DS("double", extent)) ms["double"][SCALAR].make_constant(np.double(1.234567)) - ms["longdouble"][SCALAR].reset_dataset(DS(np.dtype("longdouble"), + ms["longdouble"][SCALAR].reset_dataset(DS("longdouble", extent)) ms["longdouble"][SCALAR].make_constant(np.longdouble(1.23456789)) ms["complex64"][SCALAR].reset_dataset( - DS(np.dtype("complex64"), extent)) + DS("complex64", extent)) ms["complex64"][SCALAR].make_constant( np.complex64(1.234 + 2.345j)) ms["complex128"][SCALAR].reset_dataset( - DS(np.dtype("complex128"), extent)) + DS("complex128", extent)) ms["complex128"][SCALAR].make_constant( np.complex128(1.234567 + 2.345678j)) if file_ending not in ["bp", "bp4", "bp5"]: ms["clongdouble"][SCALAR].reset_dataset( - DS(np.dtype("clongdouble"), extent)) + DS("clongdouble", extent)) ms["clongdouble"][SCALAR].make_constant( np.clongdouble(1.23456789 + 2.34567890j)) @@ -685,30 +685,30 @@ def makeConstantRoundTrip(self, file_ending): self.assertTrue(ms["double"][SCALAR].constant) self.assertTrue(ms["int16"][SCALAR].load_chunk(o, e).dtype == - np.dtype('int16')) + 'int16') self.assertTrue(ms["int32"][SCALAR].load_chunk(o, e).dtype == - np.dtype('int32')) + 'int32') self.assertTrue(ms["int64"][SCALAR].load_chunk(o, e).dtype == - np.dtype('int64')) + 'int64') self.assertTrue(ms["uint16"][SCALAR].load_chunk(o, e).dtype == - np.dtype('uint16')) + 'uint16') self.assertTrue(ms["uint32"][SCALAR].load_chunk(o, e).dtype == - np.dtype('uint32')) + 'uint32') self.assertTrue(ms["uint64"][SCALAR].load_chunk(o, e).dtype == - np.dtype('uint64')) + 'uint64') self.assertTrue(ms["single"][SCALAR].load_chunk(o, e).dtype == - np.dtype('single')) + 'single') self.assertTrue(ms["double"][SCALAR].load_chunk(o, e).dtype == - np.dtype('double')) + 'double') self.assertTrue(ms["longdouble"][SCALAR].load_chunk(o, e).dtype - == np.dtype('longdouble')) + == 'longdouble') self.assertTrue(ms["complex64"][SCALAR].load_chunk(o, e).dtype - == np.dtype('complex64')) + == 'complex64') self.assertTrue(ms["complex128"][SCALAR].load_chunk(o, e).dtype - == np.dtype('complex128')) + == 'complex128') if file_ending not in ["bp", "bp4", "bp5"]: self.assertTrue(ms["clongdouble"][SCALAR].load_chunk(o, e) - .dtype == np.dtype('clongdouble')) + .dtype == 'clongdouble') # FIXME: why does this even work w/o a flush() ? self.assertEqual(ms["int16"][SCALAR].load_chunk(o, e), @@ -763,18 +763,18 @@ def makeDataRoundTrip(self, file_ending): extent = [42, 24, 11] ms["complex64"][SCALAR].reset_dataset( - DS(np.dtype("complex64"), extent)) + DS("complex64", extent)) ms["complex64"][SCALAR].store_chunk( np.ones(extent, dtype=np.complex64) * np.complex64(1.234 + 2.345j)) ms["complex128"][SCALAR].reset_dataset( - DS(np.dtype("complex128"), extent)) + DS("complex128", extent)) ms["complex128"][SCALAR].store_chunk( np.ones(extent, dtype=np.complex128) * np.complex128(1.234567 + 2.345678j)) if file_ending not in ["bp", "bp4", "bp5"]: ms["clongdouble"][SCALAR].reset_dataset( - DS(np.dtype("clongdouble"), extent)) + DS("clongdouble", extent)) ms["clongdouble"][SCALAR].store_chunk( np.ones(extent, dtype=np.clongdouble) * np.clongdouble(1.23456789 + 2.34567890j)) @@ -795,8 +795,8 @@ def makeDataRoundTrip(self, file_ending): np.testing.assert_almost_equal(it.time, 1.23) np.testing.assert_almost_equal(it.dt, 1.2) # TODO - # self.assertTrue(it.time.dtype == np.dtype('single')) - # self.assertTrue(it.dt.dtype == np.dtype('longdouble')) + # self.assertTrue(it.time.dtype == 'single') + # self.assertTrue(it.dt.dtype == 'longdouble') ms = it.meshes o = [1, 2, 3] @@ -807,11 +807,11 @@ def makeDataRoundTrip(self, file_ending): if file_ending not in ["bp", "bp4", "bp5"]: dc256 = ms["clongdouble"][SCALAR].load_chunk(o, e) - self.assertTrue(dc64.dtype == np.dtype('complex64')) - self.assertTrue(dc128.dtype == np.dtype('complex128')) + self.assertTrue(dc64.dtype == 'complex64') + self.assertTrue(dc128.dtype == 'complex128') if file_ending not in ["bp", "bp4", "bp5"]: self.assertTrue( - dc256.dtype == np.dtype('clongdouble')) + dc256.dtype == 'clongdouble') series.flush() @@ -853,14 +853,14 @@ def makeEmptyRoundTrip(self, file_ending): ms["LONG_DOUBLE"][SCALAR].make_empty(DT.LONG_DOUBLE, 13) if found_numpy: - ms["int16"][SCALAR].make_empty(np.dtype("int16"), 14) - ms["int32"][SCALAR].make_empty(np.dtype("int32"), 15) - ms["int64"][SCALAR].make_empty(np.dtype("int64"), 16) - ms["uint16"][SCALAR].make_empty(np.dtype("uint16"), 17) - ms["uint32"][SCALAR].make_empty(np.dtype("uint32"), 18) - ms["uint64"][SCALAR].make_empty(np.dtype("uint64"), 19) - ms["single"][SCALAR].make_empty(np.dtype("single"), 20) - ms["np_double"][SCALAR].make_empty(np.dtype("double"), 21) + ms["int16"][SCALAR].make_empty("int16", 14) + ms["int32"][SCALAR].make_empty("int32", 15) + ms["int64"][SCALAR].make_empty("int64", 16) + ms["uint16"][SCALAR].make_empty("uint16", 17) + ms["uint32"][SCALAR].make_empty("uint32", 18) + ms["uint64"][SCALAR].make_empty("uint64", 19) + ms["single"][SCALAR].make_empty("single", 20) + ms["np_double"][SCALAR].make_empty("double", 21) # flush and close file self.assertTrue(series) @@ -964,15 +964,15 @@ def makeEmptyRoundTrip(self, file_ending): # test datatypes for fixed-sized types only if found_numpy: - self.assertTrue(ms["int16"][SCALAR].dtype == np.dtype("int16")) - self.assertTrue(ms["int32"][SCALAR].dtype == np.dtype("int32")) - self.assertTrue(ms["int64"][SCALAR].dtype == np.dtype("int64")) - self.assertTrue(ms["uint16"][SCALAR].dtype == np.dtype("uint16")) - self.assertTrue(ms["uint32"][SCALAR].dtype == np.dtype("uint32")) - self.assertTrue(ms["uint64"][SCALAR].dtype == np.dtype("uint64")) - self.assertTrue(ms["single"][SCALAR].dtype == np.dtype("single")) + self.assertTrue(ms["int16"][SCALAR].dtype == "int16") + self.assertTrue(ms["int32"][SCALAR].dtype == "int32") + self.assertTrue(ms["int64"][SCALAR].dtype == "int64") + self.assertTrue(ms["uint16"][SCALAR].dtype == "uint16") + self.assertTrue(ms["uint32"][SCALAR].dtype == "uint32") + self.assertTrue(ms["uint64"][SCALAR].dtype == "uint64") + self.assertTrue(ms["single"][SCALAR].dtype == "single") self.assertTrue( - ms["np_double"][SCALAR].dtype == np.dtype("double")) + ms["np_double"][SCALAR].dtype == "double") def testEmptyRecords(self): backend_filesupport = { @@ -1754,18 +1754,18 @@ def backend_particle_patches(self, file_ending): for r in ["x", "y"]: x = e["position"][r] - x.reset_dataset(DS(np.dtype("single"), extent)) + x.reset_dataset(DS("single", extent)) # implicit: , [0, ], extent x.store_chunk(np.arange(extent[0], dtype=np.single)) o = e["positionOffset"][r] - o.reset_dataset(DS(np.dtype("uint64"), extent)) + o.reset_dataset(DS("uint64", extent)) o.store_chunk(np.arange(extent[0], dtype=np.uint64), [0, ], extent) - dset = DS(np.dtype("uint64"), [num_patches, ]) + dset = DS("uint64", [num_patches, ]) e.particle_patches["numParticles"][SCALAR].reset_dataset(dset) e.particle_patches["numParticlesOffset"][SCALAR].reset_dataset(dset) - dset = DS(np.dtype("single"), [num_patches, ]) + dset = DS("single", [num_patches, ]) e.particle_patches["offset"]["x"].reset_dataset(dset) e.particle_patches["offset"]["y"].reset_dataset(dset) e.particle_patches["extent"]["x"].reset_dataset(dset) @@ -1876,12 +1876,12 @@ def makeCloseIterationRoundTrip(self, file_ending): io.Access_Type.create ) DS = io.Dataset - data = np.array([2, 4, 6, 8], dtype=np.dtype("int")) + data = np.array([2, 4, 6, 8], dtype="int") extent = [4] it0 = series.iterations[0] E_x = it0.meshes["E"]["x"] - E_x.reset_dataset(DS(np.dtype("int"), extent)) + E_x.reset_dataset(DS("int", extent)) E_x.store_chunk(data, [0], extent) it0.close(flush=True) @@ -1902,7 +1902,7 @@ def makeCloseIterationRoundTrip(self, file_ending): it1 = series.iterations[1] E_x = it1.meshes["E"]["x"] - E_x.reset_dataset(DS(np.dtype("int"), extent)) + E_x.reset_dataset(DS("int", extent)) E_x.store_chunk(data, [0], extent) it1.close(flush=False) series.flush() @@ -1944,17 +1944,17 @@ def makeIteratorRoundTrip(self, backend, file_ending): jsonConfig ) DS = io.Dataset - data = np.array([2, 4, 6, 8], dtype=np.dtype("int")) + data = np.array([2, 4, 6, 8], dtype="int") extent = [4] for i in range(10): it = series.write_iterations()[i] E_x = it.meshes["E"]["x"] - E_x.reset_dataset(DS(np.dtype("int"), extent)) + E_x.reset_dataset(DS("int", extent)) E_x.store_chunk(data, [0], extent) B_y = it.meshes["B"]["y"] - B_y.reset_dataset(DS(np.dtype("int"), [2, 2])) + B_y.reset_dataset(DS("int", [2, 2])) span = B_y.store_chunk().current_buffer() span[0, 0] = 0 span[0, 1] = 1 @@ -2014,13 +2014,13 @@ def makeAvailableChunksRoundTrip(self, ext): DS = io.Dataset E_x = write.iterations[0].meshes["E"]["x"] - E_x.reset_dataset(DS(np.dtype("int"), [10, 4])) + E_x.reset_dataset(DS("int", [10, 4])) data = np.array( - [[2, 4, 6, 8], [10, 12, 14, 16]], dtype=np.dtype("int")) + [[2, 4, 6, 8], [10, 12, 14, 16]], dtype="int") E_x.store_chunk(data, [1, 0], [2, 4]) - data2 = np.array([[2, 4], [6, 8], [10, 12]], dtype=np.dtype("int")) + data2 = np.array([[2, 4], [6, 8], [10, 12]], dtype="int") E_x.store_chunk(data2, [4, 2], [3, 2]) - data3 = np.array([[2], [4], [6], [8]], dtype=np.dtype("int")) + data3 = np.array([[2], [4], [6], [8]], dtype="int") E_x.store_chunk(data3, [6, 0], [4, 1]) # Cleaner: write.close() @@ -2059,13 +2059,13 @@ def testAvailableChunks(self): def writeFromTemporaryStore(self, E_x): if found_numpy: - E_x.store_chunk(np.array([[4, 5, 6]], dtype=np.dtype("int")), + E_x.store_chunk(np.array([[4, 5, 6]], dtype="int"), [1, 0]) - data = np.array([[1, 2, 3]], dtype=np.dtype("int")) + data = np.array([[1, 2, 3]], dtype="int") E_x.store_chunk(data) - data2 = np.array([[7, 8, 9]], dtype=np.dtype("int")) + data2 = np.array([[7, 8, 9]], dtype="int") E_x.store_chunk(np.repeat(data2, 198, axis=0), [2, 0]) @@ -2086,7 +2086,7 @@ def writeFromTemporary(self, ext): DS = io.Dataset E_x = write.iterations[0].meshes["E"]["x"] - E_x.reset_dataset(DS(np.dtype("int"), [200, 3])) + E_x.reset_dataset(DS("int", [200, 3])) self.writeFromTemporaryStore(E_x) gc.collect() # trigger removal of temporary data to check its copied @@ -2114,7 +2114,7 @@ def writeFromTemporary(self, ext): np.testing.assert_allclose( r_d[:3, :], np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - dtype=np.dtype("int")) + dtype="int") ) def testWriteFromTemporary(self): @@ -2173,14 +2173,14 @@ def testJsonConfigADIOS2(self): global_config) DS = io.Dataset - data = np.array(range(1000), dtype=np.dtype("double")) + data = np.array(range(1000), dtype="double") E_x = series.iterations[0].meshes["E"]["x"] - E_x.reset_dataset(DS(np.dtype("double"), [1000])) + E_x.reset_dataset(DS("double", [1000])) E_x.store_chunk(data, [0], [1000]) E_y = series.iterations[0].meshes["E"]["y"] - E_y.reset_dataset(DS(np.dtype("double"), [1000], local_config)) + E_y.reset_dataset(DS("double", [1000], local_config)) E_y.store_chunk(data, [0], [1000]) self.assertTrue(series) @@ -2311,7 +2311,7 @@ def testScalarHdf5Fields(self): file = "../samples/scalar_hdf5.h5" series_write = io.Series(file, io.Access.create) E_x = series_write.write_iterations()[0].meshes["E"]["x"] - E_x.reset_dataset(io.Dataset(np.dtype(np.int_), [1])) + E_x.reset_dataset(io.Dataset(np.int_, [1])) E_x[:] = np.array([43]) series_write.close() From 777eb939ffac7c6561fd06fe25ef62953813bd02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 2 Mar 2026 18:12:51 +0100 Subject: [PATCH 5/9] Update examples --- examples/10_streaming_write.py | 4 ++-- examples/12_span_write.py | 2 +- examples/13_write_dynamic_configuration.py | 6 +++--- examples/15_compression.py | 10 +++++----- examples/7_extended_write_serial.py | 4 ++-- examples/9_particle_write_serial.py | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/examples/10_streaming_write.py b/examples/10_streaming_write.py index f840d7d1db..f856292c4c 100755 --- a/examples/10_streaming_write.py +++ b/examples/10_streaming_write.py @@ -42,7 +42,7 @@ length = 10 local_data = np.arange(i * length, (i + 1) * length, - dtype=np.dtype("double")) + dtype="double") for dim in ["x", "y", "z"]: pos = electronPositions[dim] pos.reset_dataset(io.Dataset(local_data.dtype, [length])) @@ -63,7 +63,7 @@ temperature_dataset = temperature # let's say we are in a 3x3 mesh temperature_dataset.reset_dataset( - io.Dataset(np.dtype("double"), [3, 3])) + io.Dataset("double", [3, 3])) # temperature is constant temperature_dataset.make_constant(273.15) diff --git a/examples/12_span_write.py b/examples/12_span_write.py index db99e0a556..0ed881af6b 100644 --- a/examples/12_span_write.py +++ b/examples/12_span_write.py @@ -5,7 +5,7 @@ def span_write(filename): series = io.Series(filename, io.Access_Type.create_linear) - datatype = np.dtype("double") + datatype = "double" length = 10 extent = [length] dataset = io.Dataset(datatype, extent) diff --git a/examples/13_write_dynamic_configuration.py b/examples/13_write_dynamic_configuration.py index 0dc67a8e5c..3998a08704 100644 --- a/examples/13_write_dynamic_configuration.py +++ b/examples/13_write_dynamic_configuration.py @@ -82,7 +82,7 @@ def main(): length = 10 local_data = np.arange(i * length, (i + 1) * length, - dtype=np.dtype("double")) + dtype="double") for dim in ["x", "y", "z"]: pos = electronPositions[dim] pos.reset_dataset(io.Dataset(local_data.dtype, [length])) @@ -122,10 +122,10 @@ def main(): # temperature has no x,y,z components, so skip the last layer: temperature_dataset = temperature # let's say we are in a 3x3 mesh - dataset = io.Dataset(np.dtype("double"), [3, 3], config) + dataset = io.Dataset("double", [3, 3], config) temperature_dataset.reset_dataset(dataset) # temperature is constant - local_data = np.arange(i * 9, (i + 1) * 9, dtype=np.dtype("double")) + local_data = np.arange(i * 9, (i + 1) * 9, dtype="double") local_data = local_data.reshape([3, 3]) temperature_dataset[()] = local_data diff --git a/examples/15_compression.py b/examples/15_compression.py index c7f8e0fe95..c66e8281d9 100644 --- a/examples/15_compression.py +++ b/examples/15_compression.py @@ -46,9 +46,9 @@ def write(filename, config): E.axis_labels = ["x", "y"] for dim in ["x", "y"]: component = E[dim] - component.reset_dataset(opmd.Dataset(np.dtype("float"), [10, 10])) + component.reset_dataset(opmd.Dataset("float", [10, 10])) component[:, :] = np.reshape( - np.arange(i * 100, (i + 1) * 100, dtype=np.dtype("float")), + np.arange(i * 100, (i + 1) * 100, dtype="float"), [10, 10], ) @@ -57,13 +57,13 @@ def write(filename, config): for dim in ["x", "y"]: # Do not bother with a positionOffset position_offset = e["positionOffset"][dim] - position_offset.reset_dataset(opmd.Dataset(np.dtype("int"), [100])) + position_offset.reset_dataset(opmd.Dataset("int", [100])) position_offset.make_constant(0) position = e["position"][dim] - position.reset_dataset(opmd.Dataset(np.dtype("float"), [100])) + position.reset_dataset(opmd.Dataset("float", [100])) position[:] = np.arange( - i * 100, (i + 1) * 100, dtype=np.dtype("float") + i * 100, (i + 1) * 100, dtype="float" ) diff --git a/examples/7_extended_write_serial.py b/examples/7_extended_write_serial.py index 49f5283b59..6e8a395b2b 100755 --- a/examples/7_extended_write_serial.py +++ b/examples/7_extended_write_serial.py @@ -89,7 +89,7 @@ electrons["displacement"]["x"].unit_SI = 1.e-6 del electrons["displacement"] electrons["weighting"] \ - .reset_dataset(Dataset(np.dtype("float32"), extent=[1])) \ + .reset_dataset(Dataset("float32", extent=[1])) \ .make_constant(1.e-5) mesh = cur_it.meshes["lowRez_2D_field"] @@ -128,7 +128,7 @@ d = Dataset(partial_particleOff.dtype, mpiDims) electrons["positionOffset"]["x"].reset_dataset(d) - dset = Dataset(np.dtype("uint64"), extent=[2]) + dset = Dataset("uint64", extent=[2]) electrons.particle_patches["numParticles"].reset_dataset(dset) electrons.particle_patches["numParticlesOffset"]. \ reset_dataset(dset) diff --git a/examples/9_particle_write_serial.py b/examples/9_particle_write_serial.py index 257fea89fc..dd582d4bd7 100644 --- a/examples/9_particle_write_serial.py +++ b/examples/9_particle_write_serial.py @@ -37,14 +37,14 @@ # let's set a weird user-defined record this time electrons["displacement"].unit_dimension = {Unit_Dimension.M: 1} electrons["displacement"].unit_SI = 1.e-6 - dset = Dataset(np.dtype("float64"), extent=[n_particles]) + dset = Dataset("float64", extent=[n_particles]) electrons["displacement"].reset_dataset(dset) electrons["displacement"].make_constant(42.43) # don't like it anymore? remove it with: # del electrons["displacement"] electrons["weighting"] \ - .reset_dataset(Dataset(np.dtype("float32"), extent=[n_particles])) \ + .reset_dataset(Dataset("float32", extent=[n_particles])) \ .make_constant(1.e-5) particlePos_x = np.random.rand(n_particles).astype(np.float32) From 70e15de9c7f43d7720ad1e8c652feb42a3f1597c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 2 Mar 2026 18:20:04 +0100 Subject: [PATCH 6/9] CI fixes ci rerun --- examples/12_span_write.py | 1 - test/python/unittest/API/APITest.py | 22 +++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/examples/12_span_write.py b/examples/12_span_write.py index 0ed881af6b..efe5101785 100644 --- a/examples/12_span_write.py +++ b/examples/12_span_write.py @@ -1,4 +1,3 @@ -import numpy as np import openpmd_api as io diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 025f06dff9..06f5bd4218 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -807,8 +807,8 @@ def makeDataRoundTrip(self, file_ending): if file_ending not in ["bp", "bp4", "bp5"]: dc256 = ms["clongdouble"][SCALAR].load_chunk(o, e) - self.assertTrue(dc64.dtype == 'complex64') - self.assertTrue(dc128.dtype == 'complex128') + self.assertEqual(dc64.dtype, 'complex64') + self.assertEqual(dc128.dtype, 'complex128') if file_ending not in ["bp", "bp4", "bp5"]: self.assertTrue( dc256.dtype == 'clongdouble') @@ -964,15 +964,15 @@ def makeEmptyRoundTrip(self, file_ending): # test datatypes for fixed-sized types only if found_numpy: - self.assertTrue(ms["int16"][SCALAR].dtype == "int16") - self.assertTrue(ms["int32"][SCALAR].dtype == "int32") - self.assertTrue(ms["int64"][SCALAR].dtype == "int64") - self.assertTrue(ms["uint16"][SCALAR].dtype == "uint16") - self.assertTrue(ms["uint32"][SCALAR].dtype == "uint32") - self.assertTrue(ms["uint64"][SCALAR].dtype == "uint64") - self.assertTrue(ms["single"][SCALAR].dtype == "single") - self.assertTrue( - ms["np_double"][SCALAR].dtype == "double") + self.assertEqual(ms["int16"][SCALAR].dtype, "int16") + self.assertEqual(ms["int32"][SCALAR].dtype, "int32") + self.assertEqual(ms["int64"][SCALAR].dtype, "int64") + self.assertEqual(ms["uint16"][SCALAR].dtype, "uint16") + self.assertEqual(ms["uint32"][SCALAR].dtype, "uint32") + self.assertEqual(ms["uint64"][SCALAR].dtype, "uint64") + self.assertEqual(ms["single"][SCALAR].dtype, "single") + self.assertEqual( + ms["np_double"][SCALAR].dtype, "double") def testEmptyRecords(self): backend_filesupport = { From 3616699e70b0ba9a4c353fcbfb851b55b4e99eba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 3 Mar 2026 19:40:19 +0100 Subject: [PATCH 7/9] Apply suggestion from @ax3l Co-authored-by: Axel Huebl --- examples/10_streaming_write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/10_streaming_write.py b/examples/10_streaming_write.py index f856292c4c..b1d1206370 100755 --- a/examples/10_streaming_write.py +++ b/examples/10_streaming_write.py @@ -63,7 +63,7 @@ temperature_dataset = temperature # let's say we are in a 3x3 mesh temperature_dataset.reset_dataset( - io.Dataset("double", [3, 3])) + io.Dataset(local_data.dtype, [3, 3])) # temperature is constant temperature_dataset.make_constant(273.15) From 7c10e93fbf427788f1d371b6258a20af0c4b7a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 4 Mar 2026 11:34:34 +0100 Subject: [PATCH 8/9] move py::object, ditch some needles try-catch blocks --- include/openPMD/binding/python/Numpy.hpp | 30 ++++-------------------- src/binding/python/Dataset.cpp | 10 ++++---- src/binding/python/RecordComponent.cpp | 6 +++-- 3 files changed, 14 insertions(+), 32 deletions(-) diff --git a/include/openPMD/binding/python/Numpy.hpp b/include/openPMD/binding/python/Numpy.hpp index f2c66d3d68..949f24da7e 100644 --- a/include/openPMD/binding/python/Numpy.hpp +++ b/include/openPMD/binding/python/Numpy.hpp @@ -98,11 +98,11 @@ inline Datatype dtype_from_numpy(pybind11::dtype const dt) } } -inline Datatype dtype_from_numpy(pybind11::object const &dt) +inline Datatype dtype_from_numpy(pybind11::object dt) { if (py::isinstance(dt)) { - return dtype_from_numpy(py::cast(dt)); + return dtype_from_numpy(py::cast(std::move(dt))); } else { @@ -119,22 +119,11 @@ inline Datatype dtype_from_numpy(pybind11::object const &dt) "datatype without numpy, failed importing: ") + e.what()); } - pybind11::object create_dtype; - try - { - create_dtype = numpy.attr("dtype"); - } - catch (std::exception const &e) - { - throw std::runtime_error( - std::string( - "dtype_from_numpy: Failed to access numpy.dtype: ") + - e.what()); - } + pybind11::object create_dtype = numpy.attr("dtype"); pybind11::object dtype_obj; try { - dtype_obj = create_dtype(dt); + dtype_obj = create_dtype(std::move(dt)); } catch (std::exception const &e) { @@ -143,16 +132,7 @@ inline Datatype dtype_from_numpy(pybind11::object const &dt) pybind11::str(dt).cast() + std::string(", error: ") + e.what()); } - try - { - return dtype_from_numpy(py::cast(dtype_obj)); - } - catch (std::exception const &e) - { - throw std::runtime_error( - std::string("dtype_from_numpy: Failed to cast to dtype: ") + - e.what()); - } + return dtype_from_numpy(py::cast(dtype_obj)); } } diff --git a/src/binding/python/Dataset.cpp b/src/binding/python/Dataset.cpp index b9a84ca78b..5f278f1f38 100644 --- a/src/binding/python/Dataset.cpp +++ b/src/binding/python/Dataset.cpp @@ -25,6 +25,7 @@ #include "openPMD/binding/python/auxiliary.hpp" #include +#include void init_Dataset(py::module &m) { @@ -37,11 +38,10 @@ void init_Dataset(py::module &m) py::arg("extent"), py::arg("options") = "{}") .def( - py::init( - [](py::object const &dt, Extent e, std::string options) { - auto const d = dtype_from_numpy(dt); - return new Dataset{d, std::move(e), std::move(options)}; - }), + py::init([](py::object dt, Extent e, std::string options) { + auto const d = dtype_from_numpy(std::move(dt)); + return new Dataset{d, std::move(e), std::move(options)}; + }), py::arg("dtype"), py::arg("extent"), py::arg("options") = "{}") diff --git a/src/binding/python/RecordComponent.cpp b/src/binding/python/RecordComponent.cpp index afc2a0e717..03a0eb5dad 100644 --- a/src/binding/python/RecordComponent.cpp +++ b/src/binding/python/RecordComponent.cpp @@ -50,6 +50,7 @@ #include #include #include +#include #include /** Convert a py::tuple of py::slices to Offset & Extent @@ -1011,9 +1012,10 @@ void init_RecordComponent(py::module &m) .def( "make_empty", [](RecordComponent &rc, - pybind11::object const &dt, + pybind11::object dt, uint8_t dimensionality) { - return rc.makeEmpty(dtype_from_numpy(dt), dimensionality); + return rc.makeEmpty( + dtype_from_numpy(std::move(dt)), dimensionality); }) // deprecated: pass-through C++ API From b164b887ad623039c59e6c3738f43c56d5356bbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 4 Mar 2026 12:08:31 +0100 Subject: [PATCH 9/9] Simplify 13_write_dynamic_configuration.py example rerun ci --- examples/13_write_dynamic_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/13_write_dynamic_configuration.py b/examples/13_write_dynamic_configuration.py index 3998a08704..4078497efb 100644 --- a/examples/13_write_dynamic_configuration.py +++ b/examples/13_write_dynamic_configuration.py @@ -122,7 +122,7 @@ def main(): # temperature has no x,y,z components, so skip the last layer: temperature_dataset = temperature # let's say we are in a 3x3 mesh - dataset = io.Dataset("double", [3, 3], config) + dataset = io.Dataset(local_data.dtype, [3, 3], config) temperature_dataset.reset_dataset(dataset) # temperature is constant local_data = np.arange(i * 9, (i + 1) * 9, dtype="double")