From eb091ad9a91060dcac562592a05a813d57c342f2 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Fri, 6 Feb 2026 22:58:19 -0800 Subject: [PATCH 01/14] Convert build system to scikit-build-core - Replace meson files with cmake - Build a stand-alone compiled library and header which are installed to python environment lib / include dirs - Update test and wheel matrix to include python-3.14 and drop support for python-3.9 --- .github/workflows/test.yml | 15 ++-- .github/workflows/wheels.yml | 32 +++---- CMakeLists.txt | 70 +++++++++++++++ cmake/FindFLAC.cmake | 89 +++++++++++++++++++ meson.build | 17 ---- pyproject.toml | 85 +++++++++++++++--- src/CMakeLists.txt | 2 + src/flacarray/CMakeLists.txt | 73 +++++++++++++++ .../libflacarray.pyx => _libflacarray.pyx} | 0 src/flacarray/compress.py | 2 +- src/flacarray/decompress.py | 2 +- src/flacarray/libflacarray/README.md | 13 +-- src/flacarray/libflacarray/meson.build | 24 ----- src/flacarray/meson.build | 27 ------ src/flacarray/scripts/CMakeLists.txt | 8 ++ src/flacarray/scripts/meson.build | 10 --- src/flacarray/tests/CMakeLists.txt | 13 +++ src/flacarray/tests/bindings.py | 2 +- src/flacarray/tests/meson.build | 15 ---- src/flacarray/utils.py | 2 +- src/meson.build | 1 - 21 files changed, 364 insertions(+), 138 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 cmake/FindFLAC.cmake delete mode 100644 meson.build create mode 100644 src/CMakeLists.txt create mode 100644 src/flacarray/CMakeLists.txt rename src/flacarray/{libflacarray/libflacarray.pyx => _libflacarray.pyx} (100%) delete mode 100644 src/flacarray/libflacarray/meson.build delete mode 100644 src/flacarray/meson.build create mode 100644 src/flacarray/scripts/CMakeLists.txt delete mode 100644 src/flacarray/scripts/meson.build create mode 100644 src/flacarray/tests/CMakeLists.txt delete mode 100644 src/flacarray/tests/meson.build delete mode 100644 src/meson.build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aa0fb74..0295661 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,30 +29,30 @@ jobs: matrix: include: - os: ubuntu-latest - python: "3.9" + python: "3.10" arch: Linux-x86_64 ompdisable: 0 - os: ubuntu-latest - python: "3.11" + python: "3.12" arch: Linux-x86_64 ompdisable: 0 - os: ubuntu-latest - python: "3.13" + python: "3.14" arch: Linux-x86_64 ompdisable: 0 - - os: macos-latest + - os: macos-15-intel python: "3.10" arch: MacOSX-x86_64 ompdisable: 1 - - os: macos-latest + - os: macos-15-intel python: "3.13" arch: MacOSX-x86_64 ompdisable: 1 - - os: macos-latest + - os: macos-15 python: "3.10" arch: MacOSX-arm64 ompdisable: 1 - - os: macos-latest + - os: macos-15 python: "3.13" arch: MacOSX-arm64 ompdisable: 1 @@ -85,7 +85,6 @@ jobs: && conda create --yes -n test python==${{ matrix.python }} \ && conda activate test \ && conda install --yes --file packaging/conda_build_requirements.txt - if test ${{ matrix.python }} = "3.9"; then conda install libxcrypt; fi - name: Install run: | diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 34a6534..11892cc 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -21,10 +21,6 @@ jobs: matrix: include: # Linux 64 bit manylinux - - os: ubuntu-latest - arch: x86_64 - python: 39 - builder: manylinux - os: ubuntu-latest arch: x86_64 python: 310 @@ -41,44 +37,50 @@ jobs: arch: x86_64 python: 313 builder: manylinux + - os: ubuntu-latest + arch: x86_64 + python: 314 + builder: manylinux - # MacOS x86_64. The macos-13 runner is the last - # Intel-based runner version. At some point we'll - # need to switch to macos-latest and test cross compiling. - - os: macos-13 + # MacOS x86_64. + - os: macos-15-intel arch: x86_64 python: 310 builder: macosx - - os: macos-13 + - os: macos-15-intel arch: x86_64 python: 311 builder: macosx - - os: macos-13 + - os: macos-15-intel arch: x86_64 python: 312 builder: macosx - - os: macos-13 + - os: macos-15-intel arch: x86_64 python: 313 builder: macosx # MacOS arm64 - - os: macos-latest + - os: macos-15 arch: arm64 python: 310 builder: macosx - - os: macos-latest + - os: macos-15 arch: arm64 python: 311 builder: macosx - - os: macos-latest + - os: macos-15 arch: arm64 python: 312 builder: macosx - - os: macos-latest + - os: macos-15 arch: arm64 python: 313 builder: macosx + - os: macos-15 + arch: arm64 + python: 314 + builder: macosx env: CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }} CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.builder }}2014 diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..535ba51 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,70 @@ +cmake_minimum_required(VERSION 3.21) + +project(${SKBUILD_PROJECT_NAME} LANGUAGES C) + +# Auxiliary files +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") + +# Warn if the user invokes CMake directly +if (NOT SKBUILD) + message(WARNING "\ + This CMake file is meant to be executed using 'scikit-build-core'. + Running it directly will almost certainly not produce the desired + result. If you are a user trying to install this package, use the + command below, which will install all necessary build dependencies, + compile the package in an isolated environment, and then install it. + + $ pip install . + + ") +endif() + +# We are building libraries that will eventually be linked into shared +# modules. All code should be built with PIC. +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# Use modern C +set(CMAKE_C_STANDARD 11) + +# For installing the stand-alone library and header +include(GNUInstallDirs) + +# OpenMP +if(NOT DISABLE_OPENMP) + find_package(OpenMP) +endif() + +find_package(Python COMPONENTS Interpreter Development.Module REQUIRED) +include(UseCython) + +execute_process( + COMMAND "${Python_EXECUTABLE}" -c "import numpy; print(numpy.get_include(), end='')" + OUTPUT_VARIABLE NUMPY_INCLUDE_DIR +) + +find_package(FLAC REQUIRED) + +if(DEFINED FLAC_VERSION) + if(FLAC_VERSION STREQUAL "") + message(STATUS "Cannot determine FLAC version- assuming it is >= 1.4.0") + else() + string(REGEX REPLACE "^([0-9]+)\\.[0-9]+\\..*" "\\1" + FLAC_MAJ_VERSION "${FLAC_VERSION}") + string(REGEX REPLACE "^[0-9]+\\.([0-9]+)\\..*" "\\1" + FLAC_MIN_VERSION "${FLAC_VERSION}") + if(FLAC_MAJ_VERSION GREATER 1) + # Future proofing + message(STATUS "Found FLAC version ${FLAC_VERSION}") + else() + if(FLAC_MIN_VERSION GREATER_EQUAL 4) + message(STATUS "Found FLAC version ${FLAC_VERSION}") + else() + message(FATAL_ERROR "FLAC version ${FLAC_VERSION} is not >= 1.4.0") + endif() + endif() + endif() +else() + message(STATUS "Cannot determine FLAC version- assuming it is >= 1.4.0") +endif() + +add_subdirectory(src) diff --git a/cmake/FindFLAC.cmake b/cmake/FindFLAC.cmake new file mode 100644 index 0000000..d3aced4 --- /dev/null +++ b/cmake/FindFLAC.cmake @@ -0,0 +1,89 @@ +# Copied without modification from libsndfile +# (https://github.com/libsndfile/libsndfile) +# +# (C) Erik de Castro Lopo +# +# Released under LGPL v2.1 +# https://www.gnu.org/licenses/old-licenses/lgpl-2.1.html +# +# +# - Find FLAC +# Find the native FLAC includes and libraries +# +# FLAC_USE_STATIC_LIBS: Search for static libraries. +# +# FLAC_INCLUDE_DIRS - where to find FLAC headers. +# FLAC_LIBRARIES - List of libraries when using libFLAC. +# FLAC_FOUND - True if libFLAC found. +# FLAC_DEFINITIONS - FLAC compile definitons + +# Check whether to search static or dynamic libs +set(CMAKE_FIND_LIBRARY_SUFFIXES_SAV ${CMAKE_FIND_LIBRARY_SUFFIXES}) + +if(${FLAC_USE_STATIC_LIBS}) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_STATIC_LIBRARY_SUFFIX}) +else() + set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_SAV}) +endif() + +if (FLAC_INCLUDE_DIR) + # Already in cache, be silent + set (FLAC_FIND_QUIETLY TRUE) +endif () + +find_package (Ogg QUIET) + +find_package (PkgConfig QUIET) +pkg_check_modules(PC_FLAC QUIET flac) + +set(FLAC_VERSION ${PC_FLAC_VERSION}) + +find_path (FLAC_INCLUDE_DIR FLAC/stream_decoder.h + HINTS + ${PC_FLAC_INCLUDEDIR} + ${PC_FLAC_INCLUDE_DIRS} + ${FLAC_ROOT} + ) + +# MSVC built libraries can name them *_static, which is good as it +# distinguishes import libraries from static libraries with the same extension. +find_library (FLAC_LIBRARY + NAMES + FLAC + libFLAC + libFLAC_dynamic + libFLAC_static + HINTS + ${PC_FLAC_LIBDIR} + ${PC_FLAC_LIBRARY_DIRS} + ${FLAC_ROOT} + ) + +# Handle the QUIETLY and REQUIRED arguments and set FLAC_FOUND to TRUE if +# all listed variables are TRUE. +include (FindPackageHandleStandardArgs) +find_package_handle_standard_args (FLAC + REQUIRED_VARS + FLAC_LIBRARY + FLAC_INCLUDE_DIR + VERSION_VAR + FLAC_VERSION + ) + +if (FLAC_FOUND) + set (FLAC_INCLUDE_DIRS ${FLAC_INCLUDE_DIR}) + set (FLAC_LIBRARIES ${FLAC_LIBRARY} ${OGG_LIBRARIES}) + if (NOT TARGET FLAC::FLAC) + add_library(FLAC::FLAC UNKNOWN IMPORTED) + set_target_properties(FLAC::FLAC PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${FLAC_INCLUDE_DIR}" + IMPORTED_LOCATION "${FLAC_LIBRARY}" + INTERFACE_LINK_LIBRARIES Ogg::ogg + ) + endif () +endif () + +mark_as_advanced(FLAC_INCLUDE_DIR FLAC_LIBRARY) + +# Restore library search suffix +set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_SAV}) diff --git a/meson.build b/meson.build deleted file mode 100644 index 30864ee..0000000 --- a/meson.build +++ /dev/null @@ -1,17 +0,0 @@ -project( - 'flacarray', - 'c', 'cython', - license: 'BSD-2-Clause', - meson_version: '>= 1.0.0', - default_options: [ - 'buildtype=release', - 'c_std=c11', - ] -) - -openmp = dependency('openmp', required: false) -libflac = dependency('flac', version: '>= 1.4.0', static: false) - -py = import('python').find_installation(pure: false) - -subdir('src') diff --git a/pyproject.toml b/pyproject.toml index 983e39c..adb9394 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,7 @@ + [build-system] -requires = [ - "cython >= 3.0.2", - "meson >= 1.2.3", - "meson-python >= 0.14.0", - "numpy", -] -build-backend = "mesonpy" +requires = ["scikit-build-core", "cython", "cython-cmake", "numpy"] +build-backend = "scikit_build_core.build" [project] name = "flacarray" @@ -15,7 +11,7 @@ readme = "README.md" maintainers = [ { name = "Theodore Kisner", email = "tskisner.public@gmail.com" }, ] -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "numpy", ] @@ -25,11 +21,11 @@ classifiers = [ "Programming Language :: C", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Software Development :: Libraries", ] @@ -41,5 +37,72 @@ flacarray_benchmark = "flacarray.scripts:bench_cli" "Source" = "https://github.com/hpc4cmb/flacarray/" "Issue Tracker" = "https://github.com/hpc4cmb/flacarray/issues" -[tool.meson-python.args] -compile = ['-j2'] +[tool.scikit-build] +# A list of args to pass to CMake when configuring the project. +cmake.args = [] +# A table of defines to pass to CMake when configuring the project. Additive. +cmake.define = {} +# The build type to use when building the project. +cmake.build-type = "Release" +# The source directory to use when building the project. +cmake.source-dir = "." +# Use Make as a fallback if a suitable Ninja executable is not found. +ninja.make-fallback = true +# The logging level to display. +logging.level = "WARNING" +# Files to include in the SDist even if they are skipped by default. +sdist.include = [] +# Files to exclude from the SDist even if they are included by default. +sdist.exclude = [] +# Try to build a reproducible distribution. +sdist.reproducible = true +# If set to True, CMake will be run before building the SDist. +sdist.cmake = false +# A list of packages to auto-copy into the wheel. +wheel.packages = [] +# Fill out extra tags that are not required. +wheel.expand-macos-universal-tags = false +# The CMake install prefix relative to the platlib wheel path. +wheel.install-dir = "" +# A list of license files to include in the wheel. Supports glob patterns. +wheel.license-files = ["LICENSE"] +# Run CMake as part of building the wheel. +wheel.cmake = true +# Target the platlib or the purelib. +wheel.platlib = "" +# A set of patterns to exclude from the wheel. +wheel.exclude = [] +# The build tag to use for the wheel. If empty, no build tag is used. +wheel.build-tag = "" +# If CMake is less than this value, backport a copy of FindPython. +backport.find-python = "3.26.1" +# Select the editable mode to use. Can be "redirect" (default) or "inplace". +editable.mode = "redirect" +# Turn on verbose output for the editable mode rebuilds. +editable.verbose = true +# Rebuild the project when the package is imported. +editable.rebuild = false +# Extra args to pass directly to the builder in the build step. +build.tool-args = [] +# The build targets to use when building the project. +build.targets = [] +# Verbose printout when building. +build.verbose = false +# Additional ``build-system.requires``. +build.requires = [] +# The components to install. +install.components = [] +# Whether to strip the binaries. +install.strip = true +# Add the python build environment site_packages folder to the CMake prefix paths. +search.site-packages = true +# List dynamic metadata fields and hook locations in this table. +metadata = {} +# Strictly check all config options. +strict-config = true +# Enable early previews of features not finalized yet. +experimental = false +# If set, this will provide a method for backward compatibility. +minimum-version = "0.11" # current version +# The CMake build directory. Defaults to a unique temporary directory. +build-dir = "build" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..3be3fa3 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,2 @@ + +add_subdirectory(flacarray) diff --git a/src/flacarray/CMakeLists.txt b/src/flacarray/CMakeLists.txt new file mode 100644 index 0000000..dd01629 --- /dev/null +++ b/src/flacarray/CMakeLists.txt @@ -0,0 +1,73 @@ + +# Create an object library of the compiled sources. + +set(libflcarr_SOURCES + libflacarray/compress.c + libflacarray/decompress.c + libflacarray/utils.c +) + +set(libflcarr_HEADERS + libflacarray/flacarray.h +) + +add_library(flcarr OBJECT ${libflcarr_SOURCES}) + +target_compile_options(flcarr PRIVATE "${OpenMP_C_FLAGS}") + +target_include_directories(flcarr BEFORE PUBLIC + "${CMAKE_CURRENT_SOURCE_DIR}/libflacarray" +) + +target_include_directories(flcarr PUBLIC "${FLAC_INCLUDE_DIRS}") +target_link_libraries(flcarr PUBLIC "${FLAC_LIBRARIES}") + +target_link_libraries(flcarr PUBLIC "${OpenMP_C_LIBRARIES}") + +# Create an actual installed library, usable by external compiled software + +add_library(flacarray SHARED) +target_link_libraries(flacarray PRIVATE flcarr) +set_target_properties(flacarray PROPERTIES PUBLIC_HEADER ${libflcarr_HEADERS}) + +install(TARGETS flacarray + LIBRARY DESTINATION "${SKBUILD_DATA_DIR}/lib" + PUBLIC_HEADER DESTINATION "${SKBUILD_DATA_DIR}/include" +) + +# Create a compiled python extension + +cython_transpile(_libflacarray.pyx LANGUAGE C OUTPUT_VARIABLE libflacarray_c) + +Python_add_library(_libflacarray MODULE "${libflacarray_c}" WITH_SOABI) + +target_link_libraries(_libflacarray PRIVATE flcarr) + +target_include_directories(_libflacarray PRIVATE ${NUMPY_INCLUDE_DIR}) + +install(TARGETS _libflacarray DESTINATION ${SKBUILD_PROJECT_NAME}) + +# Install python files + +set(PYTHON_SOURCES + __init__.py + array.py + compress.py + decompress.py + demo.py + hdf5_load_v0.py + hdf5_load_v1.py + hdf5_utils.py + hdf5.py + io_common.py + mpi.py + utils.py + zarr_load_v0.py + zarr_load_v1.py + zarr.py +) + +install(FILES ${PYTHON_SOURCES} DESTINATION ${SKBUILD_PROJECT_NAME}) + +add_subdirectory(scripts) +add_subdirectory(tests) diff --git a/src/flacarray/libflacarray/libflacarray.pyx b/src/flacarray/_libflacarray.pyx similarity index 100% rename from src/flacarray/libflacarray/libflacarray.pyx rename to src/flacarray/_libflacarray.pyx diff --git a/src/flacarray/compress.py b/src/flacarray/compress.py index 2bba38b..5ba0d64 100644 --- a/src/flacarray/compress.py +++ b/src/flacarray/compress.py @@ -4,7 +4,7 @@ import numpy as np -from .libflacarray import encode_flac +from ._libflacarray import encode_flac from .utils import float_to_int, function_timer diff --git a/src/flacarray/decompress.py b/src/flacarray/decompress.py index 9284257..73c3a74 100644 --- a/src/flacarray/decompress.py +++ b/src/flacarray/decompress.py @@ -4,7 +4,7 @@ import numpy as np -from .libflacarray import decode_flac +from ._libflacarray import decode_flac from .utils import ( int_to_float, keep_select, diff --git a/src/flacarray/libflacarray/README.md b/src/flacarray/libflacarray/README.md index 36dbb90..dd1ae5b 100644 --- a/src/flacarray/libflacarray/README.md +++ b/src/flacarray/libflacarray/README.md @@ -1,10 +1,11 @@ -# Compiled Extension +# Compiled Library -This directory contains sources for the `libflacarray` compiled extension. -There are some standard C source files and a Cython wrapper. This extension is -built by the top-level `project.toml` file using meson-python. You should not -have to modify any files in this directory unless you are working with the code -that directly calls libFLAC. +This directory contains sources for the `libflacarray` compiled library. +There are some standard C source files and the output library is wrapped by +Cython in the parent directory. + +You should not have to modify any files in this directory unless you are working +with the code that directly calls libFLAC. ## Developer Notes diff --git a/src/flacarray/libflacarray/meson.build b/src/flacarray/libflacarray/meson.build deleted file mode 100644 index 0652573..0000000 --- a/src/flacarray/libflacarray/meson.build +++ /dev/null @@ -1,24 +0,0 @@ - -incdir_numpy = run_command(py, - [ - '-c', - 'import numpy as np; print(np.get_include())', - ], - check: true -).stdout().strip() - -ext_sources = [ - 'libflacarray.pyx', - 'utils.c', - 'compress.c', - 'decompress.c', -] - -py.extension_module( - 'libflacarray', - ext_sources, - dependencies: [openmp, libflac], - include_directories: [incdir_numpy], - install: true, - subdir: 'flacarray', -) diff --git a/src/flacarray/meson.build b/src/flacarray/meson.build deleted file mode 100644 index 08fccab..0000000 --- a/src/flacarray/meson.build +++ /dev/null @@ -1,27 +0,0 @@ - -python_sources = [ - '__init__.py', - 'array.py', - 'compress.py', - 'decompress.py', - 'utils.py', - 'hdf5.py', - 'hdf5_utils.py', - 'hdf5_load_v0.py', - 'hdf5_load_v1.py', - 'mpi.py', - 'demo.py', - 'zarr.py', - 'zarr_load_v0.py', - 'zarr_load_v1.py', - 'io_common.py', -] - -py.install_sources( - python_sources, - subdir: 'flacarray' -) - -subdir('libflacarray') -subdir('tests') -subdir('scripts') diff --git a/src/flacarray/scripts/CMakeLists.txt b/src/flacarray/scripts/CMakeLists.txt new file mode 100644 index 0000000..971566e --- /dev/null +++ b/src/flacarray/scripts/CMakeLists.txt @@ -0,0 +1,8 @@ +# Install python files + +set(PYTHON_SOURCES + __init__.py + benchmark.py +) + +install(FILES ${PYTHON_SOURCES} DESTINATION "${SKBUILD_PROJECT_NAME}/scripts/") diff --git a/src/flacarray/scripts/meson.build b/src/flacarray/scripts/meson.build deleted file mode 100644 index e49b1ab..0000000 --- a/src/flacarray/scripts/meson.build +++ /dev/null @@ -1,10 +0,0 @@ - -python_sources = [ - '__init__.py', - 'benchmark.py', -] - -py.install_sources( - python_sources, - subdir: 'flacarray/scripts' -) diff --git a/src/flacarray/tests/CMakeLists.txt b/src/flacarray/tests/CMakeLists.txt new file mode 100644 index 0000000..814b50c --- /dev/null +++ b/src/flacarray/tests/CMakeLists.txt @@ -0,0 +1,13 @@ +# Install python files + +set(PYTHON_SOURCES + __init__.py + array.py + bindings.py + hdf5.py + runner.py + utils.py + zarr.py +) + +install(FILES ${PYTHON_SOURCES} DESTINATION "${SKBUILD_PROJECT_NAME}/tests/") diff --git a/src/flacarray/tests/bindings.py b/src/flacarray/tests/bindings.py index c3bcace..a0e5d0e 100644 --- a/src/flacarray/tests/bindings.py +++ b/src/flacarray/tests/bindings.py @@ -7,7 +7,7 @@ import numpy as np -from ..libflacarray import ( +from .._libflacarray import ( wrap_encode_i32, wrap_encode_i32_threaded, wrap_encode_i64, diff --git a/src/flacarray/tests/meson.build b/src/flacarray/tests/meson.build deleted file mode 100644 index bde8a9e..0000000 --- a/src/flacarray/tests/meson.build +++ /dev/null @@ -1,15 +0,0 @@ - -python_sources = [ - '__init__.py', - 'runner.py', - 'bindings.py', - 'array.py', - 'utils.py', - 'hdf5.py', - 'zarr.py', -] - -py.install_sources( - python_sources, - subdir: 'flacarray/tests' -) diff --git a/src/flacarray/utils.py b/src/flacarray/utils.py index ffacc7b..ba8dcea 100644 --- a/src/flacarray/utils.py +++ b/src/flacarray/utils.py @@ -10,7 +10,7 @@ import numpy as np -from .libflacarray import ( +from ._libflacarray import ( wrap_float32_to_int32, wrap_float64_to_int64, wrap_int32_to_float32, diff --git a/src/meson.build b/src/meson.build deleted file mode 100644 index f0fccce..0000000 --- a/src/meson.build +++ /dev/null @@ -1 +0,0 @@ -subdir('flacarray') From c4089e8c005a30f5e7997ba4aca14e12d68e0126 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Sat, 7 Feb 2026 00:16:53 -0800 Subject: [PATCH 02/14] Restore workaround for Python.h include of crypt.h --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0295661..fbbe01f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -85,6 +85,7 @@ jobs: && conda create --yes -n test python==${{ matrix.python }} \ && conda activate test \ && conda install --yes --file packaging/conda_build_requirements.txt + if test ${{ matrix.python }} = "3.10"; then conda install libxcrypt; fi - name: Install run: | From 50b15afa6a2676e19669bd82adffa0dfeada9b52 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Sat, 7 Feb 2026 00:31:45 -0800 Subject: [PATCH 03/14] Bump manylinux docker version --- .github/workflows/wheels.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 11892cc..5c82c2f 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -83,8 +83,8 @@ jobs: builder: macosx env: CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }} - CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.builder }}2014 - CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.builder }}2014 + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.builder }}_2_28 + CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.builder }}_2_28 CIBW_BUILD_VERBOSITY: 3 CIBW_ENVIRONMENT_LINUX: > CC=gcc @@ -102,11 +102,11 @@ jobs: - uses: actions/setup-python@v6 name: Install Python with: - python-version: '3.11' + python-version: '3.13' - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==2.22.0 + python -m pip install cibuildwheel==3.3.1 - name: Build wheel run: | From fd62c11d733075393358fe1fdd1ebd6a661c7432 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Mon, 9 Feb 2026 08:27:33 -0800 Subject: [PATCH 04/14] Fix wheel builds --- docs/docs/reference.md | 97 +++++++++++++++++++++ packaging/conda_build_requirements.txt | 4 +- packaging/pip_build_requirements.txt | 4 +- packaging/wheels/test_local_cibuildwheel.sh | 4 +- pyproject.toml | 2 +- src/flacarray/CMakeLists.txt | 6 +- src/flacarray/scripts/CMakeLists.txt | 2 +- 7 files changed, 108 insertions(+), 11 deletions(-) diff --git a/docs/docs/reference.md b/docs/docs/reference.md index f8321fa..60d7fd4 100644 --- a/docs/docs/reference.md +++ b/docs/docs/reference.md @@ -62,3 +62,100 @@ bytestream and auxiliary arrays and convert to / from numpy arrays. ::: flacarray.decompress.array_decompress ::: flacarray.decompress.array_decompress_slice + +## Compiled Library Interfaces + +The lowest-level functions in the compiled C code are exposed in the `flacarray.h` header file and the `libflacarray` library. These are installed along with the python package, and can be linked into compiled software products. + +int encode_i32( + int32_t * data, + int64_t n_stream, + int64_t stream_size, + uint32_t level, + int64_t * n_bytes, + int64_t * starts, + unsigned char ** rawbytes + ) + int encode_i32_threaded( + int32_t * data, + int64_t n_stream, + int64_t stream_size, + uint32_t level, + int64_t * n_bytes, + int64_t * starts, + unsigned char ** rawbytes + ) + int encode_i64( + int64_t * data, + int64_t n_stream, + int64_t stream_size, + uint32_t level, + int64_t * n_bytes, + int64_t * starts, + unsigned char ** rawbytes + ) + int encode_i64_threaded( + int64_t * data, + int64_t n_stream, + int64_t stream_size, + uint32_t level, + int64_t * n_bytes, + int64_t * starts, + unsigned char ** rawbytes + ) + int decode_i32( + unsigned char * rawbytes, + int64_t * starts, + int64_t * nbytes, + int64_t n_stream, + int64_t stream_size, + int64_t first_sample, + int64_t last_sample, + int32_t * data, + bool use_threads + ) + int decode_i64( + unsigned char * rawbytes, + int64_t * starts, + int64_t * nbytes, + int64_t n_stream, + int64_t stream_size, + int64_t first_sample, + int64_t last_sample, + int64_t * data, + bool use_threads + ) + int float32_to_int32( + float * input, + int64_t n_stream, + int64_t stream_size, + float * quanta, + int32_t * output, + float * offsets, + float * gains + ) + int float64_to_int64( + double * input, + int64_t n_stream, + int64_t stream_size, + double * quanta, + int64_t * output, + double * offsets, + double * gains + ) + void int64_to_float64( + int64_t * input, + int64_t n_stream, + int64_t stream_size, + double * offsets, + double * gains, + double * output + ) + void int32_to_float32( + int32_t * input, + int64_t n_stream, + int64_t stream_size, + float * offsets, + float * gains, + float * output + ) diff --git a/packaging/conda_build_requirements.txt b/packaging/conda_build_requirements.txt index 83db878..efdf1ec 100644 --- a/packaging/conda_build_requirements.txt +++ b/packaging/conda_build_requirements.txt @@ -2,8 +2,8 @@ c-compiler pkg-config cmake cython -meson -meson-python +scikit-build-core +cython-cmake libflac h5py zarr diff --git a/packaging/pip_build_requirements.txt b/packaging/pip_build_requirements.txt index 5ab72e7..b205749 100644 --- a/packaging/pip_build_requirements.txt +++ b/packaging/pip_build_requirements.txt @@ -1,7 +1,7 @@ cmake cython -meson -meson-python +scikit-build-core +cython-cmake numpy h5py zarr diff --git a/packaging/wheels/test_local_cibuildwheel.sh b/packaging/wheels/test_local_cibuildwheel.sh index d6306cb..ed83fa5 100755 --- a/packaging/wheels/test_local_cibuildwheel.sh +++ b/packaging/wheels/test_local_cibuildwheel.sh @@ -10,8 +10,8 @@ # docker exec -it /bin/bash # -export CIBW_BUILD="cp311-manylinux_x86_64" -export CIBW_MANYLINUX_X86_64_IMAGE="manylinux2014" +export CIBW_BUILD="cp313-manylinux_x86_64" +export CIBW_MANYLINUX_X86_64_IMAGE="manylinux_2_28" export CIBW_BUILD_VERBOSITY=3 # Uncomment to leave the container for debugging diff --git a/pyproject.toml b/pyproject.toml index adb9394..2736fe7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ wheel.license-files = ["LICENSE"] # Run CMake as part of building the wheel. wheel.cmake = true # Target the platlib or the purelib. -wheel.platlib = "" +wheel.platlib = true # A set of patterns to exclude from the wheel. wheel.exclude = [] # The build tag to use for the wheel. If empty, no build tag is used. diff --git a/src/flacarray/CMakeLists.txt b/src/flacarray/CMakeLists.txt index dd01629..a7e1d02 100644 --- a/src/flacarray/CMakeLists.txt +++ b/src/flacarray/CMakeLists.txt @@ -31,8 +31,8 @@ target_link_libraries(flacarray PRIVATE flcarr) set_target_properties(flacarray PROPERTIES PUBLIC_HEADER ${libflcarr_HEADERS}) install(TARGETS flacarray - LIBRARY DESTINATION "${SKBUILD_DATA_DIR}/lib" - PUBLIC_HEADER DESTINATION "${SKBUILD_DATA_DIR}/include" + LIBRARY DESTINATION "${SKBUILD_PROJECT_NAME}/lib" + PUBLIC_HEADER DESTINATION "${SKBUILD_PROJECT_NAME}/include" ) # Create a compiled python extension @@ -45,7 +45,7 @@ target_link_libraries(_libflacarray PRIVATE flcarr) target_include_directories(_libflacarray PRIVATE ${NUMPY_INCLUDE_DIR}) -install(TARGETS _libflacarray DESTINATION ${SKBUILD_PROJECT_NAME}) +install(TARGETS _libflacarray LIBRARY DESTINATION ${SKBUILD_PROJECT_NAME}) # Install python files diff --git a/src/flacarray/scripts/CMakeLists.txt b/src/flacarray/scripts/CMakeLists.txt index 971566e..9e7cfc5 100644 --- a/src/flacarray/scripts/CMakeLists.txt +++ b/src/flacarray/scripts/CMakeLists.txt @@ -5,4 +5,4 @@ set(PYTHON_SOURCES benchmark.py ) -install(FILES ${PYTHON_SOURCES} DESTINATION "${SKBUILD_PROJECT_NAME}/scripts/") +install(FILES ${PYTHON_SOURCES} DESTINATION "${SKBUILD_PROJECT_NAME}/scripts") From a63925a866bc8f5a0fef270c74693862492b7422 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Mon, 9 Feb 2026 08:28:45 -0800 Subject: [PATCH 05/14] create compiled test --- src/flacarray/libflacarray/low_level.mk | 22 ------------------- .../libflacarray/{test_low_level.c => test.c} | 0 2 files changed, 22 deletions(-) delete mode 100644 src/flacarray/libflacarray/low_level.mk rename src/flacarray/libflacarray/{test_low_level.c => test.c} (100%) diff --git a/src/flacarray/libflacarray/low_level.mk b/src/flacarray/libflacarray/low_level.mk deleted file mode 100644 index ca5831b..0000000 --- a/src/flacarray/libflacarray/low_level.mk +++ /dev/null @@ -1,22 +0,0 @@ -CC ?= gcc -#CFLAGS = -O0 -g -CFLAGS = -O3 -fopenmp -INCLUDE = -I$(CONDA_PREFIX)/include -LDFLAGS = -fopenmp -#LDFLAGS = -LIBRARIES = -L$(CONDA_PREFIX)/lib -lFLAC - -OBJ = test_low_level.o utils.o compress.o decompress.o verify.o - - -all : test_low_level - -test_low_level : $(OBJ) - $(CC) -o $@ $(OBJ) $(LDFLAGS) $(LIBRARIES) - -%.o : %.c flacarray.h - $(CC) $(CFLAGS) $(INCLUDE) -I. -c $< - -clean : - @rm -f test_low_level *.o - diff --git a/src/flacarray/libflacarray/test_low_level.c b/src/flacarray/libflacarray/test.c similarity index 100% rename from src/flacarray/libflacarray/test_low_level.c rename to src/flacarray/libflacarray/test.c From 77cca120ada158f03c83e4f508d142017b589a77 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Mon, 9 Feb 2026 08:30:06 -0800 Subject: [PATCH 06/14] Move verify function into tests --- src/flacarray/libflacarray/test.c | 145 ++++++++++++++++++++++++- src/flacarray/libflacarray/verify.c | 162 ---------------------------- 2 files changed, 144 insertions(+), 163 deletions(-) delete mode 100644 src/flacarray/libflacarray/verify.c diff --git a/src/flacarray/libflacarray/test.c b/src/flacarray/libflacarray/test.c index 44c5040..8b6fe5a 100644 --- a/src/flacarray/libflacarray/test.c +++ b/src/flacarray/libflacarray/test.c @@ -9,6 +9,7 @@ // Local function for debugging and testing + int verify( unsigned char * const bytes, int64_t * const starts, @@ -18,7 +19,149 @@ int verify( uint32_t n_channels, int64_t first_sample, int64_t last_sample -); +) { + // Verify the requested sample range. + int64_t first_decode = 0; + int64_t n_decode = stream_size; + if ((first_sample >= 0) && (last_sample >= 0)) { + // We are decoding a slice of samples. + if (last_sample > stream_size) { + return ERROR_DECODE_SAMPLE_RANGE; + } + if (first_sample > stream_size - 1) { + return ERROR_DECODE_SAMPLE_RANGE; + } + if (first_sample >= last_sample) { + return ERROR_DECODE_SAMPLE_RANGE; + } + first_decode = first_sample; + n_decode = last_sample - first_sample; + } + + // This tracks the failures across all threads. + int errors = ERROR_NONE; + + // Decoder + FLAC__StreamDecoder * decoder; + bool success; + FLAC__StreamDecoderInitStatus status; + + // Decode buffers + dec_callback_data callback_data; + callback_data.input = bytes; + callback_data.n_stream = n_stream; + callback_data.n_decode = n_decode; + callback_data.n_channels = n_channels; + callback_data.err = ERROR_NONE; + + // Allocate a temporary data buffer + int32_t * decompressed = (int32_t *)malloc( + n_stream * n_decode * n_channels * sizeof(int32_t)); + if (decompressed == NULL) { + fprintf(stderr, "Failed to allocate temp data for verification\n"); + } + + for (int64_t istream = 0; istream < n_stream; ++istream) { + if (errors != ERROR_NONE) { + // We already had a failure, skip over remaining loop iterations + continue; + } + fprintf(stderr, "Verifying stream %ld:\n", istream); + fprintf(stderr, " start byte = %ld\n", starts[istream]); + fprintf(stderr, " end byte = %ld\n", starts[istream] + nbytes[istream]); + fprintf( + stderr, " output start element = %ld\n", istream * n_decode * n_channels + ); + callback_data.cur_stream = istream; + callback_data.stream_start = starts[istream]; + callback_data.stream_end = starts[istream] + nbytes[istream]; + callback_data.stream_pos = starts[istream]; + callback_data.decomp_nelem = 0; + // Set the output buffer to the address of the beginning of this stream. + callback_data.decompressed = decompressed + istream * n_decode * n_channels; + + decoder = FLAC__stream_decoder_new(); + + status = FLAC__stream_decoder_init_stream( + decoder, + dec_read_callback, + dec_seek_callback, + dec_tell_callback, + dec_length_callback, + dec_eof_callback, + dec_write_callback, + NULL, + dec_err_callback, + (void *)&callback_data + ); + if (status != FLAC__STREAM_DECODER_INIT_STATUS_OK) { + errors |= ERROR_DECODE_INIT; + continue; + } + if (n_decode == stream_size) { + // We are decoding all samples + fprintf( + stderr, + " decoding all samples (n_decode = %ld, stream_size = %ld)\n", + n_decode, + stream_size + ); + success = FLAC__stream_decoder_process_until_end_of_stream(decoder); + fprintf(stderr, " success = %d\n", (int)success); + if (!success) { + errors |= ERROR_DECODE_PROCESS; + continue; + } + } else { + // We are decoding a slice of samples. Seek to the start. + fprintf( + stderr, + " decoding slice of samples starting at %ld, seeking...\n", + first_decode + ); + success = FLAC__stream_decoder_seek_absolute(decoder, first_decode); + fprintf(stderr, " success = %d\n", (int)success); + if (!success) { + errors |= ERROR_DECODE_PROCESS; + continue; + } + // Process single frames until we have accumulated at least the desired + // number of output samples. + int64_t curframe = 0; + while ( + callback_data.decomp_nelem < n_decode + ) { + fprintf( + stderr, + " decoding frame %ld, decomp_nelem = %ld, n_decode = %ld\n", + curframe, + callback_data.decomp_nelem, + n_decode + ); + success = FLAC__stream_decoder_process_single(decoder); + fprintf(stderr, " success = %d\n", (int)success); + if (!success) { + errors |= ERROR_DECODE_PROCESS; + continue; + } + curframe++; + } + } + success = FLAC__stream_decoder_finish(decoder); + if (!success) { + errors |= ERROR_DECODE_FINISH; + continue; + } + FLAC__stream_decoder_delete(decoder); + + // Merge any errors from the decoder callback + errors |= callback_data.err; + } + + free(decompressed); + + return errors; +} void test_32bit() { diff --git a/src/flacarray/libflacarray/verify.c b/src/flacarray/libflacarray/verify.c deleted file mode 100644 index 67057d6..0000000 --- a/src/flacarray/libflacarray/verify.c +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright (c) 2024-2025 by the parties listed in the AUTHORS file. -// All rights reserved. Use of this source code is governed by -// a BSD-style license that can be found in the LICENSE file. - -#include - - -// Verify function. - -int verify( - unsigned char * const bytes, - int64_t * const starts, - int64_t * const nbytes, - int64_t n_stream, - int64_t stream_size, - uint32_t n_channels, - int64_t first_sample, - int64_t last_sample -) { - // Verify the requested sample range. - int64_t first_decode = 0; - int64_t n_decode = stream_size; - if ((first_sample >= 0) && (last_sample >= 0)) { - // We are decoding a slice of samples. - if (last_sample > stream_size) { - return ERROR_DECODE_SAMPLE_RANGE; - } - if (first_sample > stream_size - 1) { - return ERROR_DECODE_SAMPLE_RANGE; - } - if (first_sample >= last_sample) { - return ERROR_DECODE_SAMPLE_RANGE; - } - first_decode = first_sample; - n_decode = last_sample - first_sample; - } - - // This tracks the failures across all threads. - int errors = ERROR_NONE; - - // Decoder - FLAC__StreamDecoder * decoder; - bool success; - FLAC__StreamDecoderInitStatus status; - - // Decode buffers - dec_callback_data callback_data; - callback_data.input = bytes; - callback_data.n_stream = n_stream; - callback_data.n_decode = n_decode; - callback_data.n_channels = n_channels; - callback_data.err = ERROR_NONE; - - // Allocate a temporary data buffer - int32_t * decompressed = (int32_t *)malloc( - n_stream * n_decode * n_channels * sizeof(int32_t)); - if (decompressed == NULL) { - fprintf(stderr, "Failed to allocate temp data for verification\n"); - } - - for (int64_t istream = 0; istream < n_stream; ++istream) { - if (errors != ERROR_NONE) { - // We already had a failure, skip over remaining loop iterations - continue; - } - fprintf(stderr, "Verifying stream %ld:\n", istream); - fprintf(stderr, " start byte = %ld\n", starts[istream]); - fprintf(stderr, " end byte = %ld\n", starts[istream] + nbytes[istream]); - fprintf( - stderr, " output start element = %ld\n", istream * n_decode * n_channels - ); - callback_data.cur_stream = istream; - callback_data.stream_start = starts[istream]; - callback_data.stream_end = starts[istream] + nbytes[istream]; - callback_data.stream_pos = starts[istream]; - callback_data.decomp_nelem = 0; - // Set the output buffer to the address of the beginning of this stream. - callback_data.decompressed = decompressed + istream * n_decode * n_channels; - - decoder = FLAC__stream_decoder_new(); - - status = FLAC__stream_decoder_init_stream( - decoder, - dec_read_callback, - dec_seek_callback, - dec_tell_callback, - dec_length_callback, - dec_eof_callback, - dec_write_callback, - NULL, - dec_err_callback, - (void *)&callback_data - ); - if (status != FLAC__STREAM_DECODER_INIT_STATUS_OK) { - errors |= ERROR_DECODE_INIT; - continue; - } - if (n_decode == stream_size) { - // We are decoding all samples - fprintf( - stderr, - " decoding all samples (n_decode = %ld, stream_size = %ld)\n", - n_decode, - stream_size - ); - success = FLAC__stream_decoder_process_until_end_of_stream(decoder); - fprintf(stderr, " success = %d\n", (int)success); - if (!success) { - errors |= ERROR_DECODE_PROCESS; - continue; - } - } else { - // We are decoding a slice of samples. Seek to the start. - fprintf( - stderr, - " decoding slice of samples starting at %ld, seeking...\n", - first_decode - ); - success = FLAC__stream_decoder_seek_absolute(decoder, first_decode); - fprintf(stderr, " success = %d\n", (int)success); - if (!success) { - errors |= ERROR_DECODE_PROCESS; - continue; - } - // Process single frames until we have accumulated at least the desired - // number of output samples. - int64_t curframe = 0; - while ( - callback_data.decomp_nelem < n_decode - ) { - fprintf( - stderr, - " decoding frame %ld, decomp_nelem = %ld, n_decode = %ld\n", - curframe, - callback_data.decomp_nelem, - n_decode - ); - success = FLAC__stream_decoder_process_single(decoder); - fprintf(stderr, " success = %d\n", (int)success); - if (!success) { - errors |= ERROR_DECODE_PROCESS; - continue; - } - curframe++; - } - } - success = FLAC__stream_decoder_finish(decoder); - if (!success) { - errors |= ERROR_DECODE_FINISH; - continue; - } - FLAC__stream_decoder_delete(decoder); - - // Merge any errors from the decoder callback - errors |= callback_data.err; - } - - free(decompressed); - - return errors; -} - From 2906ff8de30cb8e3e5b7ee07ce306f7c30ceaa77 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Mon, 9 Feb 2026 14:31:47 -0800 Subject: [PATCH 07/14] generate version with setuptools_scm. Propagate that to python and C. --- .gitignore | 4 ++ CMakeLists.txt | 11 +++--- pyproject.toml | 33 +++++++++++----- src/flacarray/CMakeLists.txt | 23 ++++++++++- src/flacarray/__init__.py | 3 +- src/flacarray/generate_version.sh | 27 +++++++++++++ src/flacarray/libflacarray/README.md | 24 +---------- src/flacarray/libflacarray/test.c | 59 +++++++++++++++------------- 8 files changed, 116 insertions(+), 68 deletions(-) create mode 100755 src/flacarray/generate_version.sh diff --git a/.gitignore b/.gitignore index f4c4476..c5b599d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# Generated files +src/flacarray/_version.py +src/flacarray/libflacarray/version.c + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/CMakeLists.txt b/CMakeLists.txt index 535ba51..7bf3aae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,10 +1,5 @@ cmake_minimum_required(VERSION 3.21) -project(${SKBUILD_PROJECT_NAME} LANGUAGES C) - -# Auxiliary files -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") - # Warn if the user invokes CMake directly if (NOT SKBUILD) message(WARNING "\ @@ -19,6 +14,12 @@ if (NOT SKBUILD) ") endif() +# Auxiliary files +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") + +# Define the project with name and version from pyproject.toml +project(${SKBUILD_PROJECT_NAME} LANGUAGES C VERSION ${SKBUILD_PROJECT_VERSION}) + # We are building libraries that will eventually be linked into shared # modules. All code should be built with PIC. set(CMAKE_POSITION_INDEPENDENT_CODE ON) diff --git a/pyproject.toml b/pyproject.toml index 2736fe7..97c007a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,20 +1,25 @@ [build-system] -requires = ["scikit-build-core", "cython", "cython-cmake", "numpy"] +requires = [ + "scikit-build-core", + "cython", + "cython-cmake", + "numpy", + "setuptools>=80", + "setuptools-scm>=8", +] build-backend = "scikit_build_core.build" [project] name = "flacarray" -version = "0.3.4" +dynamic = ["version"] description = "FLAC Compression of Arrays" readme = "README.md" maintainers = [ { name = "Theodore Kisner", email = "tskisner.public@gmail.com" }, ] requires-python = ">=3.10" -dependencies = [ - "numpy", -] +dependencies = ["numpy"] classifiers = [ "Development Status :: 4 - Beta", "License :: OSI Approved :: BSD License", @@ -37,6 +42,14 @@ flacarray_benchmark = "flacarray.scripts:bench_cli" "Source" = "https://github.com/hpc4cmb/flacarray/" "Issue Tracker" = "https://github.com/hpc4cmb/flacarray/issues" +# # We use setuptools_scm as the "source of truth" for the version. +# # This value is then available from cmake, and will be propagated +# # to both the python and C code. +# [tool.setuptools_scm] +# version_file = "pkg/_version.py" + +[tool.setuptools_scm] + [tool.scikit-build] # A list of args to pass to CMake when configuring the project. cmake.args = [] @@ -87,7 +100,7 @@ build.tool-args = [] # The build targets to use when building the project. build.targets = [] # Verbose printout when building. -build.verbose = false +build.verbose = true # Additional ``build-system.requires``. build.requires = [] # The components to install. @@ -96,13 +109,13 @@ install.components = [] install.strip = true # Add the python build environment site_packages folder to the CMake prefix paths. search.site-packages = true -# List dynamic metadata fields and hook locations in this table. -metadata = {} # Strictly check all config options. strict-config = true # Enable early previews of features not finalized yet. -experimental = false +experimental = true # If set, this will provide a method for backward compatibility. -minimum-version = "0.11" # current version +minimum-version = "0.11" # current version # The CMake build directory. Defaults to a unique temporary directory. build-dir = "build" +# Get version from setuptools_scm +metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" diff --git a/src/flacarray/CMakeLists.txt b/src/flacarray/CMakeLists.txt index a7e1d02..025ffb3 100644 --- a/src/flacarray/CMakeLists.txt +++ b/src/flacarray/CMakeLists.txt @@ -1,10 +1,22 @@ +# Generate the version files, if needed + +set(c_version ${CMAKE_CURRENT_SOURCE_DIR}/libflacarray/version.c) +set(py_version ${CMAKE_CURRENT_SOURCE_DIR}/_version.py) + +add_custom_command(OUTPUT ${c_version} ${py_version} + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/generate_version.sh ${SKBUILD_PROJECT_VERSION} + COMMENT "Updating version files if needed ..." + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" +) + # Create an object library of the compiled sources. set(libflcarr_SOURCES libflacarray/compress.c libflacarray/decompress.c libflacarray/utils.c + "${c_version}" ) set(libflcarr_HEADERS @@ -32,9 +44,17 @@ set_target_properties(flacarray PROPERTIES PUBLIC_HEADER ${libflcarr_HEADERS}) install(TARGETS flacarray LIBRARY DESTINATION "${SKBUILD_PROJECT_NAME}/lib" - PUBLIC_HEADER DESTINATION "${SKBUILD_PROJECT_NAME}/include" + PUBLIC_HEADER DESTINATION "${SKBUILD_HEADERS_DIR}" ) +# Create the compiled test executable + +add_executable(flacarray_c_test libflacarray/test.c) +target_link_libraries(flacarray_c_test PRIVATE flcarr) +set_target_properties(flacarray_c_test PROPERTIES PRIVATE_HEADER ${libflcarr_HEADERS}) + +install(TARGETS flacarray_c_test RUNTIME DESTINATION ${SKBUILD_SCRIPTS_DIR}) + # Create a compiled python extension cython_transpile(_libflacarray.pyx LANGUAGE C OUTPUT_VARIABLE libflacarray_c) @@ -51,6 +71,7 @@ install(TARGETS _libflacarray LIBRARY DESTINATION ${SKBUILD_PROJECT_NAME}) set(PYTHON_SOURCES __init__.py + "${py_version}" array.py compress.py decompress.py diff --git a/src/flacarray/__init__.py b/src/flacarray/__init__.py index 9e5125e..9b96887 100644 --- a/src/flacarray/__init__.py +++ b/src/flacarray/__init__.py @@ -7,7 +7,6 @@ import os - -__version__ = "0.3.4" +from ._version import __version__ from .array import FlacArray diff --git a/src/flacarray/generate_version.sh b/src/flacarray/generate_version.sh new file mode 100755 index 0000000..b70e39d --- /dev/null +++ b/src/flacarray/generate_version.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +cmake_version=$1 + +c_file="libflacarray/version.c" +c_version="" +if [ -f "${c_file}" ]; then + # Existing file, read in version + c_version=`cat "${c_file}" | sed -e 's/.*= "\(.*\)".*/\1/'` +fi + +if [ "x${c_version}" != "x${cmake_version}" ]; then + # The version has changed, update the file + echo "const char* FLACARRAY_VERSION = \"${cmake_version}\";" > "${c_file}" +fi + +py_file="_version.py" +py_version="" +if [ -f "${py_file}" ]; then + # Existing file, read in version + py_version=`cat "${py_file}" | sed -e 's/.*= "\(.*\)".*/\1/'` +fi + +if [ "x${py_version}" != "x${cmake_version}" ]; then + # The version has changed, update the file + echo "__version__ = \"${cmake_version}\"" > "${py_file}" +fi diff --git a/src/flacarray/libflacarray/README.md b/src/flacarray/libflacarray/README.md index dd1ae5b..bfe2beb 100644 --- a/src/flacarray/libflacarray/README.md +++ b/src/flacarray/libflacarray/README.md @@ -7,26 +7,4 @@ Cython in the parent directory. You should not have to modify any files in this directory unless you are working with the code that directly calls libFLAC. -## Developer Notes - -If you are hacking on the C source files, it can be convenient to compile and -test those manually independent of the overall package build system. There is a -small stand-alone test script in this directory which you can use. Edit the -`low_level.mk` make file and change any compiler options (for example, disable -optimization and enable debug symbols). Then build the test script: - - make -f low_level.mk - -and then run the `test_low_level` executable. Since this is using low-level C -memory management and pointer math, it is always good to run through valgrind -and verify correct memory handling. For example, if you are building in a conda -development environment, you may have to add the conda prefix to -`LD_LIBRARY_PATH` in order to load libFLAC at runtime: - - OMP_NUM_THREADS=4 \ - LD_LIBRARY_PATH=${CONDA_PREFIX}/lib \ - valgrind \ - --leak-check=full \ - --show-leak-kinds=all \ - --track-origins=yes \ - ./test_low_level 2>&1 | tee log +There is also a small, compiled test that is installed. diff --git a/src/flacarray/libflacarray/test.c b/src/flacarray/libflacarray/test.c index 8b6fe5a..093341e 100644 --- a/src/flacarray/libflacarray/test.c +++ b/src/flacarray/libflacarray/test.c @@ -8,6 +8,9 @@ #include "flacarray.h" +extern const char * FLACARRAY_VERSION; + + // Local function for debugging and testing int verify( @@ -66,12 +69,12 @@ int verify( // We already had a failure, skip over remaining loop iterations continue; } - fprintf(stderr, "Verifying stream %ld:\n", istream); - fprintf(stderr, " start byte = %ld\n", starts[istream]); - fprintf(stderr, " end byte = %ld\n", starts[istream] + nbytes[istream]); - fprintf( - stderr, " output start element = %ld\n", istream * n_decode * n_channels - ); + // fprintf(stderr, "Verifying stream %ld:\n", istream); + // fprintf(stderr, " start byte = %ld\n", starts[istream]); + // fprintf(stderr, " end byte = %ld\n", starts[istream] + nbytes[istream]); + // fprintf( + // stderr, " output start element = %ld\n", istream * n_decode * n_channels + // ); callback_data.cur_stream = istream; callback_data.stream_start = starts[istream]; callback_data.stream_end = starts[istream] + nbytes[istream]; @@ -100,27 +103,27 @@ int verify( } if (n_decode == stream_size) { // We are decoding all samples - fprintf( - stderr, - " decoding all samples (n_decode = %ld, stream_size = %ld)\n", - n_decode, - stream_size - ); + // fprintf( + // stderr, + // " decoding all samples (n_decode = %ld, stream_size = %ld)\n", + // n_decode, + // stream_size + // ); success = FLAC__stream_decoder_process_until_end_of_stream(decoder); - fprintf(stderr, " success = %d\n", (int)success); + // fprintf(stderr, " success = %d\n", (int)success); if (!success) { errors |= ERROR_DECODE_PROCESS; continue; } } else { // We are decoding a slice of samples. Seek to the start. - fprintf( - stderr, - " decoding slice of samples starting at %ld, seeking...\n", - first_decode - ); + // fprintf( + // stderr, + // " decoding slice of samples starting at %ld, seeking...\n", + // first_decode + // ); success = FLAC__stream_decoder_seek_absolute(decoder, first_decode); - fprintf(stderr, " success = %d\n", (int)success); + // fprintf(stderr, " success = %d\n", (int)success); if (!success) { errors |= ERROR_DECODE_PROCESS; continue; @@ -131,15 +134,15 @@ int verify( while ( callback_data.decomp_nelem < n_decode ) { - fprintf( - stderr, - " decoding frame %ld, decomp_nelem = %ld, n_decode = %ld\n", - curframe, - callback_data.decomp_nelem, - n_decode - ); + // fprintf( + // stderr, + // " decoding frame %ld, decomp_nelem = %ld, n_decode = %ld\n", + // curframe, + // callback_data.decomp_nelem, + // n_decode + // ); success = FLAC__stream_decoder_process_single(decoder); - fprintf(stderr, " success = %d\n", (int)success); + // fprintf(stderr, " success = %d\n", (int)success); if (!success) { errors |= ERROR_DECODE_PROCESS; continue; @@ -678,6 +681,8 @@ void test_64bit() { int main(int argc, char *argv[]) { + fprintf(stderr, "=========================================\n\n"); + fprintf(stderr, "Using libflacarray version %s\n\n", FLACARRAY_VERSION); test_32bit(); test_64bit(); return 0; From 92f2bd99e55e35b6fed8b798030191696a7320d8 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Mon, 9 Feb 2026 14:51:42 -0800 Subject: [PATCH 08/14] Fix sdist exclusion --- MANIFEST.in | 9 --------- pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) delete mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index c835985..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,9 +0,0 @@ -include MANIFEST.in -include pyproject.toml -include AUTHORS -include LICENSE -include *.md -recursive-include . meson.build -recursive-include flacarray -recursive-include docs -prune docs/site diff --git a/pyproject.toml b/pyproject.toml index 97c007a..d8134a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ logging.level = "WARNING" # Files to include in the SDist even if they are skipped by default. sdist.include = [] # Files to exclude from the SDist even if they are included by default. -sdist.exclude = [] +sdist.exclude = ["build", "wheelhouse", "docs/site"] # Try to build a reproducible distribution. sdist.reproducible = true # If set to True, CMake will be run before building the SDist. From 2b414ff180dc3ca9fa65ebe84e2175db1dce6212 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Mon, 9 Feb 2026 15:50:27 -0800 Subject: [PATCH 09/14] Add flacarray_config script --- pyproject.toml | 1 + src/flacarray/CMakeLists.txt | 2 +- src/flacarray/scripts/CMakeLists.txt | 1 + src/flacarray/scripts/__init__.py | 1 + src/flacarray/scripts/config.py | 94 ++++++++++++++++++++++++++++ 5 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 src/flacarray/scripts/config.py diff --git a/pyproject.toml b/pyproject.toml index d8134a4..50bbd3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ classifiers = [ [project.scripts] flacarray_benchmark = "flacarray.scripts:bench_cli" +flacarray_config = "flacarray.scripts:config_cli" [project.urls] "Documentation" = "https://hpc4cmb.github.io/flacarray/" diff --git a/src/flacarray/CMakeLists.txt b/src/flacarray/CMakeLists.txt index 025ffb3..ef5011c 100644 --- a/src/flacarray/CMakeLists.txt +++ b/src/flacarray/CMakeLists.txt @@ -44,7 +44,7 @@ set_target_properties(flacarray PROPERTIES PUBLIC_HEADER ${libflcarr_HEADERS}) install(TARGETS flacarray LIBRARY DESTINATION "${SKBUILD_PROJECT_NAME}/lib" - PUBLIC_HEADER DESTINATION "${SKBUILD_HEADERS_DIR}" + PUBLIC_HEADER DESTINATION "${SKBUILD_PROJECT_NAME}/include" ) # Create the compiled test executable diff --git a/src/flacarray/scripts/CMakeLists.txt b/src/flacarray/scripts/CMakeLists.txt index 9e7cfc5..ca47d7d 100644 --- a/src/flacarray/scripts/CMakeLists.txt +++ b/src/flacarray/scripts/CMakeLists.txt @@ -3,6 +3,7 @@ set(PYTHON_SOURCES __init__.py benchmark.py + config.py ) install(FILES ${PYTHON_SOURCES} DESTINATION "${SKBUILD_PROJECT_NAME}/scripts") diff --git a/src/flacarray/scripts/__init__.py b/src/flacarray/scripts/__init__.py index 558f0a9..ef7634b 100644 --- a/src/flacarray/scripts/__init__.py +++ b/src/flacarray/scripts/__init__.py @@ -3,3 +3,4 @@ # a BSD-style license that can be found in the LICENSE file. from .benchmark import cli as bench_cli +from .config import cli as config_cli diff --git a/src/flacarray/scripts/config.py b/src/flacarray/scripts/config.py new file mode 100644 index 0000000..711f124 --- /dev/null +++ b/src/flacarray/scripts/config.py @@ -0,0 +1,94 @@ +# Copyright (c) 2026-2026 by the parties listed in the AUTHORS file. +# All rights reserved. Use of this source code is governed by +# a BSD-style license that can be found in the LICENSE file. + +import argparse +import os +import sysconfig + +# We use absolute imports to find the location of the installed +# package. +import flacarray + + +def cli(): + parser = argparse.ArgumentParser(description="Print configuration of flacarray") + parser.add_argument( + "--version", + required=False, + default=False, + action="store_true", + help="Print the package version", + ) + parser.add_argument( + "--package", + required=False, + default=False, + action="store_true", + help="Print the package install location", + ) + parser.add_argument( + "--cflags", + required=False, + default=False, + action="store_true", + help="Print the include CFLAGS (-I...)", + ) + parser.add_argument( + "--include", + required=False, + default=False, + action="store_true", + help="Print the directory containing flacarray.h", + ) + parser.add_argument( + "--ldflags", + required=False, + default=False, + action="store_true", + help="Print the linking LDFLAGS (-L...)", + ) + parser.add_argument( + "--libs", + required=False, + default=False, + action="store_true", + help="Print the linking libraries (-lflacarray)", + ) + parser.add_argument( + "--lib", + required=False, + default=False, + action="store_true", + help="Print the path to libflacarray", + ) + + args = parser.parse_args() + + version = flacarray.__version__ + pkg_root = os.path.dirname(flacarray.__file__) + pkg_incdir = os.path.join(pkg_root, "include") + pkg_libdir = os.path.join(pkg_root, "lib") + + suffix = sysconfig.get_config_var("SHLIB_SUFFIX") + + if args.version: + print(version) + elif args.package: + print(pkg_root) + elif args.include: + print(pkg_incdir) + elif args.cflags: + print(f"-I{pkg_incdir}") + elif args.ldflags: + print(f"-L{pkg_libdir}") + elif args.libs: + print("-lflacarray") + elif args.lib: + print(os.path.join(pkg_libdir, f"libflacarray{suffix}")) + else: + parser.print_help() + + +if __name__ == "__main__": + cli() From db06d1c01e88faa9577d4558b95be10dab224df4 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Fri, 13 Feb 2026 23:00:49 -0800 Subject: [PATCH 10/14] Add external C link test. Update generation of the flacarray_config script --- .github/workflows/test.yml | 9 +++ cmake/Findflacarray.cmake | 80 +++++++++++++++++++ docs/docs/install.md | 48 +++++++++-- packaging/test_c_link/CMakeLists.txt | 28 +++++++ packaging/test_c_link/Findflacarray.cmake | 1 + packaging/test_c_link/test.c | 1 + src/flacarray/CMakeLists.txt | 14 +++- src/flacarray/libflacarray/test.c | 2 +- src/flacarray/scripts/CMakeLists.txt | 10 ++- .../scripts/{config.py => config.py.in} | 60 ++++++++++---- 10 files changed, 226 insertions(+), 27 deletions(-) create mode 100644 cmake/Findflacarray.cmake create mode 100644 packaging/test_c_link/CMakeLists.txt create mode 120000 packaging/test_c_link/Findflacarray.cmake create mode 120000 packaging/test_c_link/test.c rename src/flacarray/scripts/{config.py => config.py.in} (55%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fbbe01f..b98a43f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -123,3 +123,12 @@ jobs: && pushd test >/dev/null 2>&1 \ && mpirun -np 2 python -m mpi4py -c 'import flacarray.tests; flacarray.tests.run()' \ && popd >/dev/null 2>&1 + + - name: External C Link Tests + run: | + source ~/conda/etc/profile.d/conda.sh \ + && conda activate test \ + && export OMP_NUM_THREADS=2 \ + && cmake -S packaging/test_c_link -B build_c_link \ + && cmake --build build_c_link \ + && ctest --test-dir build_c_link diff --git a/cmake/Findflacarray.cmake b/cmake/Findflacarray.cmake new file mode 100644 index 0000000..046f43a --- /dev/null +++ b/cmake/Findflacarray.cmake @@ -0,0 +1,80 @@ +# - Find the flacarray C library +# +# First version: +# Copyright (c) 2026, Ted Kisner +# +# Usage: +# find_package(flacarray [REQUIRED]) +# +# It sets the following variables: +# flacarray_FOUND ... true if flacarray is found on the system +# flacarray_LIBRARIES ... full path to flacarray libraries +# flacarray_INCLUDE_DIRS ... flacarray include directory paths +# +# This file uses the flacarray_config script, which should be in the executable +# search path if flacarray has been installed properly in the current environment. +# +# The following variables will be checked by the function +# flacarray_ROOT ... ignore flacarray_config script and look here. +# flacarray_USE_STATIC_LIBS ... search for static versions of the libs +# + +# Check whether to search static or dynamic libs + +if(${flacarray_USE_STATIC_LIBS}) + set(FLACARRAY_LIB_NAME "flacarray_static") +else() + set(FLACARRAY_LIB_NAME "flacarray") +endif() + +if(flacarray_ROOT) + # Find libs + find_library( + FLACARRAY_LIB + NAMES ${FLACARRAY_LIB_NAME} + PATHS ${flacarray_ROOT} + PATH_SUFFIXES "lib" "lib64" + NO_DEFAULT_PATH + ) + # Find includes + find_path(flacarray_INCLUDE_DIRS + NAMES "flacarray.h" + PATHS ${flacarray_ROOT} + PATH_SUFFIXES "include" + NO_DEFAULT_PATH + ) +else() + # Use flacarray_config + find_program(FLACARRAY_CONFIG flacarray_config REQUIRED) + execute_process( + COMMAND "${FLACARRAY_CONFIG}" --package + OUTPUT_VARIABLE flacarray_ROOT + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + execute_process( + COMMAND "${FLACARRAY_CONFIG}" --include + OUTPUT_VARIABLE flacarray_INCLUDE_DIRS + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + execute_process( + COMMAND "${FLACARRAY_CONFIG}" --libraries + OUTPUT_VARIABLE FLACARRAY_LIB + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +endif() + +if(FLACARRAY_LIB) + set(flacarray_LIBRARIES ${FLACARRAY_LIB}) +endif() + +include(FindPackageHandleStandardArgs) + +find_package_handle_standard_args(flacarray + REQUIRED_VARS flacarray_INCLUDE_DIRS FLACARRAY_LIB + HANDLE_COMPONENTS +) + +mark_as_advanced( + flacarray_INCLUDE_DIRS + flacarray_LIBRARIES +) diff --git a/docs/docs/install.md b/docs/docs/install.md index a6b5b4c..98cd1bd 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -24,10 +24,6 @@ If you are using a conda environment you can install the conda package for conda install -c conda-forge flacarray -!!! note "To-Do" - - `flacarray` is not yet on conda-forge - ## Building From Source In order to build from source, you will need a C compiler and the FLAC @@ -38,10 +34,11 @@ development libraries installed. If you have conda available, you can create an environment will all the dependencies you need to build flacarray from source. For this example, we create an environment called "flacarray". First create the env with all -dependencies and activate it (FIXME: add a requirements file for dev): +dependencies and activate it. There is a list of conda requirements +provided in the source: conda create -n flacarray \ - c-compiler numpy libflac cython meson-python pkgconfig + --file packaging/conda_build_requirements.txt conda activate flacarray @@ -58,6 +55,41 @@ To also work on docs, install additional packages: ### Other Ways of Building -!!! note "To-Do" +If you have a relatively recent system python3 and libFLAC provided by your +operating system, you can build flacarray using only OS tools. Flacarray +**requires libFLAC >= 1.4.0**. For example, on Debian-based systems, install +these packages: + + sudo apt update + sudo apt install build-essential libflac-dev python3-dev python3-venv + +Now create a virtualenv and activate it: + + python3 -m venv /path/to/env + source /path/to/env/bin/activate + +Next, go into the flacarray git checkout and install to the virtualenv: + + cd flacarray + pip install . + +### Running Tests + +When building from source, you should definitely run the unit test suite after +installation. The tests are bundled in the package: + + python -c 'import flacarray.tests; flacarray.tests.run()' + +## Using From Compiled Software + +Flacarray ships with a low-level C library which can be linked against from compiled software. First, install flacarray as described above. Note that depending on how you build your compiled software, you may want to choose carefully how flacarray is installed. For example, if you are compiling your software in a conda environment using the conda compiler toolchain, it is easiest if you install flacarray through the conda package. If you are installing your software with the OS provided compiler, you may want to build flacarray from source with the same compiler. + +### Linking to Flacarray from CMake + +If you are using CMake to build your software, you can use the included FindFlacarray.cmake file in the top of the source tree. This will set several environment variables. + + + +### Other Build Systems - Discuss OS packages, document apt, rpm, homebrew options. +You can use the included flacarray_config script to print out the CFLAGS / LDFLAGS / LIBS needed to link to libflacarray and find the flacarray.h header. diff --git a/packaging/test_c_link/CMakeLists.txt b/packaging/test_c_link/CMakeLists.txt new file mode 100644 index 0000000..e205f1f --- /dev/null +++ b/packaging/test_c_link/CMakeLists.txt @@ -0,0 +1,28 @@ +cmake_minimum_required(VERSION 3.21) + +# Auxiliary files, including Findflacarray.cmake +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") + +# Add CTest +enable_testing() + +# Define the project with name and version from pyproject.toml +project(c_test LANGUAGES C VERSION 1.0) + +# Use modern C +set(CMAKE_C_STANDARD 11) + +# Find libflacarray +find_package(flacarray REQUIRED) + +# Create the compiled test executable + +add_executable(test_c_link test.c) +target_link_libraries(test_c_link PRIVATE ${flacarray_LIBRARIES}) +target_include_directories(test_c_link BEFORE PUBLIC ${flacarray_INCLUDE_DIRS}) + +# Add a test that runs this executable +add_test( + NAME test_link + COMMAND test_c_link +) diff --git a/packaging/test_c_link/Findflacarray.cmake b/packaging/test_c_link/Findflacarray.cmake new file mode 120000 index 0000000..f9197ea --- /dev/null +++ b/packaging/test_c_link/Findflacarray.cmake @@ -0,0 +1 @@ +../../cmake/Findflacarray.cmake \ No newline at end of file diff --git a/packaging/test_c_link/test.c b/packaging/test_c_link/test.c new file mode 120000 index 0000000..5060ec3 --- /dev/null +++ b/packaging/test_c_link/test.c @@ -0,0 +1 @@ +../../src/flacarray/libflacarray/test.c \ No newline at end of file diff --git a/src/flacarray/CMakeLists.txt b/src/flacarray/CMakeLists.txt index ef5011c..f1b5069 100644 --- a/src/flacarray/CMakeLists.txt +++ b/src/flacarray/CMakeLists.txt @@ -36,7 +36,7 @@ target_link_libraries(flcarr PUBLIC "${FLAC_LIBRARIES}") target_link_libraries(flcarr PUBLIC "${OpenMP_C_LIBRARIES}") -# Create an actual installed library, usable by external compiled software +# Create an shared library, usable by external compiled software add_library(flacarray SHARED) target_link_libraries(flacarray PRIVATE flcarr) @@ -47,11 +47,21 @@ install(TARGETS flacarray PUBLIC_HEADER DESTINATION "${SKBUILD_PROJECT_NAME}/include" ) +# Create a static library, usable by external compiled software + +add_library(flacarray_static STATIC) +target_link_libraries(flacarray_static PRIVATE flcarr) +set_target_properties(flacarray_static PROPERTIES PUBLIC_HEADER ${libflcarr_HEADERS}) + +install(TARGETS flacarray_static + LIBRARY DESTINATION "${SKBUILD_PROJECT_NAME}/lib" +) + # Create the compiled test executable add_executable(flacarray_c_test libflacarray/test.c) target_link_libraries(flacarray_c_test PRIVATE flcarr) -set_target_properties(flacarray_c_test PROPERTIES PRIVATE_HEADER ${libflcarr_HEADERS}) +# set_target_properties(flacarray_c_test PROPERTIES PRIVATE_HEADER ${libflcarr_HEADERS}) install(TARGETS flacarray_c_test RUNTIME DESTINATION ${SKBUILD_SCRIPTS_DIR}) diff --git a/src/flacarray/libflacarray/test.c b/src/flacarray/libflacarray/test.c index 093341e..c411051 100644 --- a/src/flacarray/libflacarray/test.c +++ b/src/flacarray/libflacarray/test.c @@ -5,7 +5,7 @@ #include #include -#include "flacarray.h" +#include extern const char * FLACARRAY_VERSION; diff --git a/src/flacarray/scripts/CMakeLists.txt b/src/flacarray/scripts/CMakeLists.txt index ca47d7d..e2f99a3 100644 --- a/src/flacarray/scripts/CMakeLists.txt +++ b/src/flacarray/scripts/CMakeLists.txt @@ -1,9 +1,17 @@ # Install python files +set(CONF_EXTRA_LINK "${FLAC_LIBRARIES};${OpenMP_C_LIBRARIES}") + +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/config.py.in" # Input file path + "${CMAKE_CURRENT_BINARY_DIR}/config.py" # Output file path + @ONLY # Only @VAR@ form only +) + set(PYTHON_SOURCES __init__.py benchmark.py - config.py + "${CMAKE_CURRENT_BINARY_DIR}/config.py" ) install(FILES ${PYTHON_SOURCES} DESTINATION "${SKBUILD_PROJECT_NAME}/scripts") diff --git a/src/flacarray/scripts/config.py b/src/flacarray/scripts/config.py.in similarity index 55% rename from src/flacarray/scripts/config.py rename to src/flacarray/scripts/config.py.in index 711f124..452d329 100644 --- a/src/flacarray/scripts/config.py +++ b/src/flacarray/scripts/config.py.in @@ -4,7 +4,6 @@ import argparse import os -import sysconfig # We use absolute imports to find the location of the installed # package. @@ -27,6 +26,13 @@ def cli(): action="store_true", help="Print the package install location", ) + parser.add_argument( + "--static", + required=False, + default=False, + action="store_true", + help="Print the values for static linking instead of dynamic", + ) parser.add_argument( "--cflags", required=False, @@ -41,6 +47,13 @@ def cli(): action="store_true", help="Print the directory containing flacarray.h", ) + parser.add_argument( + "--libdir", + required=False, + default=False, + action="store_true", + help="Print the directory containing libflacarray", + ) parser.add_argument( "--ldflags", required=False, @@ -56,38 +69,55 @@ def cli(): help="Print the linking libraries (-lflacarray)", ) parser.add_argument( - "--lib", + "--libraries", required=False, default=False, action="store_true", - help="Print the path to libflacarray", + help="Print the full library paths", ) args = parser.parse_args() version = flacarray.__version__ pkg_root = os.path.dirname(flacarray.__file__) - pkg_incdir = os.path.join(pkg_root, "include") - pkg_libdir = os.path.join(pkg_root, "lib") + pkg_incdir = f"{pkg_root}/include" + pkg_libdir = f"{pkg_root}/lib" - suffix = sysconfig.get_config_var("SHLIB_SUFFIX") + shared_suffix = "@CMAKE_SHARED_LIBRARY_SUFFIX@" + static_suffix = "@CMAKE_STATIC_LIBRARY_SUFFIX@" + # First process options that should be used in isolation. if args.version: print(version) elif args.package: print(pkg_root) elif args.include: print(pkg_incdir) - elif args.cflags: - print(f"-I{pkg_incdir}") - elif args.ldflags: - print(f"-L{pkg_libdir}") - elif args.libs: - print("-lflacarray") - elif args.lib: - print(os.path.join(pkg_libdir, f"libflacarray{suffix}")) + elif args.libdir: + print(pkg_libdir) + elif args.libraries: + # Print full library paths + if args.static: + libpath = os.path.join(pkg_libdir, f"libflacarray{static_suffix}") + else: + libpath = os.path.join(pkg_libdir, f"libflacarray{shared_suffix}") + print(f"{libpath};@CONF_EXTRA_LINK@") else: - parser.print_help() + # These options can be combined + output = "" + if args.cflags: + output = f"{output} -I{pkg_incdir}" + if args.ldflags: + output = f"{output} -L{pkg_libdir}" + if args.libs: + if args.static: + output = f"{output} -lflacarray_static" + else: + output = f"{output} -lflacarray" + if output != "": + print(output) + else: + parser.print_help() if __name__ == "__main__": From 9e0ef4e336c89f793b7b291934190d8db0363701 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Sun, 15 Feb 2026 10:35:04 -0800 Subject: [PATCH 11/14] Update docs. Fix flacarray_config and cmake tests. --- .github/workflows/docs.yml | 13 +- .github/workflows/test.yml | 11 +- cmake/Findflacarray.cmake | 27 +- docs/docs/install.md | 4 +- docs/docs/reference.md | 548 ++++++++++++++++++++---- docs/install_requirements.sh | 11 + packaging/test_c_link/CMakeLists.txt | 2 +- src/flacarray/CMakeLists.txt | 3 +- src/flacarray/libflacarray/decompress.c | 2 +- src/flacarray/libflacarray/flacarray.h | 278 +++++++++++- src/flacarray/scripts/config.py.in | 2 +- 11 files changed, 769 insertions(+), 132 deletions(-) create mode 100755 docs/install_requirements.sh diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 15b351f..42259aa 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -55,21 +55,10 @@ jobs: && popd >/dev/null 2>&1 - name: Install Docs Dependencies - # AttributeError: 'MathBlockParser' object has no attribute 'parse_axt_heading' - # https://github.com/jupyter/nbconvert/issues/2198 - # mistune = "<3.1" run: | source ~/conda/etc/profile.d/conda.sh \ && conda activate docs \ - && conda install --yes \ - mkdocs \ - mkdocstrings \ - mkdocstrings-python \ - mkdocs-material \ - mkdocs-material-extensions \ - mkdocs-jupyter \ - "mistune<3.1" \ - && pip install mkdocs-print-site-plugin \ + && ./docs/install_requirements.sh \ && git config user.name 'github-actions[bot]' \ && git config user.email 'github-actions[bot]@users.noreply.github.com' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b98a43f..cb85818 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -124,7 +124,7 @@ jobs: && mpirun -np 2 python -m mpi4py -c 'import flacarray.tests; flacarray.tests.run()' \ && popd >/dev/null 2>&1 - - name: External C Link Tests + - name: External C Link Tests (Dynamic) run: | source ~/conda/etc/profile.d/conda.sh \ && conda activate test \ @@ -132,3 +132,12 @@ jobs: && cmake -S packaging/test_c_link -B build_c_link \ && cmake --build build_c_link \ && ctest --test-dir build_c_link + + - name: External C Link Tests (Static) + run: | + source ~/conda/etc/profile.d/conda.sh \ + && conda activate test \ + && export OMP_NUM_THREADS=2 \ + && cmake -S packaging/test_c_link -B build_c_link_static \ + && cmake --build build_c_link_static \ + && ctest --test-dir build_c_link_static diff --git a/cmake/Findflacarray.cmake b/cmake/Findflacarray.cmake index 046f43a..af709dc 100644 --- a/cmake/Findflacarray.cmake +++ b/cmake/Findflacarray.cmake @@ -21,7 +21,9 @@ # Check whether to search static or dynamic libs -if(${flacarray_USE_STATIC_LIBS}) +option(flacarray_USE_STATIC_LIBS "Link to libflacarray_static.a" OFF) + +if(flacarray_USE_STATIC_LIBS) set(FLACARRAY_LIB_NAME "flacarray_static") else() set(FLACARRAY_LIB_NAME "flacarray") @@ -47,24 +49,33 @@ else() # Use flacarray_config find_program(FLACARRAY_CONFIG flacarray_config REQUIRED) execute_process( - COMMAND "${FLACARRAY_CONFIG}" --package + COMMAND ${FLACARRAY_CONFIG} --package OUTPUT_VARIABLE flacarray_ROOT OUTPUT_STRIP_TRAILING_WHITESPACE ) execute_process( - COMMAND "${FLACARRAY_CONFIG}" --include + COMMAND ${FLACARRAY_CONFIG} --include OUTPUT_VARIABLE flacarray_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE ) - execute_process( - COMMAND "${FLACARRAY_CONFIG}" --libraries - OUTPUT_VARIABLE FLACARRAY_LIB - OUTPUT_STRIP_TRAILING_WHITESPACE - ) + if(flacarray_USE_STATIC_LIBS) + execute_process( + COMMAND ${FLACARRAY_CONFIG} ${FLACARRARY_CONFIG_STATIC} --static --libraries + OUTPUT_VARIABLE FLACARRAY_LIB + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + else() + execute_process( + COMMAND ${FLACARRAY_CONFIG} ${FLACARRARY_CONFIG_STATIC} --libraries + OUTPUT_VARIABLE FLACARRAY_LIB + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + endif() endif() if(FLACARRAY_LIB) set(flacarray_LIBRARIES ${FLACARRAY_LIB}) + message(STATUS "Using flacarray libraries: ${flacarray_LIBRARIES}") endif() include(FindPackageHandleStandardArgs) diff --git a/docs/docs/install.md b/docs/docs/install.md index 98cd1bd..612664b 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -86,9 +86,7 @@ Flacarray ships with a low-level C library which can be linked against from comp ### Linking to Flacarray from CMake -If you are using CMake to build your software, you can use the included FindFlacarray.cmake file in the top of the source tree. This will set several environment variables. - - +If you are using CMake to build your software, you can copy the included `cmake/Findflacarray.cmake` file into your source tree. This will set several environment variables that you can use for compiling and linking against flacarray. See the example in `packaging/test_c_link` for a small package that uses this cmake macro. ### Other Build Systems diff --git a/docs/docs/reference.md b/docs/docs/reference.md index 60d7fd4..3c36144 100644 --- a/docs/docs/reference.md +++ b/docs/docs/reference.md @@ -63,99 +63,463 @@ bytestream and auxiliary arrays and convert to / from numpy arrays. ::: flacarray.decompress.array_decompress_slice +## Notes About Threading + +The optional threading in flacarray uses OpenMP to distribute *entire streams* +among threads. OpenMP was chosen since some downstream codes using `flacarray` +are hybrid MPI / OpenMP codebases and it is convenient to set the threading +concurrency for all software in a running job. However, the encode / decode +operations are very fast, and for small arrays the threading overhead can +actually slow things down. For large arrays (hundreds of streams, each with +millions of samples), using threads does provide speed ups (especially for +encode operations). For these reasons, the threading options for all functions +in the API are disabled by default. + +Newer versions of libFLAC (>=1.5.0) include support for threaded encodes. This +threading is done over frames within a single stream. Currently flacarray does +not use this, since it might introduce oversubscription of cores if both +threading models were enabled. Future flacarray developments might make use of +libFLAC threads if they show real-world performance improvements. + ## Compiled Library Interfaces -The lowest-level functions in the compiled C code are exposed in the `flacarray.h` header file and the `libflacarray` library. These are installed along with the python package, and can be linked into compiled software products. +The lowest-level functions in the compiled C code are exposed in the +`flacarray.h` header file and the `libflacarray` library. These are installed +along with the python package, and can be linked into compiled software +products. For an example of using these functions, see the compiled test case in +`src/flacarray/libflacarray/test.c` +### Encoding + +The encoding functions have very different internals for the unthreaded and +threaded cases. For this reason, they are split into different functions. The +32bit functions encode data to a single FLAC channel, while the 64bit functions +use two interleaved FLAC channels. + +``` c +/* + * Encode multiple 32bit integer streams. + * + * The input streams are passed as a single (concatenated), flat-packed buffer. + * This input `data` should have length `n_stream` X `stream_size`. + * + * The `n_bytes` and `starts` should be pre-allocated to hold one value per + * stream. These will be populated by the function. The `bytes` pointer + * is the address of a pointer that will be malloc'd and populated with + * the compressed bytes. The `starts` values are the starting byte offset + * of each stream in this output buffer. The `n_bytes` are the number + * of bytes following the starting point for each stream. + * + * @param[in] data The input integer data. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param level The FLAC compression level. + * @param[in] n_bytes The number of bytes per stream (filled by this function). + * @param[in] starts The starting byte per stream (filled by this function). + * @param[out] bytes The array of pointers to the bytes for each stream. + */ int encode_i32( - int32_t * data, - int64_t n_stream, - int64_t stream_size, - uint32_t level, - int64_t * n_bytes, - int64_t * starts, - unsigned char ** rawbytes - ) - int encode_i32_threaded( - int32_t * data, - int64_t n_stream, - int64_t stream_size, - uint32_t level, - int64_t * n_bytes, - int64_t * starts, - unsigned char ** rawbytes - ) - int encode_i64( - int64_t * data, - int64_t n_stream, - int64_t stream_size, - uint32_t level, - int64_t * n_bytes, - int64_t * starts, - unsigned char ** rawbytes - ) - int encode_i64_threaded( - int64_t * data, - int64_t n_stream, - int64_t stream_size, - uint32_t level, - int64_t * n_bytes, - int64_t * starts, - unsigned char ** rawbytes - ) - int decode_i32( - unsigned char * rawbytes, - int64_t * starts, - int64_t * nbytes, - int64_t n_stream, - int64_t stream_size, - int64_t first_sample, - int64_t last_sample, - int32_t * data, - bool use_threads - ) - int decode_i64( - unsigned char * rawbytes, - int64_t * starts, - int64_t * nbytes, - int64_t n_stream, - int64_t stream_size, - int64_t first_sample, - int64_t last_sample, - int64_t * data, - bool use_threads - ) - int float32_to_int32( - float * input, - int64_t n_stream, - int64_t stream_size, - float * quanta, - int32_t * output, - float * offsets, - float * gains - ) - int float64_to_int64( - double * input, - int64_t n_stream, - int64_t stream_size, - double * quanta, - int64_t * output, - double * offsets, - double * gains - ) - void int64_to_float64( - int64_t * input, - int64_t n_stream, - int64_t stream_size, - double * offsets, - double * gains, - double * output - ) - void int32_to_float32( - int32_t * input, - int64_t n_stream, - int64_t stream_size, - float * offsets, - float * gains, - float * output - ) + int32_t * const data, + int64_t n_stream, + int64_t stream_size, + uint32_t level, + int64_t * n_bytes, + int64_t * starts, + unsigned char ** bytes +); +``` + +Threaded version: + +``` c +/* + * Encode multiple 32bit integer streams with threads. + * + * The streams are distributed to the available threads, which independently + * accumulate the compressed bytes for their assigned streams. These + * thread-local compressed streams are then copied into the final output. + * + * The input streams are passed as a single (concatenated), flat-packed buffer. + * This input `data` should have length `n_stream` X `stream_size`. + * + * The `n_bytes` and `starts` should be pre-allocated to hold one value per + * stream. These will be populated by the function. The `bytes` pointer + * is the address of a pointer that will be malloc'd and populated with + * the compressed bytes. The `starts` values are the starting byte offset + * of each stream in this output buffer. The `n_bytes` are the number + * of bytes following the starting point for each stream. + * + * @param[in] data The input integer data. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param level The FLAC compression level. + * @param[in] n_bytes The number of bytes per stream (filled by this function). + * @param[in] starts The starting byte per stream (filled by this function). + * @param[out] bytes The array of pointers to the bytes for each stream. + */ +int encode_i32_threaded( + int32_t * const data, + int64_t n_stream, + int64_t stream_size, + uint32_t level, + int64_t * n_bytes, + int64_t * starts, + unsigned char ** bytes +); +``` + +The 64bit encode functions are the same except for the input data type: + +``` c +/* + * Encode multiple 64bit integer streams. + * + * The input streams are passed as a single (concatenated), flat-packed buffer. + * This input `data` should have length `n_stream` X `stream_size`. + * + * The `n_bytes` and `starts` should be pre-allocated to hold one value per + * stream. These will be populated by the function. The `bytes` pointer + * is the address of a pointer that will be malloc'd and populated with + * the compressed bytes. The `starts` values are the starting byte offset + * of each stream in this output buffer. The `n_bytes` are the number + * of bytes following the starting point for each stream. + * + * @param[in] data The input integer data. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param level The FLAC compression level. + * @param[in] n_bytes The number of bytes per stream (filled by this function). + * @param[in] starts The starting byte per stream (filled by this function). + * @param[out] bytes The array of pointers to the bytes for each stream. + */ +int encode_i64( + int64_t * const data, + int64_t n_stream, + int64_t stream_size, + uint32_t level, + int64_t * n_bytes, + int64_t * starts, + unsigned char ** bytes +); +``` + +``` c +/* + * Encode multiple 64bit integer streams with threads. + * + * The streams are distributed to the available threads, which independently + * accumulate the compressed bytes for their assigned streams. These + * thread-local compressed streams are then copied into the final output. + * + * The input streams are passed as a single (concatenated), flat-packed buffer. + * This input `data` should have length `n_stream` X `stream_size`. + * + * The `n_bytes` and `starts` should be pre-allocated to hold one value per + * stream. These will be populated by the function. The `bytes` pointer + * is the address of a pointer that will be malloc'd and populated with + * the compressed bytes. The `starts` values are the starting byte offset + * of each stream in this output buffer. The `n_bytes` are the number + * of bytes following the starting point for each stream. + * + * @param[in] data The input integer data. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param level The FLAC compression level. + * @param[in] n_bytes The number of bytes per stream (filled by this function). + * @param[in] starts The starting byte per stream (filled by this function). + * @param[out] bytes The array of pointers to the bytes for each stream. + */ +int encode_i64_threaded( + int64_t * const data, + int64_t n_stream, + int64_t stream_size, + uint32_t level, + int64_t * n_bytes, + int64_t * starts, + unsigned char ** bytes +); +``` + +### Decoding + +The decoding functions use a boolean switch to select whether to use OpenMP +threads. This is possible since the internal code is nearly the same in both the +threaded and non-threaded cases. The output buffer size is known and each thread +can independently work on decoding streams and populating slices of the output. + +``` c +/* + * Decode multiple 32bit integer streams. + * + * The input `bytes` are the concatenated buffers for all streams. + * + * The `starts` values are the starting byte offset of each stream in `bytes`. + * The `n_bytes` are the number of bytes following the starting point for each + * stream. + * + * The `first_sample` and `last_sample` arguments control the subset of values + * that are extracted from each stream. To extract all values, set `first_sample` + * to zero and `last_sample` to `stream_size`. + * + * The output data buffer should be pre-allocated to hold + * `n_stream` X (`last_sample` - `first_sample`) elements. + * + * @param[in] bytes The concatenated compressed bytes for all streams. + * @param[in] n_bytes The number of bytes per stream. + * @param[in] starts The starting byte per stream. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param first_sample The first output sample to decode for all streams. + * @param last_sample The last output sample (exclusive) to decode for all streams. + * @param[out] data The output integer data. + * @param[in] use_threads If true, used threads for decoding. + */ +int decode_i32( + unsigned char * const bytes, + int64_t * const starts, + int64_t * const nbytes, + int64_t n_stream, + int64_t stream_size, + int64_t first_sample, + int64_t last_sample, + int32_t * data, + bool use_threads +); +``` + +``` c +/* + * Decode multiple 64bit integer streams. + * + * The input `bytes` are the concatenated buffers for all streams. + * + * The `starts` values are the starting byte offset of each stream in `bytes`. + * The `n_bytes` are the number of bytes following the starting point for each + * stream. + * + * The `first_sample` and `last_sample` arguments control the subset of values + * that are extracted from each stream. To extract all values, set `first_sample` + * to zero and `last_sample` to `stream_size`. + * + * The output data buffer should be pre-allocated to hold + * `n_stream` X (`last_sample` - `first_sample`) elements. + * + * @param[in] bytes The concatenated compressed bytes for all streams. + * @param[in] n_bytes The number of bytes per stream. + * @param[in] starts The starting byte per stream. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param first_sample The first output sample to decode for all streams. + * @param last_sample The last output sample (exclusive) to decode for all streams. + * @param[out] data The output integer data. + * @param[in] use_threads If true, used threads for decoding. + */ +int decode_i64( + unsigned char * const bytes, + int64_t * const starts, + int64_t * const nbytes, + int64_t n_stream, + int64_t stream_size, + int64_t first_sample, + int64_t last_sample, + int64_t * data, + bool use_threads +); +``` + +### Type Conversions + +Flacarray supports encoding floating point data as compressed integers. This is +done by setting the `quanta` for all streams. This number represents the +smallest floating point value that can be preserved in a lossless way by +encoding to integers and back. The compiled library has helper functions that +can be used to convert back and forth. 32bit floating point values are converted +to 32bit integers and 64bit floating point values are converted to 64bit +integers. + +``` c +/* + * Convert 32bit floating point data to integers. + * + * The `input` data is the concatenated buffers for all streams. The `quanta` + * array is the smallest floating point value for each stream that will be + * preserved in a lossless way going from float to int and back. + * + * The `output` data buffer should be pre-allocated to hold + * `n_stream` X `stream_size` elements. The `offsets` and `gains` should be + * pre-allocated to hold one value per stream. + * + * @param[in] input The concatenated input data from multiple streams. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param[in] quanta The quantization value for each stream. + * @param[out] output The output integer data. + * @param[out] offsets The floating point offset subtracted from each input stream. + * @param[out] gains The gain applied to each input stream before truncation. + */ +int float32_to_int32( + float const * input, + int64_t n_stream, + int64_t stream_size, + float const * quanta, + int32_t * output, + float * offsets, + float * gains +); +``` + +``` c +/* + * Convert 64bit floating point data to integers. + * + * The `input` data is the concatenated buffers for all streams. The `quanta` + * array is the smallest floating point value for each stream that will be + * preserved in a lossless way going from float to int and back. + * + * The `output` data buffer should be pre-allocated to hold + * `n_stream` X `stream_size` elements. The `offsets` and `gains` should be + * pre-allocated to hold one value per stream. + * + * @param[in] input The concatenated input data from multiple streams. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param[in] quanta The quantization value for each stream. + * @param[out] output The output integer data. + * @param[out] offsets The floating point offset subtracted from each input stream. + * @param[out] gains The gain applied to each input stream before truncation. + */ +int float64_to_int64( + double const * input, + int64_t n_stream, + int64_t stream_size, + double const * quanta, + int64_t * output, + double * offsets, + double * gains +); +``` + +``` c +/* + * Convert 32bit integers to floating point data. + * + * The `input` data is the concatenated buffers for all streams. To restore + * the original floating point data, the integer values are scaled by 1/gain + * and then the offset is added. + * + * The `output` data buffer should be pre-allocated to hold + * `n_stream` X `stream_size` elements. + * + * @param[in] input The concatenated input data from multiple streams. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param[in] offsets The floating point offset subtracted from each original stream. + * @param[in] gains The gain applied to each original stream before truncation. + * @param[out] output The restored floating point data. + */ +void int32_to_float32( + int32_t const * input, + int64_t n_stream, + int64_t stream_size, + float const * offsets, + float const * gains, + float * output +); +``` + +``` c +/* + * Convert 64bit integers to floating point data. + * + * The `input` data is the concatenated buffers for all streams. To restore + * the original floating point data, the integer values are scaled by 1/gain + * and then the offset is added. + * + * The `output` data buffer should be pre-allocated to hold + * `n_stream` X `stream_size` elements. + * + * @param[in] input The concatenated input data from multiple streams. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param[in] offsets The floating point offset subtracted from each original stream. + * @param[in] gains The gain applied to each original stream before truncation. + * @param[out] output The restored floating point data. + */ +void int64_to_float64( + int64_t const * input, + int64_t n_stream, + int64_t stream_size, + double const * offsets, + double const * gains, + double * output +); +``` + +### Error Codes + +The error codes returned by the compiled functions are a bitwise OR of multiple possible error conditions. These are defined in terms of the bit location in the integer error code values: + +``` c +// Success (No error). +#define ERROR_NONE 0 + +// Memory allocation error. +#define ERROR_ALLOC (1 << 0) + +// Invalid FLAC compression level. +#define ERROR_INVALID_LEVEL (1 << 1) + +// Number of streams is zero. +#define ERROR_ZERO_NSTREAM (1 << 2) + +// Streamsize is zero. +#define ERROR_ZERO_STREAMSIZE (1 << 3) + +// Unable to set encoder compression level. +#define ERROR_ENCODE_SET_COMP_LEVEL (1 << 4) + +// Unable to set encoder block size. +#define ERROR_ENCODE_SET_BLOCK_SIZE (1 << 5) + +// Unable to set number of encoder channels. +#define ERROR_ENCODE_SET_CHANNELS (1 << 6) + +// Unable to set encoder bits per sample. +#define ERROR_ENCODE_SET_BPS (1 << 7) + +// Unable to initialize encoder. +#define ERROR_ENCODE_INIT (1 << 8) + +// Failed to run encoder. +#define ERROR_ENCODE_PROCESS (1 << 9) + +// Failed to finish encoding +#define ERROR_ENCODE_FINISH (1 << 10) + +// Failed to collect thread-local results. +#define ERROR_ENCODE_COLLECT (1 << 11) + +// Decoder failed to request bytes for remaining data. +#define ERROR_DECODE_READ_ZEROBUF (1 << 12) + +// Failed to initial decoder. +#define ERROR_DECODE_INIT (1 << 13) + +// Failed to process decoder bytes. +#define ERROR_DECODE_PROCESS (1 << 14) + +// Failed to finish decoding. +#define ERROR_DECODE_FINISH (1 << 15) + +// Decode stream size is invalid. +#define ERROR_DECODE_STREAMSIZE (1 << 16) + +// Decode sample range is invalid. +#define ERROR_DECODE_SAMPLE_RANGE (1 << 17) + +// Unable to seek in bytestream. +#define ERROR_DECODE_SEEK (1 << 18) + +// Unable to convert data to desired type. +#define ERROR_CONVERT_TYPE (1 << 19) +``` diff --git a/docs/install_requirements.sh b/docs/install_requirements.sh new file mode 100755 index 0000000..ecef961 --- /dev/null +++ b/docs/install_requirements.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +pip install \ + mkdocs \ + mkdocstrings \ + mkdocstrings-python \ + mkdocs-material \ + mkdocs-material-extensions \ + mkdocs-jupyter \ + mkdocs-print-site-plugin \ + mkdocstrings-c diff --git a/packaging/test_c_link/CMakeLists.txt b/packaging/test_c_link/CMakeLists.txt index e205f1f..ba71c6a 100644 --- a/packaging/test_c_link/CMakeLists.txt +++ b/packaging/test_c_link/CMakeLists.txt @@ -6,7 +6,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") # Add CTest enable_testing() -# Define the project with name and version from pyproject.toml +# Define the project with name and version project(c_test LANGUAGES C VERSION 1.0) # Use modern C diff --git a/src/flacarray/CMakeLists.txt b/src/flacarray/CMakeLists.txt index f1b5069..cd51f80 100644 --- a/src/flacarray/CMakeLists.txt +++ b/src/flacarray/CMakeLists.txt @@ -51,10 +51,9 @@ install(TARGETS flacarray add_library(flacarray_static STATIC) target_link_libraries(flacarray_static PRIVATE flcarr) -set_target_properties(flacarray_static PROPERTIES PUBLIC_HEADER ${libflcarr_HEADERS}) install(TARGETS flacarray_static - LIBRARY DESTINATION "${SKBUILD_PROJECT_NAME}/lib" + ARCHIVE DESTINATION "${SKBUILD_PROJECT_NAME}/lib" ) # Create the compiled test executable diff --git a/src/flacarray/libflacarray/decompress.c b/src/flacarray/libflacarray/decompress.c index ecab497..f6520cf 100644 --- a/src/flacarray/libflacarray/decompress.c +++ b/src/flacarray/libflacarray/decompress.c @@ -239,7 +239,7 @@ int decode( callback_data.n_channels = n_channels; callback_data.err = ERROR_NONE; - #pragma omp for schedule(static) + #pragma omp for schedule(static) if(use_threads) for (int64_t istream = 0; istream < n_stream; ++istream) { if (errors != ERROR_NONE) { // We already had a failure, skip over remaining loop iterations diff --git a/src/flacarray/libflacarray/flacarray.h b/src/flacarray/libflacarray/flacarray.h index 4e977ee..69d759d 100644 --- a/src/flacarray/libflacarray/flacarray.h +++ b/src/flacarray/libflacarray/flacarray.h @@ -17,26 +17,67 @@ // Error codes +// Success (No error). #define ERROR_NONE 0 + +// Memory allocation error. #define ERROR_ALLOC (1 << 0) + +// Invalid FLAC compression level. #define ERROR_INVALID_LEVEL (1 << 1) + +// Number of streams is zero. #define ERROR_ZERO_NSTREAM (1 << 2) + +// Streamsize is zero. #define ERROR_ZERO_STREAMSIZE (1 << 3) + +// Unable to set encoder compression level. #define ERROR_ENCODE_SET_COMP_LEVEL (1 << 4) + +// Unable to set encoder block size. #define ERROR_ENCODE_SET_BLOCK_SIZE (1 << 5) + +// Unable to set number of encoder channels. #define ERROR_ENCODE_SET_CHANNELS (1 << 6) + +// Unable to set encoder bits per sample. #define ERROR_ENCODE_SET_BPS (1 << 7) + +// Unable to initialize encoder. #define ERROR_ENCODE_INIT (1 << 8) + +// Failed to run encoder. #define ERROR_ENCODE_PROCESS (1 << 9) + +// Failed to finish encoding #define ERROR_ENCODE_FINISH (1 << 10) + +// Failed to collect thread-local results. #define ERROR_ENCODE_COLLECT (1 << 11) + +// Decoder failed to request bytes for remaining data. #define ERROR_DECODE_READ_ZEROBUF (1 << 12) + +// Failed to initial decoder. #define ERROR_DECODE_INIT (1 << 13) + +// Failed to process decoder bytes. #define ERROR_DECODE_PROCESS (1 << 14) + +// Failed to finish decoding. #define ERROR_DECODE_FINISH (1 << 15) + +// Decode stream size is invalid. #define ERROR_DECODE_STREAMSIZE (1 << 16) + +// Decode sample range is invalid. #define ERROR_DECODE_SAMPLE_RANGE (1 << 17) + +// Unable to seek in bytestream. #define ERROR_DECODE_SEEK (1 << 18) + +// Unable to convert data to desired type. #define ERROR_CONVERT_TYPE (1 << 19) // C-language arrays with a few STL-like features. @@ -206,6 +247,29 @@ void copy_interleaved_32_to_64(int64_t n_elem, int32_t * input, int64_t * output void free_interleaved(int32_t * interleaved); +// Public API + +/* + * Encode multiple 32bit integer streams. + * + * The input streams are passed as a single (concatenated), flat-packed buffer. + * This input `data` should have length `n_stream` X `stream_size`. + * + * The `n_bytes` and `starts` should be pre-allocated to hold one value per + * stream. These will be populated by the function. The `bytes` pointer + * is the address of a pointer that will be malloc'd and populated with + * the compressed bytes. The `starts` values are the starting byte offset + * of each stream in this output buffer. The `n_bytes` are the number + * of bytes following the starting point for each stream. + * + * @param[in] data The input integer data. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param level The FLAC compression level. + * @param[in] n_bytes The number of bytes per stream (filled by this function). + * @param[in] starts The starting byte per stream (filled by this function). + * @param[out] bytes The address of the pointer to the bytes for all streams. + */ int encode_i32( int32_t * const data, int64_t n_stream, @@ -216,6 +280,31 @@ int encode_i32( unsigned char ** bytes ); +/* + * Encode multiple 32bit integer streams with threads. + * + * The streams are distributed to the available threads, which independently + * accumulate the compressed bytes for their assigned streams. These + * thread-local compressed streams are then copied into the final output. + * + * The input streams are passed as a single (concatenated), flat-packed buffer. + * This input `data` should have length `n_stream` X `stream_size`. + * + * The `n_bytes` and `starts` should be pre-allocated to hold one value per + * stream. These will be populated by the function. The `bytes` pointer + * is the address of a pointer that will be malloc'd and populated with + * the compressed bytes. The `starts` values are the starting byte offset + * of each stream in this output buffer. The `n_bytes` are the number + * of bytes following the starting point for each stream. + * + * @param[in] data The input integer data. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param level The FLAC compression level. + * @param[in] n_bytes The number of bytes per stream (filled by this function). + * @param[in] starts The starting byte per stream (filled by this function). + * @param[out] bytes The address of the pointer to the bytes for all streams. + */ int encode_i32_threaded( int32_t * const data, int64_t n_stream, @@ -226,6 +315,27 @@ int encode_i32_threaded( unsigned char ** bytes ); +/* + * Encode multiple 64bit integer streams. + * + * The input streams are passed as a single (concatenated), flat-packed buffer. + * This input `data` should have length `n_stream` X `stream_size`. + * + * The `n_bytes` and `starts` should be pre-allocated to hold one value per + * stream. These will be populated by the function. The `bytes` pointer + * is the address of a pointer that will be malloc'd and populated with + * the compressed bytes. The `starts` values are the starting byte offset + * of each stream in this output buffer. The `n_bytes` are the number + * of bytes following the starting point for each stream. + * + * @param[in] data The input integer data. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param level The FLAC compression level. + * @param[in] n_bytes The number of bytes per stream (filled by this function). + * @param[in] starts The starting byte per stream (filled by this function). + * @param[out] bytes The address of the pointer to the bytes for all streams. + */ int encode_i64( int64_t * const data, int64_t n_stream, @@ -236,6 +346,31 @@ int encode_i64( unsigned char ** bytes ); +/* + * Encode multiple 64bit integer streams with threads. + * + * The streams are distributed to the available threads, which independently + * accumulate the compressed bytes for their assigned streams. These + * thread-local compressed streams are then copied into the final output. + * + * The input streams are passed as a single (concatenated), flat-packed buffer. + * This input `data` should have length `n_stream` X `stream_size`. + * + * The `n_bytes` and `starts` should be pre-allocated to hold one value per + * stream. These will be populated by the function. The `bytes` pointer + * is the address of a pointer that will be malloc'd and populated with + * the compressed bytes. The `starts` values are the starting byte offset + * of each stream in this output buffer. The `n_bytes` are the number + * of bytes following the starting point for each stream. + * + * @param[in] data The input integer data. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param level The FLAC compression level. + * @param[in] n_bytes The number of bytes per stream (filled by this function). + * @param[in] starts The starting byte per stream (filled by this function). + * @param[out] bytes The address of the pointer to the bytes for all streams. + */ int encode_i64_threaded( int64_t * const data, int64_t n_stream, @@ -246,6 +381,32 @@ int encode_i64_threaded( unsigned char ** bytes ); +/* + * Decode multiple 32bit integer streams. + * + * The input `bytes` are the concatenated buffers for all streams. + * + * The `starts` values are the starting byte offset of each stream in `bytes`. + * The `n_bytes` are the number of bytes following the starting point for each + * stream. + * + * The `first_sample` and `last_sample` arguments control the subset of values + * that are extracted from each stream. To extract all values, set `first_sample` + * to zero and `last_sample` to `stream_size`. + * + * The output data buffer should be pre-allocated to hold + * `n_stream` X (`last_sample` - `first_sample`) elements. + * + * @param[in] bytes The concatenated compressed bytes for all streams. + * @param[in] n_bytes The number of bytes per stream. + * @param[in] starts The starting byte per stream. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param first_sample The first output sample to decode for all streams. + * @param last_sample The last output sample (exclusive) to decode for all streams. + * @param[out] data The output integer data. + * @param[in] use_threads If true, used threads for decoding. + */ int decode_i32( unsigned char * const bytes, int64_t * const starts, @@ -258,6 +419,32 @@ int decode_i32( bool use_threads ); +/* + * Decode multiple 64bit integer streams. + * + * The input `bytes` are the concatenated buffers for all streams. + * + * The `starts` values are the starting byte offset of each stream in `bytes`. + * The `n_bytes` are the number of bytes following the starting point for each + * stream. + * + * The `first_sample` and `last_sample` arguments control the subset of values + * that are extracted from each stream. To extract all values, set `first_sample` + * to zero and `last_sample` to `stream_size`. + * + * The output data buffer should be pre-allocated to hold + * `n_stream` X (`last_sample` - `first_sample`) elements. + * + * @param[in] bytes The concatenated compressed bytes for all streams. + * @param[in] n_bytes The number of bytes per stream. + * @param[in] starts The starting byte per stream. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param first_sample The first output sample to decode for all streams. + * @param last_sample The last output sample (exclusive) to decode for all streams. + * @param[out] data The output integer data. + * @param[in] use_threads If true, used threads for decoding. + */ int decode_i64( unsigned char * const bytes, int64_t * const starts, @@ -270,8 +457,25 @@ int decode_i64( bool use_threads ); -// Type conversion - +/* + * Convert 32bit floating point data to integers. + * + * The `input` data is the concatenated buffers for all streams. The `quanta` + * array is the smallest floating point value for each stream that will be + * preserved in a lossless way going from float to int and back. + * + * The `output` data buffer should be pre-allocated to hold + * `n_stream` X `stream_size` elements. The `offsets` and `gains` should be + * pre-allocated to hold one value per stream. + * + * @param[in] input The concatenated input data from multiple streams. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param[in] quanta The quantization value for each stream. + * @param[out] output The output integer data. + * @param[out] offsets The floating point offset subtracted from each input stream. + * @param[out] gains The gain applied to each input stream before truncation. + */ int float32_to_int32( float const * input, int64_t n_stream, @@ -282,6 +486,25 @@ int float32_to_int32( float * gains ); +/* + * Convert 64bit floating point data to integers. + * + * The `input` data is the concatenated buffers for all streams. The `quanta` + * array is the smallest floating point value for each stream that will be + * preserved in a lossless way going from float to int and back. + * + * The `output` data buffer should be pre-allocated to hold + * `n_stream` X `stream_size` elements. The `offsets` and `gains` should be + * pre-allocated to hold one value per stream. + * + * @param[in] input The concatenated input data from multiple streams. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param[in] quanta The quantization value for each stream. + * @param[out] output The output integer data. + * @param[out] offsets The floating point offset subtracted from each input stream. + * @param[out] gains The gain applied to each input stream before truncation. + */ int float64_to_int64( double const * input, int64_t n_stream, @@ -292,15 +515,23 @@ int float64_to_int64( double * gains ); -void int64_to_float64( - int64_t const * input, - int64_t n_stream, - int64_t stream_size, - double const * offsets, - double const * gains, - double * output -); - +/* + * Convert 32bit integers to floating point data. + * + * The `input` data is the concatenated buffers for all streams. To restore + * the original floating point data, the integer values are scaled by 1/gain + * and then the offset is added. + * + * The `output` data buffer should be pre-allocated to hold + * `n_stream` X `stream_size` elements. + * + * @param[in] input The concatenated input data from multiple streams. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param[in] offsets The floating point offset subtracted from each original stream. + * @param[in] gains The gain applied to each original stream before truncation. + * @param[out] output The restored floating point data. + */ void int32_to_float32( int32_t const * input, int64_t n_stream, @@ -310,3 +541,28 @@ void int32_to_float32( float * output ); +/* + * Convert 64bit integers to floating point data. + * + * The `input` data is the concatenated buffers for all streams. To restore + * the original floating point data, the integer values are scaled by 1/gain + * and then the offset is added. + * + * The `output` data buffer should be pre-allocated to hold + * `n_stream` X `stream_size` elements. + * + * @param[in] input The concatenated input data from multiple streams. + * @param n_stream The number of input streams. + * @param stream_size The length of each stream. + * @param[in] offsets The floating point offset subtracted from each original stream. + * @param[in] gains The gain applied to each original stream before truncation. + * @param[out] output The restored floating point data. + */ +void int64_to_float64( + int64_t const * input, + int64_t n_stream, + int64_t stream_size, + double const * offsets, + double const * gains, + double * output +); diff --git a/src/flacarray/scripts/config.py.in b/src/flacarray/scripts/config.py.in index 452d329..80628bd 100644 --- a/src/flacarray/scripts/config.py.in +++ b/src/flacarray/scripts/config.py.in @@ -98,7 +98,7 @@ def cli(): elif args.libraries: # Print full library paths if args.static: - libpath = os.path.join(pkg_libdir, f"libflacarray{static_suffix}") + libpath = os.path.join(pkg_libdir, f"libflacarray_static{static_suffix}") else: libpath = os.path.join(pkg_libdir, f"libflacarray{shared_suffix}") print(f"{libpath};@CONF_EXTRA_LINK@") From 8e30bd6be68bf1a8bc757456409cba0b254383ef Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Sun, 15 Feb 2026 10:39:33 -0800 Subject: [PATCH 12/14] Revert tweak to conditional thread use --- src/flacarray/libflacarray/decompress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flacarray/libflacarray/decompress.c b/src/flacarray/libflacarray/decompress.c index f6520cf..ecab497 100644 --- a/src/flacarray/libflacarray/decompress.c +++ b/src/flacarray/libflacarray/decompress.c @@ -239,7 +239,7 @@ int decode( callback_data.n_channels = n_channels; callback_data.err = ERROR_NONE; - #pragma omp for schedule(static) if(use_threads) + #pragma omp for schedule(static) for (int64_t istream = 0; istream < n_stream; ++istream) { if (errors != ERROR_NONE) { // We already had a failure, skip over remaining loop iterations From d9459a57ee99e81a1e3e5f111bcd0667c3454118 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Sun, 15 Feb 2026 14:09:25 -0800 Subject: [PATCH 13/14] Small tweaks to the compiled test program --- CMakeLists.txt | 2 ++ pyproject.toml | 4 ++-- src/flacarray/libflacarray/test.c | 14 ++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7bf3aae..c56db2b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,8 @@ if (NOT SKBUILD) ") endif() +message(STATUS "CMAKE_BUILD_TYPE set to '${CMAKE_BUILD_TYPE}'") + # Auxiliary files list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") diff --git a/pyproject.toml b/pyproject.toml index 50bbd3c..03b82f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,11 +63,11 @@ cmake.source-dir = "." # Use Make as a fallback if a suitable Ninja executable is not found. ninja.make-fallback = true # The logging level to display. -logging.level = "WARNING" +logging.level = "INFO" # Files to include in the SDist even if they are skipped by default. sdist.include = [] # Files to exclude from the SDist even if they are included by default. -sdist.exclude = ["build", "wheelhouse", "docs/site"] +sdist.exclude = [".github", "build", "wheelhouse", "docs/site"] # Try to build a reproducible distribution. sdist.reproducible = true # If set to True, CMake will be run before building the SDist. diff --git a/src/flacarray/libflacarray/test.c b/src/flacarray/libflacarray/test.c index c411051..0591b8d 100644 --- a/src/flacarray/libflacarray/test.c +++ b/src/flacarray/libflacarray/test.c @@ -167,11 +167,9 @@ int verify( } -void test_32bit() { +void test_32bit(int64_t n_streams, int64_t stream_len) { fprintf(stderr, "============= 32bit Tests ===============\n"); - int64_t n_streams = 10; - int64_t stream_len = 1000000; int64_t input_bytes = n_streams * stream_len * sizeof(int32_t); uint32_t level = 5; @@ -419,11 +417,9 @@ void test_32bit() { } -void test_64bit() { +void test_64bit(int64_t n_streams, int64_t stream_len) { fprintf(stderr, "============= 64bit Tests ===============\n"); - int64_t n_streams = 10; - int64_t stream_len = 1000000; int64_t input_bytes = n_streams * stream_len * sizeof(int64_t); uint32_t level = 5; @@ -683,7 +679,9 @@ void test_64bit() { int main(int argc, char *argv[]) { fprintf(stderr, "=========================================\n\n"); fprintf(stderr, "Using libflacarray version %s\n\n", FLACARRAY_VERSION); - test_32bit(); - test_64bit(); + int64_t n_streams = 100; + int64_t stream_len = 100000; + test_32bit(n_streams, stream_len); + test_64bit(n_streams, stream_len); return 0; } From 80eb457f7190e5cc6b2a4edc81a4bb61ead2ac25 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Mon, 16 Feb 2026 08:21:33 -0800 Subject: [PATCH 14/14] Fix message typo --- src/flacarray/mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flacarray/mpi.py b/src/flacarray/mpi.py index b76b86f..2af0547 100644 --- a/src/flacarray/mpi.py +++ b/src/flacarray/mpi.py @@ -25,7 +25,7 @@ use_mpi = True except Exception: # There could be many possible exceptions raised... - log.debug("mpi4py not found- MPI operations disabled") + log.debug("mpi4py not importable- MPI operations disabled") use_mpi = False MPI = None