diff --git a/CHANGELOG.md b/CHANGELOG.md index af04351e..6fd99ace 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Change log +## modelstore 0.0.82 ([February 2026](https://github.com/operatorai/modelstore/pull/293)) + +**πŸ†• New functionality** + +* Added support for [statsmodels](https://www.statsmodels.org/) [#293](https://github.com/operatorai/modelstore/pull/293) +* Added support for [Backblaze B2](https://www.backblaze.com/cloud-storage) storage backend via `ModelStore.from_backblaze()` [#289](https://github.com/operatorai/modelstore/pull/289), (thanks [jeronimodeleon](https://github.com/jeronimodeleon)) + +**πŸ› Bug fixes & general updates** + +* Updated AWS and Backblaze storage backends [#292](https://github.com/operatorai/modelstore/pull/292) +* Removed deprecated `pkg_resources` from the project dependencies [#288](https://github.com/operatorai/modelstore/pull/288), (thanks [divineod](https://github.com/divineod)) +* Fixed `pkg_resources` migration issues: `PackageNotFoundError` import and platform-specific `np.float96` removal [#291](https://github.com/operatorai/modelstore/pull/291) +* Migrated from `pyenv` to `uv` for Python environment management [#290](https://github.com/operatorai/modelstore/pull/290) +* Stopped supporting Python versions that are past their EOL + ## modelstore 0.0.81 ([May 2024](https://github.com/operatorai/modelstore/pull/270)) **πŸ†• New functionality** diff --git a/bin/_build_library b/bin/_build_library index 4cdbe72b..147d50e4 100755 --- a/bin/_build_library +++ b/bin/_build_library @@ -1,13 +1,11 @@ #!/bin/bash set -e -VIRTUALENV_NAME=$(pyenv local) +echo -e "\n ⏱ Building library..." -echo "\n ⏱ Building library: $VIRTUALENV_NAME" +rm -rf dist build modelstore.egg-info +uv pip install --upgrade pip setuptools wheel -rm -rf dist build modelstore.egg_info -pip install --upgrade pip setuptools wheel +uv run python setup.py sdist bdist_wheel -python setup.py sdist bdist_wheel - -echo "\n βœ… Done: results are in the dist/ directory." +echo -e "\n βœ… Done: results are in the dist/ directory." diff --git a/bin/_cleanup b/bin/_cleanup index 8c1e919e..7d058cdf 100755 --- a/bin/_cleanup +++ b/bin/_cleanup @@ -7,4 +7,7 @@ rm -rf *.egg-info rm -rf build rm -rf dist +echo -e "\n 🧼 Removing docker things" +docker system prune --all + echo -e "\n πŸŽ‰ Done." diff --git a/bin/_release_prod b/bin/_release_prod index c22cc495..64839f65 100755 --- a/bin/_release_prod +++ b/bin/_release_prod @@ -1,15 +1,15 @@ #!/bin/bash set -e -echo "\n ⏱ Uploading library to pypi..." +echo -e "\n ⏱ Uploading library to pypi..." -pip install --upgrade twine +uv pip install --upgrade twine -twine check dist/* +uv run twine check dist/* -twine upload \ +uv run twine upload \ --username $TWINE_PROD_USERNAME \ --password $TWINE_PROD_PWD \ dist/* -echo "\n πŸŽ‰ Done." +echo -e "\n πŸŽ‰ Done." diff --git a/bin/_release_test b/bin/_release_test index 44cbc76d..be9f9032 100755 --- a/bin/_release_test +++ b/bin/_release_test @@ -1,15 +1,15 @@ #!/bin/bash set -e -echo "\n ⏱ Uploading library to testpypi..." +echo -e "\n ⏱ Uploading library to testpypi..." -pip install --upgrade twine +uv pip install --upgrade twine -twine check dist/* +uv run twine check dist/* -twine upload \ +uv run twine upload \ --username $TWINE_TEST_USERNAME \ --password $TWINE_TEST_PWD \ - --repository testpypi dist/* + --repository testpypi dist/* --verbose -echo "\n 🚒 Done." +echo -e "\n 🚒 Done." diff --git a/examples/Makefile-example b/examples/Makefile-example index d188bb91..702cb2b1 100644 --- a/examples/Makefile-example +++ b/examples/Makefile-example @@ -1,37 +1,33 @@ -VIRTUALENV_NAME=modelstore.$(shell pwd | rev | cut -d '/' -f 1 | rev) REPO_ROOT=$(shell cd ../../ && pwd) -.PHONY: name pyenv pyenv-local pyenv-prod pyenv-test pyenv-uninstall refresh gcloud +.PHONY: uninstall install install-local install-test install-prod refresh gcloud -name: - @echo $(VIRTUALENV_NAME) - -pyenv-uninstall: - @$(REPO_ROOT)/bin/_pyenv_uninstall $(VIRTUALENV_NAME) +uninstall: + @if [ -d ".venv" ]; then rm -rf .venv; echo " βœ… Removed .venv"; else echo " βœ… Nothing to do."; fi gcloud: @gcloud components update @gcloud auth application-default login -pyenv: pyenv-uninstall - # @$(REPO_ROOT)/bin/_setup_brew - @$(REPO_ROOT)/bin/_pyenv_install $(VIRTUALENV_NAME) - pip install -r https://raw.githubusercontent.com/ultralytics/yolov5/master/requirements.txt +install: uninstall + @uv venv + @uv pip install --upgrade pip setuptools wheel + @uv pip install -r requirements.txt -pyenv-local: pyenv - pip uninstall -y modelstore - pip install -e $(REPO_ROOT) +install-local: install + @uv pip uninstall modelstore + @uv pip install -e $(REPO_ROOT) -pyenv-test: pyenv - pip uninstall -y modelstore - pip install --no-cache-dir -i https://test.pypi.org/simple/ modelstore +install-test: install + @uv pip uninstall modelstore + @uv pip install --no-cache-dir --extra-index-url https://test.pypi.org/simple/ --index-strategy unsafe-best-match modelstore==0.0.82 -pyenv-prod: pyenv - pip uninstall -y modelstore - pip install --no-cache-dir --upgrade modelstore +install-prod: install + @uv pip uninstall modelstore + @uv pip install --no-cache-dir --upgrade modelstore refresh: @echo "\n πŸ”΅ Refreshing installation of modelstore" - pip install --upgrade pip setuptools wheel - pip uninstall -y modelstore - pip install --no-cache-dir -e $(REPO_ROOT) + @uv pip install --upgrade pip setuptools wheel + @uv pip uninstall modelstore + @uv pip install --no-cache-dir -e $(REPO_ROOT) diff --git a/examples/README.md b/examples/README.md index d021180b..e7f24a1f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,13 +2,13 @@ This directory contains examples of training models and storing them into a model store over different types of storage. -The Python script in `examples-by-ml-model` iterates over all of the supported ML frameworks and all of the supported storage types. For each pair, it trains a model, uploads it to storage, and then downloads/loads it back. +The Python script in `examples-by-ml-model` iterates over all of the supported ML frameworks and all of the supported storage types. For each pair, it trains a model, uploads it to storage, and then downloads/loads it back. The bash script `cli-examples` has exaples of how to run `python -m modelstore` commands. ## Pre-requisites -As with the main library, these scripts have been developed using [pyenv](https://github.com/pyenv/pyenv) and [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv). +As with the main library, these scripts have been developed using [uv](https://github.com/astral-sh/uv). ## Set up - examples by ML model @@ -24,7 +24,7 @@ And then you can use this `Makefile` command that creates a new virtual environm and installs all of the requirements: ```bash -❯ make pyenv +❯ make install ``` ## Running all of the examples diff --git a/examples/cli-examples/model.py b/examples/cli-examples/model.py index 24eb2f5f..da346cb8 100644 --- a/examples/cli-examples/model.py +++ b/examples/cli-examples/model.py @@ -33,7 +33,7 @@ def train_and_save(): "max_depth": 4, "min_samples_split": 5, "learning_rate": 0.01, - "loss": "ls", + "loss": "squared_error", } model = GradientBoostingRegressor(**params) model.fit(X_train, y_train) diff --git a/examples/cli-examples/run-all.sh b/examples/cli-examples/run-all.sh index 9695d067..b1677426 100755 --- a/examples/cli-examples/run-all.sh +++ b/examples/cli-examples/run-all.sh @@ -5,14 +5,14 @@ TARGET_DIR="downloaded_model" FILE_NAME="model.joblib" echo "\nπŸ”΅ Training a model...\n" -python model.py +uv run python model.py echo "\nπŸ”΅ Uploading the model via the CLI...\n" -MODEL_ID=$(python -m modelstore upload "$DOMAIN_NAME" "$FILE_NAME") +MODEL_ID=$(uv run python -m modelstore upload "$DOMAIN_NAME" "$FILE_NAME") echo "\nπŸ”΅ Downloading model=$MODEL_ID via the CLI...\n" mkdir -p "$TARGET_DIR" -python -m modelstore download "$DOMAIN_NAME" "$MODEL_ID" "$TARGET_DIR" +uv run python -m modelstore download "$DOMAIN_NAME" "$MODEL_ID" "$TARGET_DIR" echo "\nβœ… Done! Cleaning up..." diff --git a/examples/examples-by-ml-library/Dockerfile b/examples/examples-by-ml-library/Dockerfile new file mode 100644 index 00000000..2b0a9492 --- /dev/null +++ b/examples/examples-by-ml-library/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.11-slim +WORKDIR /usr/src/app + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y build-essential git ninja-build ccache libopenblas-dev libopencv-dev cmake && \ + apt-get install -y gcc mono-mcs g++ && \ + apt-get install -y default-jdk && \ + apt-get install -y libhdf5-dev && \ + rm -rf /var/lib/apt/lists/* + +# Install modelstore library from test.pypi.org +RUN pip install --upgrade pip setuptools wheel && \ + pip install --index-url https://test.pypi.org/simple/ \ + --extra-index-url https://pypi.org/simple/ \ + modelstore + +# Install example dependencies +COPY requirements.txt ./requirements.txt +RUN pip install -r requirements.txt + +# Copy example source +COPY . ./examples +WORKDIR /usr/src/app/examples + +ENTRYPOINT ["python", "main.py"] diff --git a/examples/examples-by-ml-library/Makefile b/examples/examples-by-ml-library/Makefile index a10989b1..1948a556 100644 --- a/examples/examples-by-ml-library/Makefile +++ b/examples/examples-by-ml-library/Makefile @@ -1,6 +1,20 @@ include ../Makefile-example -.PHONY: run +DOCKER_IMAGE ?= modelstore-examples +MODELSTORE_IN ?= filesystem +ML_FRAMEWORK ?= sklearn + +.PHONY: run docker-build docker-run + run: @./run-all.sh - \ No newline at end of file + +docker-build: + @docker build -f Dockerfile -t $(DOCKER_IMAGE) . + +docker-run: + @docker run --rm \ + -e MODEL_STORE_ROOT_PREFIX=/tmp/modelstore \ + $(DOCKER_IMAGE) \ + --modelstore-in $(MODELSTORE_IN) \ + --ml-framework $(ML_FRAMEWORK) diff --git a/examples/examples-by-ml-library/libraries/skorch_example.py b/examples/examples-by-ml-library/libraries/skorch_example.py index 8d826683..755f50c0 100644 --- a/examples/examples-by-ml-library/libraries/skorch_example.py +++ b/examples/examples-by-ml-library/libraries/skorch_example.py @@ -62,7 +62,13 @@ def train_and_upload(modelstore: ModelStore) -> dict: def load_and_test(modelstore: ModelStore, model_domain: str, model_id: str): # Load the model back into memory! print(f'‡️ Loading the skorch "{model_domain}" domain model={model_id}') - model = modelstore.load(model_domain, model_id) + loaded = modelstore.load(model_domain, model_id) + + # When multiple files are saved, modelstore returns a dict + if isinstance(loaded, dict): + model = loaded["skorch"] + else: + model = loaded # Run some example predictions _, X_test, _, y_test = load_regression_dataset(as_numpy=True) diff --git a/examples/examples-by-ml-library/main.py b/examples/examples-by-ml-library/main.py index b134c59e..7ab3da49 100644 --- a/examples/examples-by-ml-library/main.py +++ b/examples/examples-by-ml-library/main.py @@ -11,72 +11,41 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import importlib import sys import click -from libraries import ( - annoy_example, - catboost_example, - causalml_example, - fastai_example, - gensim_example, - keras_example, - lightgbm_example, - onnx_lightgbm_example, - onnx_sklearn_example, - prophet_example, - pyspark_example, - pytorch_example, - pytorch_lightning_example, - raw_file_example, - shap_example, - sklearn_example, - sklearn_with_explainer_example, - sklearn_with_extras_example, - skorch_example, - tensorflow_example, - xgboost_booster_example, - xgboost_example, - yolo_example -) -from libraries.huggingface import ( - distilbert, - dpt, - gpt2_pytorch, - gpt2_tensorflow, - sam -) from modelstores import MODELSTORES, create_model_store EXAMPLES = { - "annoy": annoy_example, - "catboost": catboost_example, - "causalml": causalml_example, - "dpt": dpt, - "fastai": fastai_example, - "file": raw_file_example, - "gensim": gensim_example, - "hf-distilbert": distilbert, - "hf-gpt2-pt": gpt2_pytorch, - "hf-gpt2-tf": gpt2_tensorflow, - "keras": keras_example, - "lightgbm": lightgbm_example, - "onnx-sklearn": onnx_sklearn_example, - "onnx-lightgbm": onnx_lightgbm_example, - "prophet": prophet_example, - "pyspark": pyspark_example, - "pytorch": pytorch_example, - "pytorch-lightning": pytorch_lightning_example, - "segment-anything": sam, - "shap": shap_example, - "sklearn": sklearn_example, - "sklearn-with-explainer": sklearn_with_explainer_example, - "sklearn-with-extras": sklearn_with_extras_example, - "skorch": skorch_example, - "tensorflow": tensorflow_example, - "xgboost": xgboost_example, - "xgboost-booster": xgboost_booster_example, - "yolov5": yolo_example, + "annoy": "libraries.annoy_example", + "catboost": "libraries.catboost_example", + "causalml": "libraries.causalml_example", + "dpt": "libraries.huggingface.dpt", + "fastai": "libraries.fastai_example", + "file": "libraries.raw_file_example", + "gensim": "libraries.gensim_example", + "hf-distilbert": "libraries.huggingface.distilbert", + "hf-gpt2-pt": "libraries.huggingface.gpt2_pytorch", + "hf-gpt2-tf": "libraries.huggingface.gpt2_tensorflow", + "keras": "libraries.keras_example", + "lightgbm": "libraries.lightgbm_example", + "onnx-sklearn": "libraries.onnx_sklearn_example", + "onnx-lightgbm": "libraries.onnx_lightgbm_example", + "prophet": "libraries.prophet_example", + "pyspark": "libraries.pyspark_example", + "pytorch": "libraries.pytorch_example", + "pytorch-lightning": "libraries.pytorch_lightning_example", + "segment-anything": "libraries.huggingface.sam", + "shap": "libraries.shap_example", + "sklearn": "libraries.sklearn_example", + "sklearn-with-explainer": "libraries.sklearn_with_explainer_example", + "sklearn-with-extras": "libraries.sklearn_with_extras_example", + "skorch": "libraries.skorch_example", + "tensorflow": "libraries.tensorflow_example", + "xgboost": "libraries.xgboost_example", + "xgboost-booster": "libraries.xgboost_booster_example", + "yolov5": "libraries.yolo_example", } @@ -109,7 +78,7 @@ def main(modelstore_in, ml_framework): # Create a model store instance modelstore = create_model_store(modelstore_in) - example = EXAMPLES[ml_framework] + example = importlib.import_module(EXAMPLES[ml_framework]) # Demo how we train and upload a model meta_data = example.train_and_upload(modelstore) diff --git a/examples/examples-by-ml-library/modelstores.py b/examples/examples-by-ml-library/modelstores.py index 519d9f77..b051e9f1 100644 --- a/examples/examples-by-ml-library/modelstores.py +++ b/examples/examples-by-ml-library/modelstores.py @@ -13,6 +13,15 @@ # limitations under the License. import os +try: + # causalml uses lazy submodule loading; modelstore's causalml manager accesses + # causalml.inference.meta.base and causalml.propensity as attributes without + # importing them first, causing AttributeError. Prime them here. + import causalml.inference.meta.base # noqa: F401 + import causalml.propensity # noqa: F401 +except ImportError: + pass + from modelstore import ModelStore from modelstore.storage.aws import AWSStorage from modelstore.storage.azure import AzureBlobStorage @@ -49,6 +58,7 @@ def create_file_system_model_store() -> ModelStore: "MODEL_STORE_ROOT_PREFIX", os.path.expanduser("~"), ) + os.makedirs(root_dir, exist_ok=True) print(f"🏦 Creating store in: {root_dir}") return ModelStore.from_file_system(root_directory=root_dir) diff --git a/examples/examples-by-ml-library/requirements.txt b/examples/examples-by-ml-library/requirements.txt index cfb92c8c..f4192431 100644 --- a/examples/examples-by-ml-library/requirements.txt +++ b/examples/examples-by-ml-library/requirements.txt @@ -14,18 +14,16 @@ minio # Data / dependencies for ML libraries numpy==1.23.5 -numba>=0.55.1 +numba>=0.55.1,<0.61.0 # llvmlite >=0.44 dropped macOS x86_64 wheels Cython>=0.29.28 python-Levenshtein>=0.12.2 -#Β Prophet -pystan>=2.19.1.1 # required to be installed before prophet - -# Machine learning libraries +#Β # Machine learning libraries annoy catboost causalml fastai +ipython gensim lightgbm<4.0.0 # ImportError: cannot import name 'FEATURE_IMPORTANCE_TYPE_MAPPER' from 'lightgbm.basic' onnx @@ -39,10 +37,10 @@ scipy==1.10.1 # More recent versions were not compatible with Gensim releases ht shap skl2onnx skorch -tensorflow; sys_platform != 'darwin' -tensorflow-macos; sys_platform == 'darwin' +tensorflow tf-keras -transformers +transformers<4.50.0 # TFGPT2LMHeadModel was removed in 4.50 torch torchvision +ultralytics xgboost diff --git a/examples/examples-by-ml-library/run-all.sh b/examples/examples-by-ml-library/run-all.sh index 5bb00615..eef5bf91 100755 --- a/examples/examples-by-ml-library/run-all.sh +++ b/examples/examples-by-ml-library/run-all.sh @@ -1,5 +1,11 @@ set -e -backends=( filesystem aws-s3 google-cloud-storage azure-container minio ) + +DOCKER_IMAGE="${DOCKER_IMAGE:-modelstore-examples}" + +echo -e "\n πŸ”΅ Building Docker image: $DOCKER_IMAGE" +docker build -f Dockerfile -t "$DOCKER_IMAGE" . +echo -e "\n βœ… Docker image built." + frameworks=( annoy catboost causalml fastai file gensim keras lightgbm \ onnx-sklearn onnx-lightgbm prophet pyspark pytorch pytorch-lightning \ sklearn sklearn-with-explainer sklearn-with-extras skorch xgboost xgboost-booster \ @@ -7,10 +13,41 @@ frameworks=( annoy catboost causalml fastai file gensim keras lightgbm \ for framework in "${frameworks[@]}" do - for backend in "${backends[@]}" - do - echo -e "\n πŸ”΅ Running the $framework example in a $backend modelstore." - python main.py --modelstore-in $backend --ml-framework $framework - echo -e "\n βœ… Finished running the $framework example in $backend." - done + echo -e "\n πŸ”΅ Running the $framework example in a filesystem modelstore." + docker run --rm \ + -e MODEL_STORE_ROOT_PREFIX=/tmp/modelstore \ + $DOCKER_IMAGE --modelstore-in filesystem --ml-framework $framework + echo -e "\n βœ… Finished running the $framework example in filesystem." + + echo -e "\n πŸ”΅ Running the $framework example in an aws-s3 modelstore." + docker run --rm \ + -e MODEL_STORE_AWS_BUCKET=$MODEL_STORE_AWS_BUCKET \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + $DOCKER_IMAGE --modelstore-in aws-s3 --ml-framework $framework + echo -e "\n βœ… Finished running the $framework example in aws-s3." + + echo -e "\n πŸ”΅ Running the $framework example in a google-cloud-storage modelstore." + docker run --rm \ + -e MODEL_STORE_GCP_PROJECT=$MODEL_STORE_GCP_PROJECT \ + -e MODEL_STORE_GCP_BUCKET=$MODEL_STORE_GCP_BUCKET \ + -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcloud-key.json \ + -v $GOOGLE_APPLICATION_CREDENTIALS:/tmp/gcloud-key.json:ro \ + $DOCKER_IMAGE --modelstore-in google-cloud-storage --ml-framework $framework + echo -e "\n βœ… Finished running the $framework example in google-cloud-storage." + + echo -e "\n πŸ”΅ Running the $framework example in an azure-container modelstore." + docker run --rm \ + -e MODEL_STORE_AZURE_CONTAINER=$MODEL_STORE_AZURE_CONTAINER \ + -e AZURE_STORAGE_CONNECTION_STRING=$AZURE_STORAGE_CONNECTION_STRING \ + $DOCKER_IMAGE --modelstore-in azure-container --ml-framework $framework + echo -e "\n βœ… Finished running the $framework example in azure-container." + + echo -e "\n πŸ”΅ Running the $framework example in a minio modelstore." + docker run --rm \ + -e MODEL_STORE_AWS_BUCKET=$MODEL_STORE_AWS_BUCKET \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + $DOCKER_IMAGE --modelstore-in minio --ml-framework $framework + echo -e "\n βœ… Finished running the $framework example in minio." done diff --git a/modelstore/models/transformers.py b/modelstore/models/transformers.py index 272b36b3..8c012f00 100644 --- a/modelstore/models/transformers.py +++ b/modelstore/models/transformers.py @@ -166,17 +166,21 @@ def load(self, model_path: str, meta_data: metadata.Summary) -> Any: logger.debug("Loading with AutoModel...") model = AutoModel.from_pretrained(model_dir) else: - from transformers import TFAutoModel, TFGPT2LMHeadModel + from transformers import TFAutoModel # In examples-by-ml-library/libraries/huggingface/gpt2*.py, we want # to load a GPT2 model with a language model head. If we just # load the model with TFAutoModel, then it won't have this. # This is a hack to get around that, like we did in the XGBoost # manager, and currently does not generalise beyond this case - model_types = { - "TFGPT2LMHeadModel": TFGPT2LMHeadModel, - # @TODO add other model types - } + # Note: TFGPT2LMHeadModel was removed in transformers 4.50 + model_types = {} + try: + from transformers import TFGPT2LMHeadModel + + model_types["TFGPT2LMHeadModel"] = TFGPT2LMHeadModel + except ImportError: + pass model_type = meta_data.model_type().type if model_type in model_types: logger.debug("Loading with %s...", model_type) diff --git a/setup.py b/setup.py index 5fff72f7..87e79caa 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="modelstore", - version="0.0.81", + version="0.0.82", packages=find_packages(exclude=["tests", "examples", "docs", "workflows"]), include_package_data=True, description="modelstore is a library for versioning, exporting, storing, and loading machine learning models",