Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,20 @@
# Change log

## modelstore 0.0.82 ([February 2026](https://github.com/operatorai/modelstore/pull/293))

**🆕 New functionality**

* Added support for [statsmodels](https://www.statsmodels.org/) [#293](https://github.com/operatorai/modelstore/pull/293)
* Added support for [Backblaze B2](https://www.backblaze.com/cloud-storage) storage backend via `ModelStore.from_backblaze()` [#289](https://github.com/operatorai/modelstore/pull/289), (thanks [jeronimodeleon](https://github.com/jeronimodeleon))

**🐛 Bug fixes & general updates**

* Updated AWS and Backblaze storage backends [#292](https://github.com/operatorai/modelstore/pull/292)
* Removed deprecated `pkg_resources` from the project dependencies [#288](https://github.com/operatorai/modelstore/pull/288), (thanks [divineod](https://github.com/divineod))
* Fixed `pkg_resources` migration issues: `PackageNotFoundError` import and platform-specific `np.float96` removal [#291](https://github.com/operatorai/modelstore/pull/291)
* Migrated from `pyenv` to `uv` for Python environment management [#290](https://github.com/operatorai/modelstore/pull/290)
* Stopped supporting Python versions that are past their EOL

## modelstore 0.0.81 ([May 2024](https://github.com/operatorai/modelstore/pull/270))

**🆕 New functionality**
Expand Down
12 changes: 5 additions & 7 deletions bin/_build_library
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
#!/bin/bash
set -e

VIRTUALENV_NAME=$(pyenv local)
echo -e "\n ⏱ Building library..."

echo "\n ⏱ Building library: $VIRTUALENV_NAME"
rm -rf dist build modelstore.egg-info
uv pip install --upgrade pip setuptools wheel

rm -rf dist build modelstore.egg_info
pip install --upgrade pip setuptools wheel
uv run python setup.py sdist bdist_wheel

python setup.py sdist bdist_wheel

echo "\n ✅ Done: results are in the dist/ directory."
echo -e "\n ✅ Done: results are in the dist/ directory."
3 changes: 3 additions & 0 deletions bin/_cleanup
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,7 @@ rm -rf *.egg-info
rm -rf build
rm -rf dist

echo -e "\n 🧼 Removing docker things"
docker system prune --all

echo -e "\n 🎉 Done."
10 changes: 5 additions & 5 deletions bin/_release_prod
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
#!/bin/bash
set -e

echo "\n ⏱ Uploading library to pypi..."
echo -e "\n ⏱ Uploading library to pypi..."

pip install --upgrade twine
uv pip install --upgrade twine

twine check dist/*
uv run twine check dist/*

twine upload \
uv run twine upload \
--username $TWINE_PROD_USERNAME \
--password $TWINE_PROD_PWD \
dist/*

echo "\n 🎉 Done."
echo -e "\n 🎉 Done."
12 changes: 6 additions & 6 deletions bin/_release_test
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
#!/bin/bash
set -e

echo "\n ⏱ Uploading library to testpypi..."
echo -e "\n ⏱ Uploading library to testpypi..."

pip install --upgrade twine
uv pip install --upgrade twine

twine check dist/*
uv run twine check dist/*

twine upload \
uv run twine upload \
--username $TWINE_TEST_USERNAME \
--password $TWINE_TEST_PWD \
--repository testpypi dist/*
--repository testpypi dist/* --verbose

echo "\n 🚢 Done."
echo -e "\n 🚢 Done."
42 changes: 19 additions & 23 deletions examples/Makefile-example
Original file line number Diff line number Diff line change
@@ -1,37 +1,33 @@
VIRTUALENV_NAME=modelstore.$(shell pwd | rev | cut -d '/' -f 1 | rev)
REPO_ROOT=$(shell cd ../../ && pwd)

.PHONY: name pyenv pyenv-local pyenv-prod pyenv-test pyenv-uninstall refresh gcloud
.PHONY: uninstall install install-local install-test install-prod refresh gcloud

name:
@echo $(VIRTUALENV_NAME)

pyenv-uninstall:
@$(REPO_ROOT)/bin/_pyenv_uninstall $(VIRTUALENV_NAME)
uninstall:
@if [ -d ".venv" ]; then rm -rf .venv; echo " ✅ Removed .venv"; else echo " ✅ Nothing to do."; fi

gcloud:
@gcloud components update
@gcloud auth application-default login

pyenv: pyenv-uninstall
# @$(REPO_ROOT)/bin/_setup_brew
@$(REPO_ROOT)/bin/_pyenv_install $(VIRTUALENV_NAME)
pip install -r https://raw.githubusercontent.com/ultralytics/yolov5/master/requirements.txt
install: uninstall
@uv venv
@uv pip install --upgrade pip setuptools wheel
@uv pip install -r requirements.txt

pyenv-local: pyenv
pip uninstall -y modelstore
pip install -e $(REPO_ROOT)
install-local: install
@uv pip uninstall modelstore
@uv pip install -e $(REPO_ROOT)

pyenv-test: pyenv
pip uninstall -y modelstore
pip install --no-cache-dir -i https://test.pypi.org/simple/ modelstore
install-test: install
@uv pip uninstall modelstore
@uv pip install --no-cache-dir --extra-index-url https://test.pypi.org/simple/ --index-strategy unsafe-best-match modelstore==0.0.82

pyenv-prod: pyenv
pip uninstall -y modelstore
pip install --no-cache-dir --upgrade modelstore
install-prod: install
@uv pip uninstall modelstore
@uv pip install --no-cache-dir --upgrade modelstore

refresh:
@echo "\n 🔵 Refreshing installation of modelstore"
pip install --upgrade pip setuptools wheel
pip uninstall -y modelstore
pip install --no-cache-dir -e $(REPO_ROOT)
@uv pip install --upgrade pip setuptools wheel
@uv pip uninstall modelstore
@uv pip install --no-cache-dir -e $(REPO_ROOT)
6 changes: 3 additions & 3 deletions examples/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@

This directory contains examples of training models and storing them into a model store over different types of storage.

The Python script in `examples-by-ml-model` iterates over all of the supported ML frameworks and all of the supported storage types. For each pair, it trains a model, uploads it to storage, and then downloads/loads it back.
The Python script in `examples-by-ml-model` iterates over all of the supported ML frameworks and all of the supported storage types. For each pair, it trains a model, uploads it to storage, and then downloads/loads it back.

The bash script `cli-examples` has exaples of how to run `python -m modelstore` commands.

## Pre-requisites

As with the main library, these scripts have been developed using [pyenv](https://github.com/pyenv/pyenv) and [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv).
As with the main library, these scripts have been developed using [uv](https://github.com/astral-sh/uv).

## Set up - examples by ML model

Expand All @@ -24,7 +24,7 @@ And then you can use this `Makefile` command that creates a new virtual environm
and installs all of the requirements:

```bash
❯ make pyenv
❯ make install
```

## Running all of the examples
Expand Down
2 changes: 1 addition & 1 deletion examples/cli-examples/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def train_and_save():
"max_depth": 4,
"min_samples_split": 5,
"learning_rate": 0.01,
"loss": "ls",
"loss": "squared_error",
}
model = GradientBoostingRegressor(**params)
model.fit(X_train, y_train)
Expand Down
6 changes: 3 additions & 3 deletions examples/cli-examples/run-all.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@ TARGET_DIR="downloaded_model"
FILE_NAME="model.joblib"

echo "\n🔵 Training a model...\n"
python model.py
uv run python model.py

echo "\n🔵 Uploading the model via the CLI...\n"
MODEL_ID=$(python -m modelstore upload "$DOMAIN_NAME" "$FILE_NAME")
MODEL_ID=$(uv run python -m modelstore upload "$DOMAIN_NAME" "$FILE_NAME")

echo "\n🔵 Downloading model=$MODEL_ID via the CLI...\n"
mkdir -p "$TARGET_DIR"
python -m modelstore download "$DOMAIN_NAME" "$MODEL_ID" "$TARGET_DIR"
uv run python -m modelstore download "$DOMAIN_NAME" "$MODEL_ID" "$TARGET_DIR"

echo "\n✅ Done! Cleaning up..."

Expand Down
26 changes: 26 additions & 0 deletions examples/examples-by-ml-library/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
FROM python:3.11-slim
WORKDIR /usr/src/app

ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
apt-get install -y build-essential git ninja-build ccache libopenblas-dev libopencv-dev cmake && \
apt-get install -y gcc mono-mcs g++ && \
apt-get install -y default-jdk && \
apt-get install -y libhdf5-dev && \
rm -rf /var/lib/apt/lists/*

# Install modelstore library from test.pypi.org
RUN pip install --upgrade pip setuptools wheel && \
pip install --index-url https://test.pypi.org/simple/ \
--extra-index-url https://pypi.org/simple/ \
modelstore

# Install example dependencies
COPY requirements.txt ./requirements.txt
RUN pip install -r requirements.txt

# Copy example source
COPY . ./examples
WORKDIR /usr/src/app/examples

ENTRYPOINT ["python", "main.py"]
18 changes: 16 additions & 2 deletions examples/examples-by-ml-library/Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,20 @@
include ../Makefile-example

.PHONY: run
DOCKER_IMAGE ?= modelstore-examples
MODELSTORE_IN ?= filesystem
ML_FRAMEWORK ?= sklearn

.PHONY: run docker-build docker-run

run:
@./run-all.sh


docker-build:
@docker build -f Dockerfile -t $(DOCKER_IMAGE) .

docker-run:
@docker run --rm \
-e MODEL_STORE_ROOT_PREFIX=/tmp/modelstore \
$(DOCKER_IMAGE) \
--modelstore-in $(MODELSTORE_IN) \
--ml-framework $(ML_FRAMEWORK)
8 changes: 7 additions & 1 deletion examples/examples-by-ml-library/libraries/skorch_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,13 @@ def train_and_upload(modelstore: ModelStore) -> dict:
def load_and_test(modelstore: ModelStore, model_domain: str, model_id: str):
# Load the model back into memory!
print(f'⤵️ Loading the skorch "{model_domain}" domain model={model_id}')
model = modelstore.load(model_domain, model_id)
loaded = modelstore.load(model_domain, model_id)

# When multiple files are saved, modelstore returns a dict
if isinstance(loaded, dict):
model = loaded["skorch"]
else:
model = loaded

# Run some example predictions
_, X_test, _, y_test = load_regression_dataset(as_numpy=True)
Expand Down
91 changes: 30 additions & 61 deletions examples/examples-by-ml-library/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,72 +11,41 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import importlib
import sys

import click
from libraries import (
annoy_example,
catboost_example,
causalml_example,
fastai_example,
gensim_example,
keras_example,
lightgbm_example,
onnx_lightgbm_example,
onnx_sklearn_example,
prophet_example,
pyspark_example,
pytorch_example,
pytorch_lightning_example,
raw_file_example,
shap_example,
sklearn_example,
sklearn_with_explainer_example,
sklearn_with_extras_example,
skorch_example,
tensorflow_example,
xgboost_booster_example,
xgboost_example,
yolo_example
)
from libraries.huggingface import (
distilbert,
dpt,
gpt2_pytorch,
gpt2_tensorflow,
sam
)
from modelstores import MODELSTORES, create_model_store

EXAMPLES = {
"annoy": annoy_example,
"catboost": catboost_example,
"causalml": causalml_example,
"dpt": dpt,
"fastai": fastai_example,
"file": raw_file_example,
"gensim": gensim_example,
"hf-distilbert": distilbert,
"hf-gpt2-pt": gpt2_pytorch,
"hf-gpt2-tf": gpt2_tensorflow,
"keras": keras_example,
"lightgbm": lightgbm_example,
"onnx-sklearn": onnx_sklearn_example,
"onnx-lightgbm": onnx_lightgbm_example,
"prophet": prophet_example,
"pyspark": pyspark_example,
"pytorch": pytorch_example,
"pytorch-lightning": pytorch_lightning_example,
"segment-anything": sam,
"shap": shap_example,
"sklearn": sklearn_example,
"sklearn-with-explainer": sklearn_with_explainer_example,
"sklearn-with-extras": sklearn_with_extras_example,
"skorch": skorch_example,
"tensorflow": tensorflow_example,
"xgboost": xgboost_example,
"xgboost-booster": xgboost_booster_example,
"yolov5": yolo_example,
"annoy": "libraries.annoy_example",
"catboost": "libraries.catboost_example",
"causalml": "libraries.causalml_example",
"dpt": "libraries.huggingface.dpt",
"fastai": "libraries.fastai_example",
"file": "libraries.raw_file_example",
"gensim": "libraries.gensim_example",
"hf-distilbert": "libraries.huggingface.distilbert",
"hf-gpt2-pt": "libraries.huggingface.gpt2_pytorch",
"hf-gpt2-tf": "libraries.huggingface.gpt2_tensorflow",
"keras": "libraries.keras_example",
"lightgbm": "libraries.lightgbm_example",
"onnx-sklearn": "libraries.onnx_sklearn_example",
"onnx-lightgbm": "libraries.onnx_lightgbm_example",
"prophet": "libraries.prophet_example",
"pyspark": "libraries.pyspark_example",
"pytorch": "libraries.pytorch_example",
"pytorch-lightning": "libraries.pytorch_lightning_example",
"segment-anything": "libraries.huggingface.sam",
"shap": "libraries.shap_example",
"sklearn": "libraries.sklearn_example",
"sklearn-with-explainer": "libraries.sklearn_with_explainer_example",
"sklearn-with-extras": "libraries.sklearn_with_extras_example",
"skorch": "libraries.skorch_example",
"tensorflow": "libraries.tensorflow_example",
"xgboost": "libraries.xgboost_example",
"xgboost-booster": "libraries.xgboost_booster_example",
"yolov5": "libraries.yolo_example",
}


Expand Down Expand Up @@ -109,7 +78,7 @@ def main(modelstore_in, ml_framework):

# Create a model store instance
modelstore = create_model_store(modelstore_in)
example = EXAMPLES[ml_framework]
example = importlib.import_module(EXAMPLES[ml_framework])

# Demo how we train and upload a model
meta_data = example.train_and_upload(modelstore)
Expand Down
Loading
Loading