Add azure support for storing MLRun artifacts

charts/mlrun-ce/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: mlrun-ce
-version: 0.11.0-rc.27
+version: 0.11.0-rc.28
 description: MLRun Open Source Stack
 home: https://iguazio.com
 icon: https://www.iguazio.com/wp-content/uploads/2019/10/Iguazio-Logo.png

charts/mlrun-ce/templates/NOTES.txt

@@ -81,7 +81,7 @@ SeaweedFS Admin UI is available at:
 {{ .Values.global.externalHostAddress }}:{{ .Values.seaweedfs.adminService.nodePort | default 30093 }}
 {{- end }}
 {{- end }}
--  S3 credentials: {{ .Values.s3.accessKey }} / {{ .Values.s3.secretKey }}
+-  S3 credentials: {{ .Values.storage.s3.accessKey }} / {{ .Values.storage.s3.secretKey }}
 {{- end }}
 {{- end }}
 

charts/mlrun-ce/templates/_helpers.tpl

@@ -151,24 +151,24 @@ S3 Service Port - returns the port for pipeline config
 {{- end -}}
 
 {{/*
-S3 Access Key - uses top-level s3.accessKey for all components (MLRun, Jupyter, Pipelines)
+S3 Access Key - uses top-level s3.storage.accessKey for all components (MLRun, Jupyter, Pipelines)
 */}}
 {{- define "mlrun-ce.s3.accessKey" -}}
-{{- .Values.s3.accessKey -}}
+{{- .Values.storage.s3.accessKey -}}
 {{- end -}}
 
 {{/*
-S3 Secret Key - uses top-level s3.secretKey for all components (MLRun, Jupyter, Pipelines)
+S3 Secret Key - uses top-level s3.storage.secretKey for all components (MLRun, Jupyter, Pipelines)
 */}}
 {{- define "mlrun-ce.s3.secretKey" -}}
-{{- .Values.s3.secretKey -}}
+{{- .Values.storage.s3.secretKey -}}
 {{- end -}}
 
 {{/*
-S3 Bucket - uses top-level s3.bucket for all components
+S3 Bucket - uses top-level s3.storage.bucket for all components
 */}}
 {{- define "mlrun-ce.s3.bucket" -}}
-{{- .Values.s3.bucket -}}
+{{- .Values.storage.s3.bucket -}}
 {{- end -}}
 
 {{/*
@@ -181,6 +181,71 @@ Legacy Minio Service URL - kept for backward compatibility
 {{ include "mlrun-ce.s3.service.host" . }}
 {{- end -}}
 
+{{/*
+=============================================================================
+Storage Path Helpers
+Handles both S3 and Azure Blob storage backends
+=============================================================================
+*/}}
+
+{{- define "mlrun-ce.httpdb.realPath" -}}
+{{- if eq .Values.storage.mode "azure-blob" -}}
+az://
+{{- else -}}
+s3://
+{{- end -}}
+{{- end -}}
+
+{{- define "mlrun-ce.artifactPath" -}}
+{{- $bucket := .Values.global.infrastructure.aws.bucketName | default "mlrun" -}}
+{{- $container := .Values.storage.azure.containerName | default "" -}}
+{{- if eq .Values.storage.mode "azure-blob" -}}
+az://{{ $container }}/projects/{{ `{{run.project}}` }}/artifacts
+{{- else -}}
+s3://{{ $bucket }}/projects/{{ `{{run.project}}` }}/artifacts
+{{- end -}}
+{{- end -}}
+
+{{- define "mlrun-ce.featureStore.dataPrefix" -}}
+{{- $bucket := .Values.global.infrastructure.aws.bucketName | default "mlrun" -}}
+{{- $container := .Values.storage.azure.containerName | default "" -}}
+{{- if eq .Values.storage.mode "azure-blob" -}}
+az://{{ $container }}/projects/{project}/FeatureStore/{name}/{kind}
+{{- else -}}
+s3://{{ $bucket }}/projects/{project}/FeatureStore/{name}/{kind}
+{{- end -}}
+{{- end -}}
+
+{{- define "mlrun-ce.model-endpoint.monitoring.userSpace" -}}
+{{- $bucket := .Values.global.infrastructure.aws.bucketName | default "mlrun" -}}
+{{- $container := .Values.storage.azure.containerName | default "" -}}
+{{- if eq .Values.storage.mode "azure-blob" -}}
+az://{{ $container }}/projects/{{ `{{project}}` }}/model-endpoints/{{ `{{kind}}` }}
+{{- else -}}
+s3://{{ $bucket }}/projects/{{ `{{project}}` }}/model-endpoints/{{ `{{kind}}` }}
+{{- end -}}
+{{- end -}}
+
+{{- define "mlrun-ce.model-endpoint.monitoring.application" -}}
+{{- $bucket := .Values.global.infrastructure.aws.bucketName | default "mlrun" -}}
+{{- $container := .Values.storage.azure.containerName | default "" -}}
+{{- if eq .Values.storage.mode "azure-blob" -}}
+az://{{ $container }}/users/pipelines/{{ `{{project}}` }}/monitoring-apps/
+{{- else -}}
+s3://{{ $bucket }}/users/pipelines/{{ `{{project}}` }}/monitoring-apps/
+{{- end -}}
+{{- end -}}
+
+{{- define "mlrun-ce.model-endpoint.monitoring.default" -}}
+{{- $bucket := .Values.global.infrastructure.aws.bucketName | default "mlrun" -}}
+{{- $container := .Values.storage.azure.containerName | default "" -}}
+{{- if eq .Values.storage.mode "azure-blob" -}}
+az://{{ $container }}/projects/{{ `{{project}}` }}/model-endpoints/{{ `{{kind}}` }}
+{{- else -}}
+s3://{{ $bucket }}/projects/{{ `{{project}}` }}/model-endpoints/{{ `{{kind}}` }}
+{{- end -}}
+{{- end -}}
+
 {{/*
 MLRun storage auto mount params
 Global toggle is for fast toggling between on-prem/standalone and s3 cases
@@ -191,7 +256,7 @@ Uses SeaweedFS as the storage backend
   {{- if .Values.mlrun.storageAutoMountParams -}}
     {{ .Values.mlrun.storageAutoMountParams }}
   {{- else if not .Values.global.infrastructure.aws.s3NonAnonymous -}}
-    "secret_name=s3-credentials"
+    "secret_name=storage-credentials"
   {{- else -}}
     "non_anonymous=True"
   {{- end -}}

charts/mlrun-ce/templates/config/jupyter-env-configmap.yaml

@@ -1,20 +1,18 @@
 {{ if .Values.jupyterNotebook.enabled}}
-{{- $bucket_name := .Values.global.infrastructure.aws.bucketName | default "mlrun" }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: jupyter-common-env
 data:
-  # S3 credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_ENDPOINT_URL_S3)
-  # are loaded from the 's3-credentials' Secret
+  # Storage credentials are loaded from the 'storage-credentials' Secret
+  # (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_ENDPOINT_URL_S3 for s3, or AZURE_STORAGE_* for azure-blob)
   MLRUN_STORAGE__AUTO_MOUNT_TYPE: {{ .Values.mlrun.storageAutoMountType | default "secret_env" }}
   S3_NON_ANONYMOUS: {{ .Values.global.infrastructure.aws.s3NonAnonymous  | toString | title | quote | default "\"True\"" }}
   MLRUN_CE__MODE: {{ .Values.jupyterNotebook.ce.mode | default "full" }}
   MLRUN_CE__VERSION: {{ .Chart.Version }}
   MLRUN_FUNCTION__SPEC__SERVICE_ACCOUNT__DEFAULT: {{ .Values.mlrun.api.functionSpecServiceAccountDefault | default "" | quote }}
-  MLRUN_FEATURE_STORE__DATA_PREFIXES__DEFAULT: s3:///{{ $bucket_name }}/projects/{project}/FeatureStore/{name}/{kind}
+  MLRUN_HTTPDB__REAL_PATH: {{ include "mlrun-ce.httpdb.realPath" . | quote }}
   MLRUN_STORAGE__AUTO_MOUNT_PARAMS: {{ include "mlrun.storage.auto.mount.params" . }}
   MLRUN_FEATURE_STORE__DATA_PREFIXES__NOSQL: ""
   MLRUN_FEATURE_STORE__DEFAULT_TARGETS: parquet
-  MLRUN_HTTPDB__REAL_PATH: s3://
 {{- end}}

charts/mlrun-ce/templates/config/mlrun-env-configmap.yaml

@@ -1,6 +1,4 @@
 {{ if .Values.mlrun.enabled}}
-{{- $bucket_name := .Values.global.infrastructure.aws.bucketName | default "mlrun" }}
-
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -11,20 +9,20 @@ data:
   MLRUN_STORAGE__AUTO_MOUNT_PARAMS: {{ include "mlrun.storage.auto.mount.params" . }}
   MLRUN_HTTPDB__PROJECTS__LEADER: mlrun
   MLRUN_HTTPDB__PROJECTS__FOLLOWERS: nuclio
-  # S3 credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_ENDPOINT_URL_S3)
-  # are loaded from the 's3-credentials' Secret
+  # Storage credentials are loaded from the 'storage-credentials' Secret
+  # (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_ENDPOINT_URL_S3 for s3, or AZURE_STORAGE_* for azure-blob)
   MLRUN_FUNCTION__SPEC__SERVICE_ACCOUNT__DEFAULT: {{ .Values.mlrun.api.functionSpecServiceAccountDefault | default "" | quote }}
-  MLRUN_HTTPDB__REAL_PATH: s3://
-  MLRUN_ARTIFACT_PATH: s3://{{ $bucket_name }}/projects/{{ `{{run.project}}` }}/artifacts
-  MLRUN_FEATURE_STORE__DATA_PREFIXES__DEFAULT: s3://{{ $bucket_name }}/projects/{project}/FeatureStore/{name}/{kind}
-  MLRUN_MODEL_ENDPOINT_MONITORING__STORE_PREFIXES__USER_SPACE: s3://{{ $bucket_name }}/projects/{{ `{{project}}` }}/model-endpoints/{{ `{{kind}}` }}
-  MLRUN_MODEL_ENDPOINT_MONITORING__STORE_PREFIXES__MONITORING_APPLICATION: s3://{{ $bucket_name }}/users/pipelines/{{ `{{project}}` }}/monitoring-apps/
+  MLRUN_HTTPDB__REAL_PATH: {{ include "mlrun-ce.httpdb.realPath" . | quote }}
+  MLRUN_ARTIFACT_PATH: {{ include "mlrun-ce.artifactPath" . | quote }}
+  MLRUN_FEATURE_STORE__DATA_PREFIXES__DEFAULT: {{ include "mlrun-ce.featureStore.dataPrefix" . | quote }}
+  MLRUN_MODEL_ENDPOINT_MONITORING__STORE_PREFIXES__USER_SPACE: {{ include "mlrun-ce.model-endpoint.monitoring.userSpace" . | quote }}
+  MLRUN_MODEL_ENDPOINT_MONITORING__STORE_PREFIXES__MONITORING_APPLICATION: {{ include "mlrun-ce.model-endpoint.monitoring.application" . | quote }}
+  MLRUN_MODEL_ENDPOINT_MONITORING__STORE_PREFIXES__DEFAULT: {{ include "mlrun-ce.model-endpoint.monitoring.default" . | quote }}
   MLRUN_FEATURE_STORE__DATA_PREFIXES__NOSQL: ""
-  MLRUN_CE__MODE: {{ .Values.mlrun.ce.mode }}
-  MLRUN_CE__VERSION: {{ .Chart.Version }}
+  MLRUN_CE__MODE: {{ .Values.mlrun.ce.mode | quote }}
+  MLRUN_CE__VERSION: {{ .Chart.Version | quote }}
   MLRUN_DEFAULT_TENSORBOARD_LOGS_PATH: /home/jovyan/data/tensorboard/{{ `{{project}} `}}
   MLRUN_FEATURE_STORE__DEFAULT_TARGETS: parquet
-  MLRUN_MODEL_ENDPOINT_MONITORING__STORE_PREFIXES__DEFAULT: s3://{{ $bucket_name }}/projects/{{ `{{project}}` }}/model-endpoints/{{ `{{kind}}` }}
   MLRUN_GRAFANA_URL: http://{{ .Values.global.externalHostAddress }}:{{ index .Values "kube-prometheus-stack" "grafana" "service" "nodePort" }}
   MLRUN_DEFAULT_FUNCTION_POD_RESOURCES__LIMITS__CPU: "{{ .Values.mlrun.defaultFunctionPodResources.limits.cpu | default "" }}"
   MLRUN_DEFAULT_FUNCTION_POD_RESOURCES__LIMITS__MEMORY: "{{ .Values.mlrun.defaultFunctionPodResources.limits.memory | default "" }}"

charts/mlrun-ce/templates/pipelines/configmaps/ml-pipeline-ui-configmap.yaml

@@ -11,7 +11,7 @@ data:
                 "name": "AWS_ACCESS_KEY_ID",
                 "valueFrom": {
                   "secretKeyRef": {
-                    "name": "mlpipeline-minio-artifact",
+                    "name": "mlpipeline-seaweedfs-artifact",
                     "key": "accesskey"
                   }
                 }
@@ -20,7 +20,7 @@ data:
                 "name": "AWS_SECRET_ACCESS_KEY",
                 "valueFrom": {
                   "secretKeyRef": {
-                    "name": "mlpipeline-minio-artifact",
+                    "name": "mlpipeline-seaweedfs-artifact",
                     "key": "secretkey"
                   }
                 }

charts/mlrun-ce/templates/pipelines/configmaps/workflow-controller-configmap.yaml

@@ -13,10 +13,10 @@ data:
       bucket: "{{ include "mlrun-ce.s3.bucket" . }}"
       insecure: true
       accessKeySecret:
-        name: mlpipeline-minio-artifact
+        name: mlpipeline-seaweedfs-artifact
         key: accesskey
       secretKeySecret:
-        name: mlpipeline-minio-artifact
+        name: mlpipeline-seaweedfs-artifact
         key: secretkey
   executor: |
     imagePullPolicy: IfNotPresent

charts/mlrun-ce/templates/pipelines/deployments/ml-pipeline-ui.yaml

@@ -43,12 +43,12 @@ spec:
           valueFrom:
             secretKeyRef:
               key: accesskey
-              name: mlpipeline-minio-artifact
+              name: mlpipeline-seaweedfs-artifact
         - name: MINIO_SECRET_KEY
           valueFrom:
             secretKeyRef:
               key: secretkey
-              name: mlpipeline-minio-artifact
+              name: mlpipeline-seaweedfs-artifact
         - name: ALLOW_CUSTOM_VISUALIZATIONS
           value: "true"
         image: {{ .Values.pipelines.images.ui.repository }}:{{ .Values.pipelines.images.ui.tag }}

charts/mlrun-ce/templates/pipelines/deployments/ml-pipeline.yaml

@@ -57,12 +57,12 @@ spec:
             - name: OBJECTSTORECONFIG_ACCESSKEY
               valueFrom:
                 secretKeyRef:
-                  name: mlpipeline-minio-artifact
+                  name: mlpipeline-seaweedfs-artifact
                   key: accesskey
             - name: OBJECTSTORECONFIG_SECRETACCESSKEY
               valueFrom:
                 secretKeyRef:
-                  name: mlpipeline-minio-artifact
+                  name: mlpipeline-seaweedfs-artifact
                   key: secretkey
             - name: OBJECTSTORECONFIG_HOST
               valueFrom:

charts/mlrun-ce/templates/pipelines/secrets/mlpipeline-minio-artifact.yaml → charts/mlrun-ce/templates/pipelines/secrets/mlpipeline-seaweedfs-artifact.yaml

@@ -8,6 +8,6 @@ metadata:
   annotations:
   labels:
     application-crd-id: kubeflow-pipelines
-  name: mlpipeline-minio-artifact
+  name: mlpipeline-seaweedfs-artifact
 type: Opaque
 {{- end -}}

charts/mlrun-ce/values.yaml

@@ -23,12 +23,27 @@ global:
 # =============================================================================
 # S3-compatible storage configuration
 # These credentials are used by MLRun, Jupyter, and Kubeflow Pipelines
-# to access the S3-compatible storage backend (SeaweedFS)
+# to access the storage backend.
 # =============================================================================
-s3:
-  accessKey: "seaweed"
-  secretKey: "seaweed123"
-  bucket: "mlrun"
+
+# storage.mode selects which backend credentials are injected into the 'storage-credentials' Secret.
+# Options:
+#   s3 (default) - uses storage.s3.accessKey/secretKey/bucket with SeaweedFS endpoint
+#   azure-blob   - uses storage.azure.* fields
+storage:
+  mode: s3
+  s3:
+    accessKey: "seaweed"
+    secretKey: "seaweed123"
+    bucket: "mlrun"
+  azure:
+    containerName: ""
+    connectionString: ""
+    accountName: ""
+    accountKey: ""
+    sasToken: ""
+    clientSecret: ""
+    tenantId: ""
 
 nuclio:
   global:
@@ -146,7 +161,7 @@ mlrun:
       - configMapRef:
           name: mlrun-common-env
       - secretRef:
-          name: s3-credentials
+          name: storage-credentials
           optional: true
       - configMapRef:
           name: mlrun-pipelines-config
@@ -261,7 +276,7 @@ jupyterNotebook:
         name: jupyter-common-env
         optional: true
     - secretRef:
-        name: s3-credentials
+        name: storage-credentials
         optional: true
   persistence:
     enabled: true