Skip to content

Commit 8185ea2

Browse files
Support using separate Pulsar cluster as package service (#829)
* Support using separate Pulsar cluster as package service * Fix ci * Fix ci * Fix ci * Fix ci * Fix vulnerabilities
1 parent a76127b commit 8185ea2

45 files changed

Lines changed: 2400 additions & 136 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
124 Bytes
Binary file not shown.
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
apiVersion: compute.functionmesh.io/v1alpha1
2+
kind: Function
3+
metadata:
4+
name: function-download-sample-package-service
5+
namespace: default
6+
spec:
7+
image: streamnative/pulsar-functions-pulsarctl-java-runner:3.2.2.1
8+
className: org.apache.pulsar.functions.api.examples.ExclamationFunction
9+
forwardSourceMessageProperty: true
10+
maxPendingAsyncRequests: 1000
11+
replicas: 1
12+
logTopic: persistent://public/default/logging-function-logs-package-service
13+
logTopicAgent: sidecar
14+
input:
15+
topics:
16+
- persistent://public/default/input-download-java-package-service-topic
17+
typeClassName: java.lang.String
18+
output:
19+
topic: persistent://public/default/output-download-java-package-service-topic
20+
typeClassName: java.lang.String
21+
resources:
22+
requests:
23+
cpu: 50m
24+
memory: 1G
25+
limits:
26+
cpu: "0.2"
27+
memory: 1.1G
28+
secretsMap:
29+
"name":
30+
path: "test-secret"
31+
key: "username"
32+
"pwd":
33+
path: "test-secret"
34+
key: "password"
35+
pulsar:
36+
pulsarConfig: "test-pulsar-runtime"
37+
tlsConfig:
38+
enabled: false
39+
allowInsecure: false
40+
hostnameVerification: true
41+
certSecretName: sn-platform-tls-broker
42+
certSecretKey: ""
43+
authConfig:
44+
oauth2Config:
45+
audience: urn:sn:pulsar:sndev:test
46+
issuerUrl: https://auth.sncloud-stg.dev/
47+
keySecretName: sn-platform-oauth2-private-key
48+
keySecretKey: auth.json
49+
packageService:
50+
pulsarConfig: "test-pulsar-package-service"
51+
tlsConfig:
52+
enabled: false
53+
allowInsecure: false
54+
hostnameVerification: true
55+
certSecretName: sn-platform-tls-broker
56+
certSecretKey: ""
57+
authConfig:
58+
oauth2Config:
59+
audience: urn:sn:pulsar:sndev:test
60+
issuerUrl: https://auth.sncloud-stg.dev/
61+
keySecretName: sn-platform-oauth2-private-key
62+
keySecretKey: auth.json
63+
java:
64+
jar: pulsar-functions-api-examples.jar
65+
jarLocation: function://public/default/test-java-function
66+
clusterName: test
67+
autoAck: true
68+
---
69+
apiVersion: v1
70+
kind: ConfigMap
71+
metadata:
72+
name: test-pulsar-runtime
73+
data:
74+
webServiceURL: http://invalid-runtime-admin.default.svc.cluster.local:8080
75+
brokerServiceURL: pulsar://sn-platform-pulsar-broker.default.svc.cluster.local:6650
76+
---
77+
apiVersion: v1
78+
kind: ConfigMap
79+
metadata:
80+
name: test-pulsar-package-service
81+
data:
82+
webServiceURL: http://sn-platform-pulsar-broker.default.svc.cluster.local:8080
83+
brokerServiceURL: pulsar://sn-platform-pulsar-broker.default.svc.cluster.local:6650
84+
---
85+
apiVersion: v1
86+
data:
87+
username: YWRtaW4=
88+
password: MWYyZDFlMmU2N2Rm
89+
kind: Secret
90+
metadata:
91+
name: test-secret
92+
type: Opaque
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
#
20+
21+
set -e
22+
23+
E2E_DIR=$(dirname "$0")
24+
BASE_DIR=$(cd "${E2E_DIR}"/../../../../..;pwd)
25+
PULSAR_NAMESPACE=${PULSAR_NAMESPACE:-"default"}
26+
PULSAR_RELEASE_NAME=${PULSAR_RELEASE_NAME:-"sn-platform"}
27+
E2E_KUBECONFIG=${E2E_KUBECONFIG:-"/tmp/e2e-k8s.config"}
28+
MANIFESTS_FILE="${BASE_DIR}"/.ci/tests/integration-oauth2/cases/java-download-function-with-package-service/manifests.yaml
29+
30+
source "${BASE_DIR}"/.ci/helm.sh
31+
32+
FUNCTION_NAME=function-download-sample-package-service
33+
STS_NAME=${FUNCTION_NAME}-function
34+
35+
if [ ! "$KUBECONFIG" ]; then
36+
export KUBECONFIG=${E2E_KUBECONFIG}
37+
fi
38+
39+
kubectl apply -f "${MANIFESTS_FILE}" > /dev/null 2>&1
40+
41+
verify_fm_result=$(ci::verify_function_mesh ${FUNCTION_NAME} 2>&1)
42+
if [ $? -ne 0 ]; then
43+
echo "$verify_fm_result"
44+
kubectl delete -f "${MANIFESTS_FILE}" > /dev/null 2>&1 || true
45+
exit 1
46+
fi
47+
48+
sts_yaml=$(kubectl get statefulset "${STS_NAME}" -o yaml 2>&1)
49+
echo "${sts_yaml}" | grep "prefix: PACKAGE_" > /dev/null 2>&1 || {
50+
echo "packageService env prefix is not injected"
51+
kubectl delete -f "${MANIFESTS_FILE}" > /dev/null 2>&1 || true
52+
exit 1
53+
}
54+
echo "${sts_yaml}" | grep "name: test-pulsar-package-service" > /dev/null 2>&1 || {
55+
echo "packageService configmap is not injected"
56+
kubectl delete -f "${MANIFESTS_FILE}" > /dev/null 2>&1 || true
57+
exit 1
58+
}
59+
echo "${sts_yaml}" | grep "/etc/oauth2-package-service" > /dev/null 2>&1 || {
60+
echo "packageService oauth2 volume mount is not injected"
61+
kubectl delete -f "${MANIFESTS_FILE}" > /dev/null 2>&1 || true
62+
exit 1
63+
}
64+
65+
verify_java_result=$(NAMESPACE=${PULSAR_NAMESPACE} CLUSTER=${PULSAR_RELEASE_NAME} ci::verify_exclamation_function_with_auth \
66+
persistent://public/default/input-download-java-package-service-topic \
67+
persistent://public/default/output-download-java-package-service-topic \
68+
test-message test-message! 10 2>&1)
69+
if [ $? -ne 0 ]; then
70+
echo "$verify_java_result"
71+
kubectl delete -f "${MANIFESTS_FILE}" > /dev/null 2>&1 || true
72+
exit 1
73+
fi
74+
75+
verify_log_topic=$(ci::verify_log_topic_with_auth persistent://public/default/logging-function-logs-package-service "it is not a NAR file" 10 2>&1)
76+
if [ $? -ne 0 ]; then
77+
echo "$verify_log_topic"
78+
kubectl delete -f "${MANIFESTS_FILE}" > /dev/null 2>&1 || true
79+
exit 1
80+
fi
81+
82+
kubectl delete -f "${MANIFESTS_FILE}" > /dev/null 2>&1 || true
83+
echo "e2e-test: ok" | yq eval -

.ci/tests/integration-oauth2/e2e.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,8 @@ verify:
131131
expected: expected.data.yaml
132132
- query: timeout 5m bash .ci/tests/integration-oauth2/cases/java-download-function/verify.sh
133133
expected: expected.data.yaml
134+
- query: timeout 5m bash .ci/tests/integration-oauth2/cases/java-download-function-with-package-service/verify.sh
135+
expected: expected.data.yaml
134136
- query: timeout 5m bash .ci/tests/integration-oauth2/cases/java-download-function-generic-auth/verify.sh
135137
expected: expected.data.yaml
136138
- query: timeout 5m bash .ci/tests/integration-oauth2/cases/py-download-function/verify.sh

.ci/tests/integration-oauth2/e2e_with_downloader.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,8 @@ verify:
125125
expected: expected.data.yaml
126126
- query: timeout 5m bash .ci/tests/integration-oauth2/cases/java-download-function/verify.sh
127127
expected: expected.data.yaml
128+
- query: timeout 5m bash .ci/tests/integration-oauth2/cases/java-download-function-with-package-service/verify.sh
129+
expected: expected.data.yaml
128130
- query: timeout 5m bash .ci/tests/integration-oauth2/cases/java-download-function-generic-auth/verify.sh
129131
expected: expected.data.yaml
130132
- query: timeout 5m bash .ci/tests/integration-oauth2/cases/py-download-function/verify.sh

.github/workflows/bundle-release.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,10 @@ jobs:
4949
username: ${{ secrets.DOCKER_USER }}
5050
password: ${{ secrets.DOCKER_PASSWORD }}
5151

52-
- name: Set up GO 1.24.11
52+
- name: Set up GO 1.24.13
5353
uses: actions/setup-go@v5
5454
with:
55-
go-version: 1.24.11
55+
go-version: 1.24.13
5656
id: go
5757

5858
- name: InstallKubebuilder
@@ -180,10 +180,10 @@ jobs:
180180
username: ${{ secrets.DOCKER_USER }}
181181
password: ${{ secrets.DOCKER_PASSWORD }}
182182

183-
- name: Set up GO 1.24.11
183+
- name: Set up GO 1.24.13
184184
uses: actions/setup-go@v5
185185
with:
186-
go-version: 1.24.11
186+
go-version: 1.24.13
187187
id: go
188188

189189
- name: InstallKubebuilder

.github/workflows/olm-verify.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,10 @@ jobs:
3434
- name: checkout
3535
uses: actions/checkout@v2
3636

37-
- name: Set up GO 1.24.11
37+
- name: Set up GO 1.24.13
3838
uses: actions/setup-go@v5
3939
with:
40-
go-version: 1.24.11
40+
go-version: 1.24.13
4141
id: go
4242

4343
- name: InstallKubebuilder

.github/workflows/project.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
strategy:
1919
fail-fast: false
2020
matrix:
21-
go-version: [1.22.12, 1.24.11]
21+
go-version: [1.22.12, 1.24.13]
2222
steps:
2323
- name: Free Disk Space (Ubuntu)
2424
uses: jlumbroso/free-disk-space@v1.3.0

.github/workflows/release.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,10 @@ jobs:
3737
username: ${{ secrets.DOCKER_USER }}
3838
password: ${{ secrets.DOCKER_PASSWORD }}
3939

40-
- name: Set up GO 1.24.11
40+
- name: Set up GO 1.24.13
4141
uses: actions/setup-go@v5
4242
with:
43-
go-version: 1.24.11
43+
go-version: 1.24.13
4444
id: go
4545

4646
- name: InstallKubebuilder

.github/workflows/test-helm-charts.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,11 @@ jobs:
8383
run: hack/kind-cluster-build.sh --name chart-testing -c 1 -v 10 --k8sVersion v1.23.17
8484
if: steps.list-changed.outputs.changed == 'true'
8585

86-
- name: Set up GO 1.24.11
86+
- name: Set up GO 1.24.13
8787
if: steps.list-changed.outputs.changed == 'true'
8888
uses: actions/setup-go@v5
8989
with:
90-
go-version: 1.24.11
90+
go-version: 1.24.13
9191
id: go
9292

9393
- name: setup kubebuilder 3.6.0

0 commit comments

Comments
 (0)