Skip to content

Commit 589e665

Browse files
drgrootdrgroot
committed
feat: initial working cut of the provisioner
for ticket #1
1 parent 0b44508 commit 589e665

15 files changed

Lines changed: 446 additions & 692 deletions

‎.env‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
ACCESS_KEY_ID=user
2+
ACCESS_SECRET_ACCESS_KEY=password
3+
DEFAULT_BUCKET=servc

‎.github/workflows/test.yml‎

Lines changed: 80 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@ name: 'Unit Tests'
22
on:
33
- push
44

5+
env:
6+
ACCESS_KEY_ID: user
7+
ACCESS_SECRET_ACCESS_KEY: password
8+
DEFAULT_BUCKET: mylake
9+
510
jobs:
611
unittest:
712
runs-on: ubuntu-latest
@@ -14,23 +19,62 @@ jobs:
1419
- 3.13
1520

1621
services:
17-
rabbitmq:
18-
image: rabbitmq
19-
env:
20-
RABBITMQ_DEFAULT_USER: guest
21-
RABBITMQ_DEFAULT_PASS: guest
22-
ports:
23-
- 5672:5672
24-
25-
redis:
26-
image: redis
22+
catalog:
23+
image: tabulario/iceberg-rest
2724
options: >-
28-
--health-cmd "redis-cli ping"
25+
--health-cmd "curl http://localhost:8181/health"
2926
--health-interval 10s
3027
--health-timeout 5s
3128
--health-retries 5
29+
# env:
30+
# AWS_ACCESS_KEY_ID: ${{ env.ACCESS_KEY_ID }}
31+
# AWS_SECRET_ACCESS_KEY: ${{ env.ACCESS_SECRET_ACCESS_KEY }}
32+
# AWS_REGION: us-east-1
33+
# WAREHOUSE: s3://${{ env.DEFAULT_BUCKET }}
34+
# CATALOG_WAREHOUSE: s3://${{ env.DEFAULT_BUCKET }}
35+
# CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO
36+
# CATALOG_S3_ENDPOINT: http://s3:9000
37+
# CATALOG_S3_PATH__STYLE__ACCESS: "true"
38+
# CATALOG_DEFAULT__NAMESPACE: default
39+
# CATALOG_JDBC_USER: user
40+
# CATALOG_JDBC_PASSWORD: pass
41+
# CATALOG_URI: jdbc:sqlite:file:/tmp/pyiceberg_catalog.db
3242
ports:
33-
- 6379:6379
43+
- 8181:8181
44+
45+
# s3:
46+
# image: bitnami/minio:latest
47+
# env:
48+
# MINIO_ROOT_USER: ${{ env.ACCESS_KEY_ID }}
49+
# MINIO_ROOT_PASSWORD: ${{ env.ACCESS_SECRET_ACCESS_KEY }}
50+
# MINIO_ACCESS_KEY: ${{ env.ACCESS_KEY_ID }}
51+
# MINIO_SECRET_KEY: ${{ env.ACCESS_SECRET_ACCESS_KEY }}
52+
# options: >-
53+
# --health-cmd "curl http://localhost:9000/minio/health/live"
54+
# --health-interval 10s
55+
# --health-timeout 5s
56+
# --health-retries 5
57+
# ports:
58+
# - 9000:9000
59+
# - 9001:9001
60+
61+
# rabbitmq:
62+
# image: rabbitmq
63+
# env:
64+
# RABBITMQ_DEFAULT_USER: guest
65+
# RABBITMQ_DEFAULT_PASS: guest
66+
# ports:
67+
# - 5672:5672
68+
69+
# redis:
70+
# image: redis
71+
# options: >-
72+
# --health-cmd "redis-cli ping"
73+
# --health-interval 10s
74+
# --health-timeout 5s
75+
# --health-retries 5
76+
# ports:
77+
# - 6379:6379
3478

3579
steps:
3680
- if: github.server_url != 'https://github.com'
@@ -59,13 +103,34 @@ jobs:
59103
- name: Type check
60104
run: mypy provisioner --check-untyped-defs
61105

106+
- name: Install minioclient
107+
env:
108+
MC_HOST_servc: http://${{ env.ACCESS_KEY_ID }}:${{ env.ACCESS_SECRET_ACCESS_KEY }}@${{ github.server_url != 'https://github.com' && 's3' || 'localhost' }}:9000
109+
run: |
110+
sudo apt-get update && sudo apt-get install -y wget
111+
112+
wget https://dl.min.io/client/mc/release/linux-amd64/mc
113+
chmod +x mc
114+
sudo mv mc /usr/local/bin
115+
116+
mc mb servc/${{ env.DEFAULT_BUCKET }}
117+
62118
- name: Run tests
63119
env:
120+
CONF__BLOBPROVISIONER__S3_ACCESS_KEY_ID: ${{ env.ACCESS_KEY_ID }}
121+
CONF__BLOBPROVISIONER__S3_SECRET_ACCESS_KEY: ${{ env.ACCESS_SECRET_ACCESS_KEY }}
122+
CONF__BLOBPROVISIONER__S3_ENDPOINT: http://${{ github.server_url != 'https://github.com' && 's3' || 'localhost' }}:9000
123+
CONF__BLOBPROVISIONER__S3_BUCKET: ${{ env.DEFAULT_BUCKET }}
124+
CONF__BLOBPROVISIONER__CATALOG_URI: http://${{ github.server_url != 'https://github.com' && 'catalog' || 'localhost' }}:8181
125+
CONF__LAKE__DATABASE: servc
64126
CONF__LAKE__CATALOG_NAME: default
65-
CONF__LAKE__CATALOG_PROPERTIES__TYPE: sql
66-
CONF__LAKE__CATALOG_PROPERTIES__URI: sqlite:////tmp/lake.db
127+
CONF__LAKE__CATALOG_PROPERTIES__TYPE: rest
128+
CONF__LAKE__CATALOG_PROPERTIES__URI: http://${{ github.server_url != 'https://github.com' && 'catalog' || 'localhost' }}:8181
67129
CONF__LAKE__CATALOG_PROPERTIES__INIT_CATALOG_TABLES: true
68-
CONF__LAKE__CATALOG_PROPERTIES__WAREHOUSE: file:///tmp/warehouse
130+
CONF__LAKE__CATALOG_PROPERTIES__WAREHOUSE: s3://${{ env.DEFAULT_BUCKET }}
131+
CONF__LAKE__CATALOG_PROPERTIES__S3__ACCESS_KEY_ID: ${{ env.ACCESS_KEY_ID }}
132+
CONF__LAKE__CATALOG_PROPERTIES__S3__SECRET_ACCESS_KEY: ${{ env.ACCESS_SECRET_ACCESS_KEY }}
133+
CONF__LAKE__CATALOG_PROPERTIES__S3__ENDPOINT: http://${{ github.server_url != 'https://github.com' && 's3' || 'localhost' }}:9000
69134
TEST_QUEUE: test
70135
CACHE_URL: redis://${{ github.server_url != 'https://github.com' && 'redis' || 'localhost' }}
71136
BUS_URL: amqp://guest:guest@${{ github.server_url != 'https://github.com' && 'rabbitmq' || 'localhost' }}

‎Dockerfile.dev‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
FROM minio/mc AS minio
2+
3+
FROM mcr.microsoft.com/vscode/devcontainers/python
4+
5+
COPY --from=minio /usr/bin/mc /usr/bin/mc

‎docker-compose.yml‎

Lines changed: 52 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,65 @@
1-
version: "3.8"
2-
31
services:
42
python:
5-
image: mcr.microsoft.com/vscode/devcontainers/python
3+
image: python-minio
4+
build:
5+
context: .
6+
dockerfile: Dockerfile.dev
67
volumes:
78
- .:/workspace:cached
89
- /Users/yusuf/Documents/bin:/custom_bin:ro
910
command: sleep infinity
1011
environment:
11-
CACHE_URL: redis://redis:6379/0
12-
BUS_URL: amqp://rabbitmq:rabbitmq@rabbitmq/
12+
CONF__BLOBPROVISIONER__S3_ACCESS_KEY_ID: ${ACCESS_KEY_ID}
13+
CONF__BLOBPROVISIONER__S3_SECRET_ACCESS_KEY: ${ACCESS_SECRET_ACCESS_KEY}
14+
CONF__BLOBPROVISIONER__S3_ENDPOINT: http://s3:9000
15+
CONF__BLOBPROVISIONER__S3_BUCKET: ${DEFAULT_BUCKET}
16+
CONF__BLOBPROVISIONER__CATALOG_URI: http://catalog:8181
17+
CONF__LAKE__DATABASE: servc
1318
CONF__LAKE__CATALOG_NAME: default
14-
CONF__LAKE__CATALOG_PROPERTIES__TYPE: sql
15-
CONF__LAKE__CATALOG_PROPERTIES__URI: sqlite:////tmp/lake.db
19+
CONF__LAKE__CATALOG_PROPERTIES__TYPE: rest
20+
CONF__LAKE__CATALOG_PROPERTIES__URI: http://catalog:8181
1621
CONF__LAKE__CATALOG_PROPERTIES__INIT_CATALOG_TABLES: true
17-
CONF__LAKE__CATALOG_PROPERTIES__WAREHOUSE: file:///tmp/warehouse
22+
CONF__LAKE__CATALOG_PROPERTIES__WAREHOUSE: s3://${DEFAULT_BUCKET}
23+
CONF__LAKE__CATALOG_PROPERTIES__S3__ACCESS_KEY_ID: ${ACCESS_KEY_ID}
24+
CONF__LAKE__CATALOG_PROPERTIES__S3__SECRET_ACCESS_KEY: ${ACCESS_SECRET_ACCESS_KEY}
25+
CONF__LAKE__CATALOG_PROPERTIES__S3__ENDPOINT: http://s3:9000
26+
links:
27+
- catalog
28+
- s3
1829

19-
redis:
20-
image: redis
21-
restart: unless-stopped
22-
expose:
23-
- "6379"
24-
ports:
25-
- "6379:6379"
30+
catalog:
31+
image: tabulario/iceberg-rest
32+
environment:
33+
AWS_ACCESS_KEY_ID: ${ACCESS_KEY_ID}
34+
AWS_SECRET_ACCESS_KEY: ${ACCESS_SECRET_ACCESS_KEY}
35+
AWS_REGION: us-east-1
36+
WAREHOUSE: s3://${DEFAULT_BUCKET}
37+
CATALOG_WAREHOUSE: s3://${DEFAULT_BUCKET}
38+
CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO
39+
CATALOG_S3_ENDPOINT: http://s3:9000
40+
CATALOG_S3_PATH__STYLE__ACCESS: true
41+
CATALOG_DEFAULT__NAMESPACE: default
42+
CATALOG_JDBC_USER: user
43+
CATALOG_JDBC_PASSWORD: pass
44+
CATALOG_URI: jdbc:sqlite:file:/tmp/pyiceberg_catalog.db
2645

27-
rabbitmq:
28-
image: rabbitmq:4-management
46+
s3:
47+
image: quay.io/minio/minio
48+
command: server /data --console-address ":9001"
2949
environment:
30-
RABBITMQ_DEFAULT_USER: rabbitmq
31-
RABBITMQ_DEFAULT_PASS: rabbitmq
32-
RABBITMQ_DEFAULT_VHOST: /
50+
MINIO_ROOT_USER: ${ACCESS_KEY_ID}
51+
MINIO_ROOT_PASSWORD: ${ACCESS_SECRET_ACCESS_KEY}
3352
ports:
34-
- "5672:5672"
35-
- "15672:15672"
36-
expose:
37-
- "5672"
38-
- "15672"
53+
- "9000:9000"
54+
- "9001:9001"
55+
56+
create_bucket:
57+
restart: "no"
58+
image: minio/mc
59+
entrypoint: sh
60+
command: -c "sleep 10s && mc alias set minio http://s3:9000 ${ACCESS_KEY_ID} ${ACCESS_SECRET_ACCESS_KEY} && mc mb minio/${DEFAULT_BUCKET}"
61+
environment:
62+
MINIO_ROOT_USER: ${ACCESS_KEY_ID}
63+
MINIO_ROOT_PASSWORD: ${ACCESS_SECRET_ACCESS_KEY}
64+
depends_on:
65+
- s3

‎main.py‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,19 @@
11
#!/usr/bin/env python
22

33
from servc.server import start_server
4+
from servc.svc.com.worker.types import RESOLVER_MAPPING
5+
from servc_typings.domains.orchestrator import Events
46

5-
from orchestrator.domains.add_dag import add_dag
6-
from orchestrator.domains.trigger import trigger
7-
from orchestrator.domains.update_status import update
8-
from orchestrator.config import QUEUE_NAME
7+
from provisioner.events.dag_updated import registerDag
8+
9+
eventResolver: RESOLVER_MAPPING = {}
10+
eventResolver[Events.DAG_UPDATED.value] = registerDag
911

1012

1113
def main():
1214
return start_server(
13-
resolver={
14-
"add_dag": add_dag,
15-
"trigger": trigger,
16-
"update_status": update,
17-
},
18-
route=QUEUE_NAME,
15+
resolver={},
16+
eventResolver=eventResolver,
1917
)
2018

2119

‎provisioner/__init__.py‎

Whitespace-only changes.

‎provisioner/com/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)