feat: dedicated storage class and nodepool for archive nodes

cmd/main.go

@@ -114,11 +114,12 @@ func main() {
 
 	platformCfg := platform.Config{
 		NodepoolName:        os.Getenv("SEI_NODEPOOL_NAME"),
+		NodepoolArchive:     os.Getenv("SEI_NODEPOOL_ARCHIVE"),
 		TolerationKey:       os.Getenv("SEI_TOLERATION_KEY"),
-		TolerationVal:       os.Getenv("SEI_TOLERATION_VALUE"),
 		ServiceAccount:      os.Getenv("SEI_SERVICE_ACCOUNT"),
 		StorageClassPerf:    os.Getenv("SEI_STORAGE_CLASS_PERF"),
 		StorageClassDefault: os.Getenv("SEI_STORAGE_CLASS_DEFAULT"),
+		StorageClassArchive: os.Getenv("SEI_STORAGE_CLASS_ARCHIVE"),
 		StorageSizeDefault:  os.Getenv("SEI_STORAGE_SIZE_DEFAULT"),
 		StorageSizeArchive:  os.Getenv("SEI_STORAGE_SIZE_ARCHIVE"),
 		ResourceCPUArchive:  os.Getenv("SEI_RESOURCE_CPU_ARCHIVE"),

config/manager/manager.yaml

@@ -37,20 +37,22 @@ spec:
         env:
           - name: SEI_NODEPOOL_NAME
             value: sei-node
+          - name: SEI_NODEPOOL_ARCHIVE
+            value: sei-archive
           - name: SEI_TOLERATION_KEY
             value: sei.io/workload
-          - name: SEI_TOLERATION_VALUE
-            value: sei-node
           - name: SEI_SERVICE_ACCOUNT
             value: seid-node
           - name: SEI_STORAGE_CLASS_PERF
             value: gp3-10k-750
           - name: SEI_STORAGE_CLASS_DEFAULT
             value: gp3
+          - name: SEI_STORAGE_CLASS_ARCHIVE
+            value: io2-archive
           - name: SEI_STORAGE_SIZE_DEFAULT
             value: 2000Gi
           - name: SEI_STORAGE_SIZE_ARCHIVE
-            value: 4000Gi
+            value: 25000Gi
           - name: SEI_RESOURCE_CPU_ARCHIVE
             value: "16"
           - name: SEI_RESOURCE_MEM_ARCHIVE

config/storage/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - storage-classes.yaml

config/storage/storage-classes.yaml

@@ -0,0 +1,24 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: gp3-10k-750
+provisioner: ebs.csi.aws.com
+allowVolumeExpansion: true
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  type: gp3
+  iops: "10000"
+  throughput: "750"
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: io2-archive
+provisioner: ebs.csi.aws.com
+allowVolumeExpansion: true
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  type: io2
+  iopsPerGB: "3"

internal/controller/node/resources.go

@@ -55,10 +55,12 @@ func buildNodePodSpec(node *seiv1alpha1.SeiNode, platform PlatformConfig) corev1
 		},
 	}
 
+	pool := platform.NodepoolForMode(nodeMode(node))
+
 	spec := corev1.PodSpec{
 		ServiceAccountName: platform.ServiceAccount,
 		Tolerations: []corev1.Toleration{
-			{Key: platform.TolerationKey, Value: platform.TolerationVal, Effect: corev1.TaintEffectNoSchedule},
+			{Key: platform.TolerationKey, Value: pool, Effect: corev1.TaintEffectNoSchedule},
 		},
 		Affinity: &corev1.Affinity{
 			NodeAffinity: &corev1.NodeAffinity{
@@ -67,7 +69,7 @@ func buildNodePodSpec(node *seiv1alpha1.SeiNode, platform PlatformConfig) corev1
 						MatchExpressions: []corev1.NodeSelectorRequirement{{
 							Key:      "karpenter.sh/nodepool",
 							Operator: corev1.NodeSelectorOpIn,
-							Values:   []string{platform.NodepoolName},
+							Values:   []string{pool},
 						}},
 					}},
 				},

internal/controller/node/resources_test.go

@@ -624,3 +624,83 @@ func TestGenerateNodeDataPVC(t *testing.T) {
 	storage := pvc.Spec.Resources.Requests[corev1.ResourceStorage]
 	g.Expect(storage.String()).To(Equal("2000Gi"))
 }
+
+func newArchiveNode(name, namespace string) *seiv1alpha1.SeiNode {
+	return &seiv1alpha1.SeiNode{
+		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
+		Spec: seiv1alpha1.SeiNodeSpec{
+			ChainID: "pacific-1",
+			Image:   "ghcr.io/sei-protocol/seid:v6.4.1",
+			Archive: &seiv1alpha1.ArchiveSpec{},
+			Sidecar: &seiv1alpha1.SidecarConfig{Port: 7777},
+		},
+	}
+}
+
+func TestGenerateNodeDataPVC_Archive(t *testing.T) {
+	g := NewWithT(t)
+	node := newArchiveNode("archive-0", "pacific-1")
+
+	pvc := generateNodeDataPVC(node, platformtest.Config())
+
+	g.Expect(*pvc.Spec.StorageClassName).To(Equal("io2-archive"))
+
+	storage := pvc.Spec.Resources.Requests[corev1.ResourceStorage]
+	g.Expect(storage.String()).To(Equal("25000Gi"))
+}
+
+func TestBuildNodePodSpec_Archive_SchedulesOnArchiveNodepool(t *testing.T) {
+	g := NewWithT(t)
+	node := newArchiveNode("archive-0", "pacific-1")
+
+	spec := buildNodePodSpec(node, platformtest.Config())
+
+	g.Expect(spec.Tolerations).To(HaveLen(1))
+	g.Expect(spec.Tolerations[0].Key).To(Equal("sei.io/workload"))
+	g.Expect(spec.Tolerations[0].Value).To(Equal("sei-archive"))
+
+	terms := spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms
+	g.Expect(terms).To(HaveLen(1))
+	g.Expect(terms[0].MatchExpressions).To(HaveLen(1))
+	g.Expect(terms[0].MatchExpressions[0].Key).To(Equal("karpenter.sh/nodepool"))
+	g.Expect(terms[0].MatchExpressions[0].Values).To(ConsistOf("sei-archive"))
+}
+
+func TestBuildNodePodSpec_FullNode_SchedulesOnDefaultNodepool(t *testing.T) {
+	g := NewWithT(t)
+	node := newSnapshotNode("syncer-0", "pacific-1")
+
+	spec := buildNodePodSpec(node, platformtest.Config())
+
+	g.Expect(spec.Tolerations[0].Value).To(Equal("sei-node"))
+
+	terms := spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms
+	g.Expect(terms[0].MatchExpressions[0].Values).To(ConsistOf("sei-node"))
+}
+
+func TestDefaultStorageForMode_Archive(t *testing.T) {
+	g := NewWithT(t)
+	cfg := platformtest.Config()
+
+	sc, size := defaultStorageForMode(string(seiconfig.ModeArchive), cfg)
+	g.Expect(sc).To(Equal("io2-archive"))
+	g.Expect(size).To(Equal("25000Gi"))
+}
+
+func TestDefaultStorageForMode_FullNode(t *testing.T) {
+	g := NewWithT(t)
+	cfg := platformtest.Config()
+
+	sc, size := defaultStorageForMode(string(seiconfig.ModeFull), cfg)
+	g.Expect(sc).To(Equal("gp3-10k-750"))
+	g.Expect(size).To(Equal("2000Gi"))
+}
+
+func TestDefaultResourcesForMode_Archive(t *testing.T) {
+	g := NewWithT(t)
+	cfg := platformtest.Config()
+
+	res := defaultResourcesForMode(string(seiconfig.ModeArchive), cfg)
+	g.Expect(res.Requests[corev1.ResourceCPU]).To(Equal(resource.MustParse("16")))
+	g.Expect(res.Requests[corev1.ResourceMemory]).To(Equal(resource.MustParse("256Gi")))
+}

internal/controller/node/sizing.go

@@ -39,14 +39,13 @@ func defaultResourcesForMode(mode string, platform PlatformConfig) corev1.Resour
 func defaultStorageForMode(mode string, platform PlatformConfig) (storageClass string, size string) {
 	switch mode {
 	case string(seiconfig.ModeArchive):
-		return platform.StorageClassPerf, platform.StorageSizeArchive
+		return platform.StorageClassArchive, platform.StorageSizeArchive
 	case string(seiconfig.ModeFull), string(seiconfig.ModeValidator):
 		return platform.StorageClassPerf, platform.StorageSizeDefault
 	default:
 		return platform.StorageClassDefault, platform.StorageSizeDefault
 	}
 }
-
 func makeResources(cpu, memory string) corev1.ResourceRequirements {
 	return corev1.ResourceRequirements{
 		Requests: corev1.ResourceList{

internal/platform/platform.go

@@ -9,18 +9,22 @@ const (
 
 	// DataDir is the mount path for the sei data volume inside node pods.
 	DataDir = "/sei"
+
+	// modeArchive matches seiconfig.ModeArchive without importing sei-config.
+	modeArchive = "archive"
 )
 
 // Config holds infrastructure-level settings that vary per deployment
 // environment. All fields are required and read from environment variables
 // in main.go. See platformtest.Config() for test fixtures.
 type Config struct {
 	NodepoolName        string
+	NodepoolArchive     string
 	TolerationKey       string
-	TolerationVal       string
 	ServiceAccount      string
 	StorageClassPerf    string
 	StorageClassDefault string
+	StorageClassArchive string
 	StorageSizeDefault  string
 	StorageSizeArchive  string
 	ResourceCPUArchive  string
@@ -43,17 +47,28 @@ type Config struct {
 	GatewayPublicDomain string
 }
 
+// NodepoolForMode returns the Karpenter NodePool name for the given
+// sei-config mode string. Archive nodes use a dedicated pool; all
+// other modes share the default pool.
+func (c Config) NodepoolForMode(mode string) string {
+	if mode == modeArchive {
+		return c.NodepoolArchive
+	}
+	return c.NodepoolName
+}
+
 // Validate returns an error if required fields are missing.
 func (c Config) Validate() error {
 	required := map[string]string{
 		"SEI_NODEPOOL_NAME":         c.NodepoolName,
 		"SEI_TOLERATION_KEY":        c.TolerationKey,
-		"SEI_TOLERATION_VALUE":      c.TolerationVal,
 		"SEI_SERVICE_ACCOUNT":       c.ServiceAccount,
 		"SEI_STORAGE_CLASS_PERF":    c.StorageClassPerf,
 		"SEI_STORAGE_CLASS_DEFAULT": c.StorageClassDefault,
+		"SEI_STORAGE_CLASS_ARCHIVE": c.StorageClassArchive,
 		"SEI_STORAGE_SIZE_DEFAULT":  c.StorageSizeDefault,
 		"SEI_STORAGE_SIZE_ARCHIVE":  c.StorageSizeArchive,
+		"SEI_NODEPOOL_ARCHIVE":      c.NodepoolArchive,
 		"SEI_RESOURCE_CPU_ARCHIVE":  c.ResourceCPUArchive,
 		"SEI_RESOURCE_MEM_ARCHIVE":  c.ResourceMemArchive,
 		"SEI_RESOURCE_CPU_DEFAULT":  c.ResourceCPUDefault,

internal/platform/platformtest/config.go

@@ -8,13 +8,14 @@ import "github.com/sei-protocol/sei-k8s-controller/internal/platform"
 func Config() platform.Config {
 	return platform.Config{
 		NodepoolName:        "sei-node",
+		NodepoolArchive:     "sei-archive",
 		TolerationKey:       "sei.io/workload",
-		TolerationVal:       "sei-node",
 		ServiceAccount:      "seid-node",
 		StorageClassPerf:    "gp3-10k-750",
 		StorageClassDefault: "gp3",
+		StorageClassArchive: "io2-archive",
 		StorageSizeDefault:  "2000Gi",
-		StorageSizeArchive:  "4000Gi",
+		StorageSizeArchive:  "25000Gi",
 		ResourceCPUArchive:  "16",
 		ResourceMemArchive:  "256Gi",
 		ResourceCPUDefault:  "4",

internal/task/bootstrap_resources.go

@@ -162,6 +162,8 @@ func buildBootstrapPodSpec(node *seiv1alpha1.SeiNode, snap *seiv1alpha1.Snapshot
 	seidInit := bootstrapSeidInitContainer(node)
 	seidInit.Image = bootstrapImage
 
+	pool := platformCfg.NodepoolForMode(bootstrapNodeMode(node))
+
 	return corev1.PodSpec{
 		Hostname:                      fmt.Sprintf("%s-0", node.Name),
 		Subdomain:                     serviceName,
@@ -170,7 +172,20 @@ func buildBootstrapPodSpec(node *seiv1alpha1.SeiNode, snap *seiv1alpha1.Snapshot
 		RestartPolicy:                 corev1.RestartPolicyNever,
 		TerminationGracePeriodSeconds: ptr.To(bootstrapTerminationGracePeriod),
 		Tolerations: []corev1.Toleration{
-			{Key: platformCfg.TolerationKey, Value: platformCfg.TolerationVal, Effect: corev1.TaintEffectNoSchedule},
+			{Key: platformCfg.TolerationKey, Value: pool, Effect: corev1.TaintEffectNoSchedule},
+		},
+		Affinity: &corev1.Affinity{
+			NodeAffinity: &corev1.NodeAffinity{
+				RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
+					NodeSelectorTerms: []corev1.NodeSelectorTerm{{
+						MatchExpressions: []corev1.NodeSelectorRequirement{{
+							Key:      "karpenter.sh/nodepool",
+							Operator: corev1.NodeSelectorOpIn,
+							Values:   []string{pool},
+						}},
+					}},
+				},
+			},
 		},
 		Volumes:        []corev1.Volume{dataVolume},
 		InitContainers: []corev1.Container{seidInit, sidecar},