diff --git a/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml b/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml index ee756236f..226830c31 100644 --- a/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml +++ b/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml @@ -136,7 +136,7 @@ jobs: appframeworksS1, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/arm-AL2023-int-test-workflow.yml b/.github/workflows/arm-AL2023-int-test-workflow.yml index 39986ebca..a65cf941f 100644 --- a/.github/workflows/arm-AL2023-int-test-workflow.yml +++ b/.github/workflows/arm-AL2023-int-test-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-al2023 diff --git a/.github/workflows/arm-RHEL-build-test-push-workflow.yml b/.github/workflows/arm-RHEL-build-test-push-workflow.yml index 07848683c..f86a5e632 100644 --- a/.github/workflows/arm-RHEL-build-test-push-workflow.yml +++ b/.github/workflows/arm-RHEL-build-test-push-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-rhel diff --git a/.github/workflows/arm-RHEL-int-test-workflow.yml b/.github/workflows/arm-RHEL-int-test-workflow.yml index 83c7e8fb9..d2a127ca8 100644 --- a/.github/workflows/arm-RHEL-int-test-workflow.yml +++ b/.github/workflows/arm-RHEL-int-test-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-rhel diff --git a/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml b/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml index a22bd10c4..5c6179c20 100644 --- a/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml +++ b/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml @@ -136,7 +136,7 @@ jobs: appframeworksS1, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/arm-Ubuntu-int-test-workflow.yml b/.github/workflows/arm-Ubuntu-int-test-workflow.yml index b3f133abe..8d138f5bb 100644 --- a/.github/workflows/arm-Ubuntu-int-test-workflow.yml +++ b/.github/workflows/arm-Ubuntu-int-test-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-ubuntu diff --git a/.github/workflows/build-test-push-workflow.yml b/.github/workflows/build-test-push-workflow.yml index 01e655adb..dda66569a 100644 --- a/.github/workflows/build-test-push-workflow.yml +++ b/.github/workflows/build-test-push-workflow.yml @@ -200,7 +200,7 @@ jobs: managerappframeworkm4, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/distroless-build-test-push-workflow.yml b/.github/workflows/distroless-build-test-push-workflow.yml index 8aa6172f7..63d9bee33 100644 --- a/.github/workflows/distroless-build-test-push-workflow.yml +++ b/.github/workflows/distroless-build-test-push-workflow.yml @@ -195,7 +195,7 @@ jobs: managerappframeworkm4, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/distroless-int-test-workflow.yml b/.github/workflows/distroless-int-test-workflow.yml index 01a024247..5f16a8472 100644 --- a/.github/workflows/distroless-int-test-workflow.yml +++ b/.github/workflows/distroless-int-test-workflow.yml @@ -73,7 +73,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-distroless diff --git a/.github/workflows/int-test-workflow.yml b/.github/workflows/int-test-workflow.yml index 001a34cee..c545b69eb 100644 --- a/.github/workflows/int-test-workflow.yml +++ b/.github/workflows/int-test-workflow.yml @@ -70,7 +70,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image diff --git a/.github/workflows/manual-int-test-workflow.yml b/.github/workflows/manual-int-test-workflow.yml index 96629316e..0e3e5d357 100644 --- a/.github/workflows/manual-int-test-workflow.yml +++ b/.github/workflows/manual-int-test-workflow.yml @@ -29,7 +29,7 @@ jobs: managerscaling, managercrcrud, licensemanager, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest env: diff --git a/.github/workflows/namespace-scope-int-workflow.yml b/.github/workflows/namespace-scope-int-workflow.yml index fc3d3554f..640f74dc6 100644 --- a/.github/workflows/namespace-scope-int-workflow.yml +++ b/.github/workflows/namespace-scope-int-workflow.yml @@ -25,7 +25,7 @@ jobs: managerscaling, managercrcrud, licensemanager, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest env: diff --git a/.github/workflows/nightly-int-test-workflow.yml b/.github/workflows/nightly-int-test-workflow.yml index 3f886dd9a..184927f04 100644 --- a/.github/workflows/nightly-int-test-workflow.yml +++ b/.github/workflows/nightly-int-test-workflow.yml @@ -66,7 +66,7 @@ jobs: managerscaling, managercrcrud, licensemanager, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image diff --git a/test/appframework_aws/c3/appframework_aws_suite_test.go b/test/appframework_aws/c3/appframework_aws_suite_test.go index 6468b018a..963f9ecc0 100644 --- a/test/appframework_aws/c3/appframework_aws_suite_test.go +++ b/test/appframework_aws/c3/appframework_aws_suite_test.go @@ -17,24 +17,13 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/joho/godotenv" "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) @@ -42,9 +31,6 @@ var ( appListV2 []string testDataS3Bucket = os.Getenv("TEST_BUCKET") testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) @@ -55,87 +41,15 @@ var ( func TestBasic(t *testing.T) { RegisterFailHandler(Fail) - // Find and load the .env file from the current directory upwards - err := loadEnvFile() - Expect(err).ToNot(HaveOccurred(), "Error loading .env file") - - sc, _ := GinkgoConfiguration() - sc.Timeout = 240 * time.Minute - - RunSpecs(t, "Running "+testSuiteName, sc) -} - -//func TestMain(m *testing.M) { -// Run the tests -// os.Exit(m.Run()) -//} - -// loadEnvFile traverses up the directory tree to find a .env file -func loadEnvFile() error { - // Get the current working directory - dir, err := os.Getwd() - if err != nil { - return err - } + Expect(testenv.LoadEnvFile()).ToNot(HaveOccurred(), "Error loading .env file") - // Traverse up the directory tree - for { - // Check if .env file exists in the current directory - envFile := filepath.Join(dir, ".env") - if _, err := os.Stat(envFile); err == nil { - // .env file found, load it - return godotenv.Load(envFile) - } - - // Move up to the parent directory - parentDir := filepath.Dir(dir) - if parentDir == dir { - // Reached the root directory - return nil - } - dir = parentDir - } + RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupS3AppsSuite(testSuiteName, testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_aws/c3/appframework_aws_test.go b/test/appframework_aws/c3/appframework_aws_test.go index 857ba1742..3eddca4e8 100644 --- a/test/appframework_aws/c3/appframework_aws_test.go +++ b/test/appframework_aws/c3/appframework_aws_test.go @@ -25,7 +25,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -53,44 +52,12 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") testenv.SpecifiedTestTimeout = 4000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -174,7 +141,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -193,23 +160,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -274,22 +231,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -355,7 +303,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -395,7 +343,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -414,23 +362,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -483,22 +421,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -577,7 +506,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -593,14 +522,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -650,9 +573,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -670,8 +591,7 @@ var _ = Describe("c3appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -693,12 +613,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, deployment.GetName(), int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -745,9 +660,7 @@ var _ = Describe("c3appfw test", func() { //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -763,8 +676,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -814,7 +726,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("smoke, c3, masterappframeworkc3, appframework: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -858,7 +770,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -874,14 +786,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -923,14 +829,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -953,7 +853,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps @@ -1021,7 +921,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1031,7 +931,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} @@ -1044,7 +944,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1063,14 +963,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1127,14 +1021,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1164,7 +1052,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps @@ -1230,7 +1118,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1240,7 +1128,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1251,7 +1139,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1270,14 +1158,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1350,14 +1232,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1387,7 +1263,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, masterappframeworkc3, appframework: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { /* Test Steps @@ -1413,7 +1289,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files @@ -1432,7 +1308,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1448,14 +1324,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1560,7 +1430,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1573,16 +1443,10 @@ var _ = Describe("c3appfw test", func() { shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1637,14 +1501,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1724,7 +1582,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, masterappframeworkc3, appframework: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { /* Test Steps @@ -1772,7 +1630,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1787,14 +1645,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1836,14 +1688,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1907,7 +1753,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1973,7 +1819,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1983,7 +1829,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1994,7 +1840,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -2012,14 +1858,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2088,14 +1928,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2187,7 +2021,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Master @@ -2206,7 +2040,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2316,7 +2150,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Master @@ -2335,7 +2169,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2372,11 +2206,7 @@ var _ = Describe("c3appfw test", func() { // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify all apps are installed on indexers appList = append(testenv.BigSingleApp, testenv.ExtraApps...) @@ -2419,7 +2249,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2437,7 +2267,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2458,14 +2288,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2513,7 +2337,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2531,7 +2355,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2552,14 +2376,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2622,7 +2440,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2637,14 +2455,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2726,7 +2538,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2743,7 +2555,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2787,14 +2599,8 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2814,7 +2620,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkc3, appframework: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2843,7 +2649,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -2854,7 +2660,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -2863,7 +2669,7 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2881,14 +2687,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2932,7 +2732,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -2950,7 +2750,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2974,14 +2774,8 @@ var _ = Describe("c3appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3036,7 +2830,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3061,18 +2855,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { /* Test Steps @@ -3133,7 +2923,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, masterappframeworkc3, appframework: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps @@ -3156,14 +2946,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Download Technology add-on app from S3 testcaseEnvInst.Log.Info("Download Technology add-on app from S3") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileListIdxc) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileListIdxc) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Create directory for file upload to S3 @@ -3182,7 +2972,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, s3TestDirShc, 180) @@ -3193,7 +2983,7 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 180) @@ -3239,14 +3029,8 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3271,7 +3055,7 @@ var _ = Describe("c3appfw test", func() { // // Download ES App from S3 // appVersion = "V2" // testcaseEnvInst.Log.Info("Download updated ES app from S3") - // err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + // err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") // // Upload V2 ES app to S3 for Search Head Cluster diff --git a/test/appframework_aws/c3/manager_appframework_test.go b/test/appframework_aws/c3/manager_appframework_test.go index a49fd3eb4..0fab3e695 100644 --- a/test/appframework_aws/c3/manager_appframework_test.go +++ b/test/appframework_aws/c3/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -51,44 +50,11 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -172,7 +138,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -191,23 +157,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -272,22 +228,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -331,7 +278,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) default: fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with provider set as %v", testenv.ClusterProvider)) } // Upload V1 apps to S3 for Monitoring Console @@ -496,7 +443,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -536,7 +483,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -555,23 +502,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -624,22 +561,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -718,7 +646,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -734,14 +662,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -791,9 +713,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -811,8 +731,7 @@ var _ = Describe("c3appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -838,12 +757,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, deployment.GetName(), int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -890,9 +804,7 @@ var _ = Describe("c3appfw test", func() { //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -908,8 +820,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -963,7 +874,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("smoke, c3, managerappframeworkc3, appframework: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -1007,7 +918,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1023,14 +934,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1072,14 +977,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1102,7 +1001,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, managerappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps @@ -1170,7 +1069,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1180,7 +1079,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} @@ -1193,7 +1092,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1212,14 +1111,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1276,14 +1169,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1313,7 +1200,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, managerappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps @@ -1379,7 +1266,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1389,7 +1276,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1400,7 +1287,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1419,14 +1306,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1499,14 +1380,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1536,7 +1411,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, managerappframeworkc3, appframework: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { /* Test Steps @@ -1562,7 +1437,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files @@ -1581,7 +1456,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1597,14 +1472,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1709,7 +1578,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1722,16 +1591,10 @@ var _ = Describe("c3appfw test", func() { shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1786,14 +1649,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1873,7 +1730,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, managerappframeworkc3, appframework: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { /* Test Steps @@ -1921,7 +1778,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1936,14 +1793,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1985,14 +1836,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2056,7 +1901,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, managerappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -2122,7 +1967,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2132,7 +1977,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -2143,7 +1988,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -2161,14 +2006,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2237,14 +2076,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2336,7 +2169,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Manager @@ -2355,7 +2188,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2465,7 +2298,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Manager @@ -2484,7 +2317,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2568,7 +2401,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2586,7 +2419,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2607,14 +2440,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2662,7 +2489,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2680,7 +2507,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2701,14 +2528,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2771,7 +2592,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2786,14 +2607,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2875,7 +2690,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2892,7 +2707,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2936,14 +2751,8 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2963,7 +2772,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, managerappframeworkc3, appframework: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2992,7 +2801,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -3003,7 +2812,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -3012,7 +2821,7 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3030,14 +2839,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3081,7 +2884,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -3099,7 +2902,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3123,14 +2926,8 @@ var _ = Describe("c3appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3185,7 +2982,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3210,18 +3007,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { /* Test Steps @@ -3282,7 +3075,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, managerappframeworkc3, appframework: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps @@ -3305,14 +3098,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Download Technology add-on app from S3 testcaseEnvInst.Log.Info("Download Technology add-on app from S3") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileListIdxc) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileListIdxc) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Create directory for file upload to S3 @@ -3331,7 +3124,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, s3TestDirShc, 180) @@ -3342,7 +3135,7 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 180) @@ -3388,14 +3181,8 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3420,7 +3207,7 @@ var _ = Describe("c3appfw test", func() { // // Download ES App from S3 // appVersion = "V2" // testcaseEnvInst.Log.Info("Download updated ES app from S3") - // err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + // err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") // // Upload V2 ES app to S3 for Search Head Cluster diff --git a/test/appframework_aws/m4/appframework_aws_suite_test.go b/test/appframework_aws/m4/appframework_aws_suite_test.go index aa21c7084..95c94f9d0 100644 --- a/test/appframework_aws/m4/appframework_aws_suite_test.go +++ b/test/appframework_aws/m4/appframework_aws_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataS3Bucket = os.Getenv("TEST_BUCKET") testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) @@ -52,54 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) - sc, _ := GinkgoConfiguration() - sc.Timeout = 240 * time.Minute - - RunSpecs(t, "Running "+testSuiteName, sc) + RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupS3AppsSuite(testSuiteName, testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_aws/m4/appframework_aws_test.go b/test/appframework_aws/m4/appframework_aws_test.go index fca638acb..ca5845c9f 100644 --- a/test/appframework_aws/m4/appframework_aws_test.go +++ b/test/appframework_aws/m4/appframework_aws_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -53,16 +52,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -71,30 +61,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps @@ -135,7 +105,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -171,7 +141,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -187,19 +157,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -270,17 +231,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -313,7 +265,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps @@ -354,7 +306,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -390,7 +342,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -403,19 +355,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -472,17 +415,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -512,7 +446,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps @@ -567,7 +501,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -582,17 +516,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -634,9 +559,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -654,8 +577,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -724,9 +646,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -742,8 +662,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -791,7 +710,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -833,7 +752,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -848,14 +767,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -897,14 +810,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -924,7 +831,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -968,7 +875,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) @@ -1004,7 +911,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -1015,19 +922,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1079,17 +977,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1182,7 +1071,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1228,7 +1117,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -1242,14 +1131,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1291,14 +1174,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" @@ -1365,7 +1242,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, masterappframeworkm4, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1431,7 +1308,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1441,7 +1318,7 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} @@ -1453,7 +1330,7 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} @@ -1473,17 +1350,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1553,17 +1421,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1651,7 +1510,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Master @@ -1670,7 +1529,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1683,7 +1542,7 @@ var _ = Describe("m4appfw test", func() { siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1788,7 +1647,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Master @@ -1807,7 +1666,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1821,7 +1680,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1870,7 +1729,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { /* Test Steps @@ -1892,7 +1751,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -1908,7 +1767,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -1920,7 +1779,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) @@ -1928,17 +1787,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1962,7 +1812,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { /* Test Steps @@ -1984,7 +1834,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2000,7 +1850,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2012,7 +1862,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -2020,17 +1870,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2054,7 +1895,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { /* Test Steps @@ -2090,7 +1931,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2102,19 +1943,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2166,7 +1998,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2208,7 +2040,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -2222,14 +2054,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2256,8 +2082,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Master") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2267,8 +2092,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2351,7 +2175,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { /* Test Steps @@ -2383,7 +2207,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2398,7 +2222,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) @@ -2410,7 +2234,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2470,7 +2294,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, masterappframeworkm4, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2499,7 +2323,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -2510,7 +2334,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -2519,7 +2343,7 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2537,14 +2361,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2562,7 +2380,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { /* Test Steps @@ -2584,7 +2402,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -2600,7 +2418,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2612,7 +2430,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2623,17 +2441,8 @@ var _ = Describe("m4appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2657,7 +2466,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { /* Test Steps @@ -2684,7 +2493,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2696,7 +2505,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") @@ -2709,14 +2518,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_aws/m4/manager_appframework_test.go b/test/appframework_aws/m4/manager_appframework_test.go index 03e41df85..701d7cbb4 100644 --- a/test/appframework_aws/m4/manager_appframework_test.go +++ b/test/appframework_aws/m4/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -52,16 +51,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -70,30 +60,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps @@ -134,7 +104,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -170,7 +140,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -186,19 +156,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -269,25 +230,13 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify Monitoring Console is Ready and stays in ready state + // Verify Monitoring Console is ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) // Get Pod age to check for pod resets later @@ -312,7 +261,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps @@ -353,7 +302,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -389,7 +338,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -402,19 +351,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -471,17 +411,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -511,7 +442,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps @@ -566,7 +497,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -581,17 +512,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -633,9 +555,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -653,8 +573,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -723,9 +642,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -741,8 +658,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -752,9 +668,6 @@ var _ = Describe("m4appfw test", func() { err = deployment.UpdateCR(ctx, idxc) Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - // Ensure Indexer cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) - // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -790,7 +703,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -832,7 +745,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -847,14 +760,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -896,14 +803,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -923,7 +824,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -967,7 +868,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) @@ -1003,7 +904,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -1014,19 +915,11 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1078,17 +971,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1108,6 +992,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Modify config map to trigger manual update") config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) + config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) err = deployment.UpdateCR(ctx, config) Expect(err).To(Succeed(), "Unable to update config map") @@ -1124,21 +1009,6 @@ var _ = Describe("m4appfw test", func() { config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) Expect(err).To(Succeed(), "Unable to get config map for manual poll") - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) err = deployment.UpdateCR(ctx, config) @@ -1161,7 +1031,7 @@ var _ = Describe("m4appfw test", func() { Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - // ############ VERIFY APPS UPDATED TO V2 ############# + //############### UPGRADE APPS ################ appVersion = "V2" cmAppSourceInfo.CrAppVersion = appVersion cmAppSourceInfo.CrAppList = appListV2 @@ -1180,7 +1050,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1226,7 +1096,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -1240,14 +1110,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1289,14 +1153,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" @@ -1340,7 +1198,7 @@ var _ = Describe("m4appfw test", func() { // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# + // ########## Verify Manual Poll disabled after the poll is triggered ################# // Verify config map set back to off after poll trigger testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) @@ -1363,8 +1221,8 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4, integration, managerappframeworkm4, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + It("integration, m4, managerappframeworkm4, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps ################## SETUP #################### @@ -1429,7 +1287,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1439,7 +1297,7 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} @@ -1451,7 +1309,7 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} @@ -1471,17 +1329,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1551,17 +1400,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1594,281 +1434,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("integration, m4, managerappframeworkm4, appframework: can deploy a M4, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to S3 for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to S3 for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from S3 during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to S3 for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all test apps from S3 - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to S3 for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to S3 for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to S3 for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to S3 for Deployer - testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for SearchHeadCluster to reach Ready phase - err = testcaseEnvInst.WaitForSearchHeadClusterPhase(ctx, deployment, testcaseEnvInst.GetName(), shc.Name, enterpriseApi.PhaseReady, 60*time.Second) - Expect(err).To(Succeed(), "Timed out waiting for SearchHeadCluster to reach Ready phase") - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Deployer - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("smoke, m4, managerappframeworkm4, appframework: can deploy a M4, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to S3 for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to S3 for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from S3 during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to S3 for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all test apps from S3 - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to S3 for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to S3 for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to S3 for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to S3 for Deployer - testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { /* Test Steps @@ -1890,7 +1456,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -1906,7 +1472,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -1918,25 +1484,16 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1960,7 +1517,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { /* Test Steps @@ -1982,7 +1539,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -1998,7 +1555,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2010,7 +1567,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -2018,17 +1575,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2052,7 +1600,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { /* Test Steps @@ -2088,7 +1636,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2100,19 +1648,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2164,7 +1703,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2206,7 +1745,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -2220,14 +1759,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2254,8 +1787,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Manager CR with latest config cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Manager") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Manager") // Set AppsRepoPollInterval for Cluster Manager to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Manager to 180 seconds") @@ -2265,8 +1797,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2349,7 +1880,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { /* Test Steps @@ -2381,7 +1912,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2396,7 +1927,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) @@ -2408,7 +1939,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2468,7 +1999,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, managerappframeworkm4, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2497,7 +2028,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -2508,7 +2039,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -2517,7 +2048,7 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2560,7 +2091,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { /* Test Steps @@ -2582,7 +2113,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -2598,7 +2129,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2610,7 +2141,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2621,17 +2152,8 @@ var _ = Describe("m4appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2655,7 +2177,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { /* Test Steps @@ -2682,7 +2204,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2694,7 +2216,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") @@ -2707,14 +2229,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_aws/s1/appframework_aws_suite_test.go b/test/appframework_aws/s1/appframework_aws_suite_test.go index 252889490..ef3f14fe8 100644 --- a/test/appframework_aws/s1/appframework_aws_suite_test.go +++ b/test/appframework_aws/s1/appframework_aws_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataS3Bucket = os.Getenv("TEST_BUCKET") testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) @@ -52,47 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupS3AppsSuite(testSuiteName, testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_aws/s1/appframework_aws_test.go b/test/appframework_aws/s1/appframework_aws_test.go index bf91e80ce..3b044274a 100644 --- a/test/appframework_aws/s1/appframework_aws_test.go +++ b/test/appframework_aws/s1/appframework_aws_test.go @@ -23,7 +23,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -46,43 +45,14 @@ var _ = Describe("s1appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator) }) Context("Standalone deployment (S1) with App Framework", func() { @@ -130,7 +100,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -164,7 +134,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -185,7 +155,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -310,7 +280,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -334,7 +304,7 @@ var _ = Describe("s1appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -354,7 +324,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -481,7 +451,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -511,7 +481,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -531,7 +501,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -566,8 +536,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -599,8 +568,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale down Standalone") scaledReplicaCount = 1 standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone after scaling down") standalone.Spec.Replicas = int32(scaledReplicaCount) err = deployment.UpdateCR(ctx, standalone) @@ -669,7 +637,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -686,7 +654,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -706,8 +674,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -773,7 +740,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload ES app to S3 @@ -782,7 +749,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload ES app to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, s3TestDir, 60) appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ @@ -805,7 +772,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone with App Framework") // Ensure Standalone goes to Ready phase testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -829,7 +796,7 @@ var _ = Describe("s1appfw test", func() { // Download ES App from S3 testcaseEnvInst.Log.Info("Download updated ES app from S3") - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV2, downloadDirV2, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload V2 apps to S3 for Standalone @@ -884,7 +851,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -894,7 +861,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -974,7 +941,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) @@ -1003,7 +970,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 0) @@ -1023,7 +990,7 @@ var _ = Describe("s1appfw test", func() { // Create Standalone Deployment with App Framework standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1148,7 +1115,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download the extra apps from S3 for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to S3 for first Standalone @@ -1166,7 +1133,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1180,7 +1147,7 @@ var _ = Describe("s1appfw test", func() { AppFrameworkConfig: appFrameworkSpec, } - // Create App framework Spec + // Create App Framework Spec appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, s3TestDirStandalone2, 60) @@ -1250,7 +1217,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -1278,7 +1245,7 @@ var _ = Describe("s1appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone @@ -1289,7 +1256,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1309,7 +1276,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1357,7 +1324,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone @@ -1366,7 +1333,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1383,7 +1350,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -1447,7 +1414,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone @@ -1456,7 +1423,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1473,7 +1440,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -1529,7 +1496,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1547,7 +1514,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1609,7 +1576,7 @@ var _ = Describe("s1appfw test", func() { * Verify no pod resets triggered due to app install * Verify App enabled and version by running splunk cmd // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec + * Create App Framework volume with random credentials and apply to Spec * Check for changes in App phase to determine if next poll has been triggered ############ UPGRADE V2 APPS ########### * Upload V2 apps to S3 App Source @@ -1640,7 +1607,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1659,7 +1626,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) Expect(err).To(Succeed(), "Unable to obtain secret object") @@ -1679,7 +1646,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec + // Create App Framework volume with invalid credentials and apply to Spec testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) Expect(err).To(Succeed(), "Unable to update secret Object") @@ -1752,7 +1719,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload apps to S3 for Standalone @@ -1761,7 +1728,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 120) spec := enterpriseApi.StandaloneSpec{ @@ -1778,7 +1745,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -1844,7 +1811,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 @@ -1856,7 +1823,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 15 - // Create App framework Spec + // Create App Framework Spec appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1911,7 +1878,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big app") // Upload big-size app to S3 for Standalone @@ -1920,7 +1887,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1937,7 +1904,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App Download is completed on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -1991,7 +1958,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -2029,7 +1996,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -2050,7 +2017,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Standalone CR testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind) diff --git a/test/appframework_az/c3/appframework_azure_suite_test.go b/test/appframework_az/c3/appframework_azure_suite_test.go index 313c7c4fc..83875f709 100644 --- a/test/appframework_az/c3/appframework_azure_suite_test.go +++ b/test/appframework_az/c3/appframework_azure_suite_test.go @@ -14,11 +14,9 @@ package azurec3appfw import ( - "context" "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) @@ -43,9 +32,6 @@ var ( AzureDataContainer = os.Getenv("TEST_CONTAINER") AzureContainer = os.Getenv("INDEXES_CONTAINER") AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) @@ -53,49 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } + testenvInstance, appListV1, appListV2 = testenv.SetupAzureAppsSuite(testSuiteName, downloadDirV1, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_az/c3/appframework_azure_test.go b/test/appframework_az/c3/appframework_azure_test.go index c4c6b4eff..b20c56c44 100644 --- a/test/appframework_az/c3/appframework_azure_test.go +++ b/test/appframework_az/c3/appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -49,43 +48,11 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -171,7 +138,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -190,23 +157,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -264,22 +221,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -345,7 +293,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -385,7 +333,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -404,23 +352,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -474,22 +412,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -568,7 +497,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -584,14 +513,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -631,9 +554,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -651,8 +572,7 @@ var _ = Describe("c3appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -674,12 +594,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, deployment.GetName(), int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -718,9 +633,7 @@ var _ = Describe("c3appfw test", func() { //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -736,8 +649,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -787,7 +699,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -831,7 +743,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -847,14 +759,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -897,14 +803,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -927,7 +827,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps @@ -950,7 +850,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Azure") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from Azure") @@ -977,7 +877,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) @@ -988,7 +888,7 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -1034,14 +934,8 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1071,7 +965,7 @@ var _ = Describe("c3appfw test", func() { // Download ES App from Azure appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 + containerName = "/" + AzureDataContainer + "/" + testenv.AppLocationV2 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -1084,14 +978,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1108,7 +996,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps @@ -1176,7 +1064,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1186,7 +1074,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} @@ -1199,7 +1087,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1218,14 +1106,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1299,14 +1181,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1336,7 +1212,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps @@ -1405,7 +1281,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1415,7 +1291,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1426,7 +1302,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1445,14 +1321,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1514,14 +1384,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1551,7 +1415,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { /* Test Steps @@ -1577,7 +1441,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1597,7 +1461,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1613,14 +1477,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1725,7 +1583,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1738,16 +1596,10 @@ var _ = Describe("c3appfw test", func() { shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1803,14 +1655,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1890,7 +1736,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { /* Test Steps @@ -1938,7 +1784,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1953,14 +1799,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2003,14 +1843,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2074,7 +1908,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -2140,7 +1974,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2150,7 +1984,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -2161,7 +1995,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -2179,14 +2013,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2256,14 +2084,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2355,7 +2177,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2375,7 +2197,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2485,7 +2307,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2505,7 +2327,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2589,7 +2411,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2608,7 +2430,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2629,14 +2451,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2684,7 +2500,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2703,7 +2519,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2724,14 +2540,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2795,7 +2605,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2810,14 +2620,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2901,7 +2705,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2919,7 +2723,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2963,14 +2767,8 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2990,7 +2788,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -3026,7 +2824,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) @@ -3035,7 +2833,7 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3053,14 +2851,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3104,7 +2896,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -3123,7 +2915,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3147,14 +2939,8 @@ var _ = Describe("c3appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3209,7 +2995,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3234,18 +3020,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { /* Test Steps diff --git a/test/appframework_az/c3/manager_appframework_azure_test.go b/test/appframework_az/c3/manager_appframework_azure_test.go index 8a8a1ec5c..17092862c 100644 --- a/test/appframework_az/c3/manager_appframework_azure_test.go +++ b/test/appframework_az/c3/manager_appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -48,43 +47,11 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -169,7 +136,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -188,23 +155,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -262,22 +219,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -343,7 +291,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -383,7 +331,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -402,23 +350,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -472,22 +410,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -566,7 +495,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -582,14 +511,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -629,9 +552,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -649,8 +570,7 @@ var _ = Describe("c3appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -672,12 +592,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, deployment.GetName(), int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -716,9 +631,7 @@ var _ = Describe("c3appfw test", func() { //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -734,8 +647,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -785,7 +697,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -829,7 +741,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -845,14 +757,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -895,14 +801,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -925,7 +825,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps @@ -948,7 +848,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Azure") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from Azure") @@ -975,7 +875,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) @@ -986,7 +886,7 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -1032,14 +932,8 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1069,7 +963,7 @@ var _ = Describe("c3appfw test", func() { // Download ES App from Azure appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 + containerName = "/" + AzureDataContainer + "/" + testenv.AppLocationV2 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -1082,14 +976,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - // // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1107,7 +995,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration,managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps @@ -1175,7 +1063,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1185,7 +1073,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} @@ -1198,7 +1086,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1217,14 +1105,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1298,14 +1180,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1335,7 +1211,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps @@ -1404,7 +1280,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1414,7 +1290,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1425,7 +1301,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1444,14 +1320,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1514,14 +1384,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1551,7 +1415,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { /* Test Steps @@ -1577,7 +1441,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1597,7 +1461,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1613,14 +1477,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1725,7 +1583,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1738,16 +1596,10 @@ var _ = Describe("c3appfw test", func() { shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1803,14 +1655,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1890,7 +1736,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { /* Test Steps @@ -1938,7 +1784,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1953,14 +1799,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2003,14 +1843,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2074,7 +1908,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -2140,7 +1974,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2150,7 +1984,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -2161,7 +1995,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -2179,14 +2013,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2256,14 +2084,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2355,7 +2177,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2375,7 +2197,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2485,7 +2307,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2505,7 +2327,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2589,7 +2411,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2608,7 +2430,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2629,14 +2451,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2684,7 +2500,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2703,7 +2519,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2724,14 +2540,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2795,7 +2605,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2810,14 +2620,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2901,7 +2705,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2919,7 +2723,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2963,14 +2767,8 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2990,7 +2788,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -3026,7 +2824,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) @@ -3035,7 +2833,7 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3053,14 +2851,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3104,7 +2896,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -3123,7 +2915,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3147,14 +2939,8 @@ var _ = Describe("c3appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3209,7 +2995,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3234,18 +3020,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { /* Test Steps diff --git a/test/appframework_az/m4/appframework_azure_suite_test.go b/test/appframework_az/m4/appframework_azure_suite_test.go index 6184e543b..a416bedc9 100644 --- a/test/appframework_az/m4/appframework_azure_suite_test.go +++ b/test/appframework_az/m4/appframework_azure_suite_test.go @@ -14,11 +14,9 @@ package azurem4appfw import ( - "context" "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) @@ -43,9 +32,6 @@ var ( AzureDataContainer = os.Getenv("TEST_CONTAINER") AzureContainer = os.Getenv("INDEXES_CONTAINER") AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) @@ -53,53 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } + testenvInstance, appListV1, appListV2 = testenv.SetupAzureAppsSuite(testSuiteName, downloadDirV1, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_az/m4/appframework_azure_test.go b/test/appframework_az/m4/appframework_azure_test.go index ce5cba659..e8ede677d 100644 --- a/test/appframework_az/m4/appframework_azure_test.go +++ b/test/appframework_az/m4/appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -52,16 +51,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -70,31 +60,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps @@ -135,7 +104,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -171,7 +140,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -187,28 +156,15 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -259,17 +215,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -302,7 +249,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps @@ -343,7 +290,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -379,7 +326,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -392,19 +339,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -462,17 +400,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -502,7 +431,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps @@ -557,7 +486,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -572,17 +501,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -614,9 +534,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -634,8 +552,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -696,9 +613,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -714,8 +629,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -763,7 +677,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -805,7 +719,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) @@ -820,14 +734,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -870,14 +778,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -897,7 +799,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -941,7 +843,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) @@ -977,7 +879,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) @@ -988,19 +890,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1052,17 +945,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1154,7 +1038,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1200,7 +1084,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) @@ -1214,14 +1098,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1264,14 +1142,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" @@ -1338,7 +1210,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4azure, integration, masterappframeworkazurem4, appframeworkazure: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1404,7 +1276,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1414,7 +1286,7 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} @@ -1426,7 +1298,7 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} @@ -1446,17 +1318,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1527,17 +1390,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1625,7 +1479,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1645,7 +1499,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1658,7 +1512,7 @@ var _ = Describe("m4appfw test", func() { siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1762,7 +1616,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1782,7 +1636,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1796,7 +1650,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1845,7 +1699,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { /* Test Steps @@ -1867,7 +1721,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1884,7 +1738,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -1896,7 +1750,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) @@ -1904,17 +1758,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1938,7 +1783,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { /* Test Steps @@ -1960,7 +1805,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1977,7 +1822,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -1989,7 +1834,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -1997,17 +1842,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2031,7 +1867,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { /* Test Steps @@ -2068,7 +1904,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -2080,19 +1916,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2146,7 +1973,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2188,7 +2015,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) @@ -2202,14 +2029,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2236,8 +2057,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Master") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2247,8 +2067,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2332,7 +2151,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { /* Test Steps @@ -2364,7 +2183,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2380,7 +2199,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) @@ -2392,7 +2211,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2423,17 +2242,8 @@ var _ = Describe("m4appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2452,7 +2262,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4azure, integration, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2488,7 +2298,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") @@ -2497,7 +2307,7 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2515,14 +2325,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2540,7 +2344,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { /* Test Steps @@ -2562,7 +2366,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2579,7 +2383,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -2591,7 +2395,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2602,17 +2406,8 @@ var _ = Describe("m4appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2636,7 +2431,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { /* Test Steps @@ -2663,7 +2458,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -2675,7 +2470,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") @@ -2688,14 +2483,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_az/m4/manager_appframework_azure_test.go b/test/appframework_az/m4/manager_appframework_azure_test.go index 96d24efc2..d82223754 100644 --- a/test/appframework_az/m4/manager_appframework_azure_test.go +++ b/test/appframework_az/m4/manager_appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -51,16 +50,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -69,28 +59,7 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator) }) Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { @@ -134,7 +103,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -170,7 +139,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -186,28 +155,15 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -258,17 +214,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -342,7 +289,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -378,7 +325,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -391,19 +338,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -461,17 +399,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -556,7 +485,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -571,17 +500,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -613,9 +533,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -633,8 +551,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -695,9 +612,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -713,8 +628,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -762,7 +676,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -804,7 +718,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) @@ -819,14 +733,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -869,14 +777,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -896,7 +798,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -940,7 +842,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) @@ -976,7 +878,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) @@ -987,19 +889,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1051,17 +944,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1153,7 +1037,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1199,7 +1083,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) @@ -1213,14 +1097,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1263,14 +1141,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" @@ -1337,7 +1209,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4azure, integration, managerappframeworkazurem4, appframeworkazure: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1403,7 +1275,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1413,7 +1285,7 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} @@ -1425,7 +1297,7 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} @@ -1445,17 +1317,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1526,17 +1389,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1624,7 +1478,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1644,7 +1498,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1657,7 +1511,7 @@ var _ = Describe("m4appfw test", func() { siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1762,7 +1616,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1782,7 +1636,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1796,7 +1650,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1867,7 +1721,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1884,7 +1738,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -1896,7 +1750,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) @@ -1904,17 +1758,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1960,7 +1805,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1977,7 +1822,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -1989,7 +1834,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -1997,17 +1842,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2068,7 +1904,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -2080,19 +1916,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2146,7 +1973,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2188,7 +2015,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) @@ -2202,14 +2029,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2236,8 +2057,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Master") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2247,8 +2067,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2364,7 +2183,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2380,7 +2199,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) @@ -2392,7 +2211,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2423,17 +2242,8 @@ var _ = Describe("m4appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2488,7 +2298,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") @@ -2497,7 +2307,7 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2515,14 +2325,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2562,7 +2366,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2579,7 +2383,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -2591,7 +2395,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2602,17 +2406,8 @@ var _ = Describe("m4appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2663,7 +2458,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -2675,7 +2470,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") @@ -2688,14 +2483,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_az/s1/appframework_azure_suite_test.go b/test/appframework_az/s1/appframework_azure_suite_test.go index d3c06c3c4..3eff208c0 100644 --- a/test/appframework_az/s1/appframework_azure_suite_test.go +++ b/test/appframework_az/s1/appframework_azure_suite_test.go @@ -14,11 +14,9 @@ package azures1appfw import ( - "context" "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) @@ -43,9 +32,6 @@ var ( AzureDataContainer = os.Getenv("TEST_CONTAINER") AzureContainer = os.Getenv("INDEXES_CONTAINER") AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) @@ -53,49 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } + testenvInstance, appListV1, appListV2 = testenv.SetupAzureAppsSuite(testSuiteName, downloadDirV1, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_az/s1/appframework_azure_test.go b/test/appframework_az/s1/appframework_azure_test.go index 92736ef92..d43a24954 100644 --- a/test/appframework_az/s1/appframework_azure_test.go +++ b/test/appframework_az/s1/appframework_azure_test.go @@ -20,7 +20,6 @@ import ( "strings" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -42,45 +41,14 @@ var _ = Describe("s1appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") azTestDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator) }) Context("Standalone deployment (S1) with App Framework", func() { @@ -131,7 +99,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -164,7 +132,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -185,7 +153,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -300,7 +268,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -324,7 +292,7 @@ var _ = Describe("s1appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -344,7 +312,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -474,7 +442,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -505,7 +473,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -525,7 +493,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -550,8 +518,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -575,8 +542,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale down Standalone") scaledReplicaCount = 1 standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone after scaling down") standalone.Spec.Replicas = int32(scaledReplicaCount) err = deployment.UpdateCR(ctx, standalone) @@ -646,7 +612,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -663,7 +629,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -683,8 +649,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -751,7 +716,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Azure") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -762,7 +727,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload ES app to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, azTestDir, 60) appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ @@ -785,7 +750,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone with App Framework") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -805,7 +770,7 @@ var _ = Describe("s1appfw test", func() { azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) // Download ES App from Azure - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 + containerName = "/" + AzureDataContainer + "/" + testenv.AppLocationV2 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -861,7 +826,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -872,7 +837,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -952,7 +917,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) @@ -981,7 +946,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 0) @@ -1001,7 +966,7 @@ var _ = Describe("s1appfw test", func() { // Create Standalone Deployment with App Framework standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1115,7 +1080,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download the extra apps from Azure for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1134,7 +1099,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1148,7 +1113,7 @@ var _ = Describe("s1appfw test", func() { AppFrameworkConfig: appFrameworkSpec, } - // Create App framework Spec + // Create App Framework Spec appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, azTestDirStandalone2, 60) @@ -1218,7 +1183,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -1246,7 +1211,7 @@ var _ = Describe("s1appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1258,7 +1223,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1278,7 +1243,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1326,7 +1291,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1337,7 +1302,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1354,7 +1319,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) @@ -1397,7 +1362,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1408,7 +1373,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1425,7 +1390,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -1482,7 +1447,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1500,7 +1465,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1564,7 +1529,7 @@ var _ = Describe("s1appfw test", func() { * Verify no pod resets triggered due to app install * Verify App enabled and version by running splunk cmd // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec + * Create App Framework volume with random credentials and apply to Spec * Check for changes in App phase to determine if next poll has been triggered ############ UPGRADE V2 APPS ########### * Upload V2 apps to Azure App Source @@ -1595,7 +1560,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1614,7 +1579,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") secretStruct, _ := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) secretData := secretStruct.Data @@ -1633,7 +1598,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec + // Create App Framework volume with invalid credentials and apply to Spec testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) Expect(err).To(Succeed(), "Unable to update secret Object") @@ -1707,7 +1672,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1717,7 +1682,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 120) spec := enterpriseApi.StandaloneSpec{ @@ -1734,7 +1699,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -1811,7 +1776,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 15 - // Create App framework Spec + // Create App Framework Spec appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1866,7 +1831,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big app") @@ -1876,7 +1841,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1893,7 +1858,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App Download is completed on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -1947,7 +1912,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -1985,7 +1950,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -2006,7 +1971,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Standalone CR testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind) diff --git a/test/appframework_gcp/c3/appframework_gcs_suite_test.go b/test/appframework_gcp/c3/appframework_gcs_suite_test.go index 9aa061bad..c73826a7a 100644 --- a/test/appframework_gcp/c3/appframework_gcs_suite_test.go +++ b/test/appframework_gcp/c3/appframework_gcs_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataGcsBucket = os.Getenv("TEST_BUCKET") testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) @@ -52,51 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to Gcs - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Gcs after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupGCPAppsSuite(testSuiteName, testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files.") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files.") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_gcp/c3/appframework_gcs_test.go b/test/appframework_gcp/c3/appframework_gcs_test.go index 3a189f7a0..a9250959a 100644 --- a/test/appframework_gcp/c3/appframework_gcs_test.go +++ b/test/appframework_gcp/c3/appframework_gcs_test.go @@ -17,7 +17,6 @@ import ( "context" //"encoding/json" "fmt" - "path/filepath" //"strings" //"time" @@ -26,7 +25,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" //splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -54,44 +52,12 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") testenv.SpecifiedTestTimeout = 5000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to GCS - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -175,7 +141,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -194,23 +160,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -275,22 +231,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -356,7 +303,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -396,7 +343,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -415,23 +362,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -484,22 +421,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -522,7 +450,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It(" c3gcp, masterappframeworkc3gcp, c3_gcp_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -566,7 +494,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -582,14 +510,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -631,14 +553,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -691,7 +607,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -716,11 +632,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_gcp/c3/manager_appframework_test.go b/test/appframework_gcp/c3/manager_appframework_test.go index 7c10a6b87..cb492d28e 100644 --- a/test/appframework_gcp/c3/manager_appframework_test.go +++ b/test/appframework_gcp/c3/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -51,45 +50,12 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") testenv.SpecifiedTestTimeout = 100000 }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Gcs - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator) }) XContext("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -97,10 +63,10 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -112,7 +78,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs + * Upload V2 apps on GCS * Wait for Monitoring Console and C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -125,13 +91,13 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source @@ -159,21 +125,21 @@ var _ = Describe("c3appfw test", func() { // Verify Monitoring Console is ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -242,32 +208,32 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil // get revision number of the resource resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - // Upload V2 apps to Gcs for Indexer Cluster + // Upload V2 apps to GCS for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V2 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + // Upload V2 apps to GCS for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered @@ -322,7 +288,7 @@ var _ = Describe("c3appfw test", func() { switch testenv.ClusterProvider { case "eks": licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from Gcs") + Expect(err).To(Succeed(), "Unable to download license file from GCS") // Create License Config Map testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) case "azure": @@ -337,10 +303,10 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) default: fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with provider set as %v", testenv.ClusterProvider)) } - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console oldImage := "Refer to RELATED_SPLUNK_IMAGE_ENTERPRISE" newImage := "splunk/splunk:latest" @@ -464,10 +430,10 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V2 apps to Gcs for Monitoring Console + * Upload V2 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V2 apps to GCS for Indexer Cluster and Search Head Cluster * Create app source for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ########### INITIAL VERIFICATIONS ########### @@ -479,7 +445,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods ############## DOWNGRADE APPS ############### - * Upload V1 apps on Gcs + * Upload V1 apps on GCS * Wait for Monitoring Console and C3 pods to be ready ########### FINAL VERIFICATIONS ############# * Verify Apps are Downloaded in App Deployment Info @@ -492,16 +458,16 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V2 apps to Gcs for Monitoring Console + // Upload V2 apps to GCS for Monitoring Console appVersion := "V2" appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -527,21 +493,21 @@ var _ = Describe("c3appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Upload V2 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V2 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V2 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -598,32 +564,32 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) //############## DOWNGRADE APPS ############### - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil // get revision number of the resource resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion = "V1" appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexers", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexers", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexers", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + // Upload V1 apps to GCS for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered @@ -672,7 +638,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps on Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps on GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app config and wait for pods to be ready ########## INITIAL VERIFICATIONS ############ @@ -707,23 +673,23 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP ################## - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) appFileList := testenv.GetAppFileList(appListV1) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -796,9 +762,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -816,8 +780,7 @@ var _ = Describe("c3appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -839,12 +802,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, deployment.GetName(), int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -891,9 +849,7 @@ var _ = Describe("c3appfw test", func() { //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -909,8 +865,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -960,12 +915,12 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It(" c3gcp, c3_mgr_gcp_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs + * Upload V1 apps to GCS * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ############# INITIAL VERIFICATIONS ########## @@ -988,23 +943,23 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster + // Upload V1 apps to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1050,20 +1005,20 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) //############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete V1 apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil - // Upload V2 apps to Gcs + // Upload V2 apps to GCS appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS", appVersion)) appFileList = testenv.GetAppFileList(appListV2) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered @@ -1099,13 +1054,13 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps ################## SETUP #################### * Split Applist into clusterlist and local list - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster for local and cluster scope * Create app sources for Cluster Manager and Deployer with local and cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -1117,7 +1072,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs + * Upload V2 apps on GCS * Wait for all C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -1135,39 +1090,39 @@ var _ = Describe("c3appfw test", func() { appListLocal := appListV1[len(appListV1)/2:] appListCluster := appListV1[:len(appListV1)/2] - // Upload appListLocal list of apps to Gcs (to be used for local install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Indexers", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1177,7 +1132,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} @@ -1190,7 +1145,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1241,8 +1196,8 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil @@ -1250,24 +1205,24 @@ var _ = Describe("c3appfw test", func() { appListLocal = appListV1[len(appListV1)/2:] appListCluster = appListV1[:len(appListV1)/2] - // Upload appListLocal list of V2 apps to Gcs (to be used for local install) + // Upload appListLocal list of V2 apps to GCS (to be used for local install) appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Indexers", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) + // Upload appListCluster list of V2 apps to GCS (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered @@ -1310,13 +1265,13 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps ################## SETUP #################### * Split Applist into clusterlist and local list - * Upload V2 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope + * Upload V2 apps to GCS for Indexer Cluster and Search Head Cluster for local and cluster scope * Create app sources for Cluster Manager and Deployer with local and cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -1328,7 +1283,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V2 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### Downgrade APPS ################ - * Upload V1 apps on Gcs + * Upload V1 apps on GCS * Wait for all C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -1346,37 +1301,37 @@ var _ = Describe("c3appfw test", func() { appListLocal := appListV2[len(appListV2)/2:] appListCluster := appListV2[:len(appListV2)/2] - // Upload appListLocal list of apps to Gcs (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Idxc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListLocal list of apps to Gcs (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Shc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1386,7 +1341,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1397,7 +1352,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1464,8 +1419,8 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) //############# DOWNGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil @@ -1473,24 +1428,24 @@ var _ = Describe("c3appfw test", func() { appListLocal = appListV1[len(appListV1)/2:] appListCluster = appListV1[:len(appListV1)/2] - // Upload appListLocal list of V1 apps to Gcs (to be used for local install) + // Upload appListLocal list of V1 apps to GCS (to be used for local install) appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) + // Upload appListCluster list of V2 apps to GCS (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered @@ -1533,13 +1488,13 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, appframework: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { /* Test Steps ################## SETUP #################### * Create App Source for C3 SVA (Cluster Manager and Deployer) - * Add more apps than usual on Gcs for this test + * Add more apps than usual on GCS for this test * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ############### VERIFICATIONS ############### * Verify Apps are Downloaded in App Deployment Info @@ -1557,28 +1512,28 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appList) appVersion := "V1" - // Download apps from Gcs - testcaseEnvInst.Log.Info("Download bigger amount of apps from Gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + // Download apps from GCS + testcaseEnvInst.Log.Info("Download bigger amount of apps from GCS for this test") + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files appList = append(appListV1, appList...) appFileList = testenv.GetAppFileList(appList) - // Upload app to Gcs for Indexer Cluster + // Upload app to GCS for Indexer Cluster gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to Gcs test directory for Indexer Cluster") + Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload app to Gcs for Search Head Cluster + // Upload app to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1629,10 +1584,10 @@ var _ = Describe("c3appfw test", func() { It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled for manual update", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Gcs + * Upload V1 apps to GCS * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ########## INITIAL VERIFICATION ############# @@ -1660,13 +1615,13 @@ var _ = Describe("c3appfw test", func() { */ // ################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source @@ -1694,19 +1649,19 @@ var _ = Describe("c3appfw test", func() { // Verify Monitoring Console is ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster + // Upload V1 apps to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1719,7 +1674,7 @@ var _ = Describe("c3appfw test", func() { shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -1756,28 +1711,28 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) // ############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete V1 apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil - // Upload V2 apps to Gcs for C3 + // Upload V2 apps to GCS for C3 appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to Gcs", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to GCS", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + // Upload V2 apps to GCS for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered @@ -1870,12 +1825,12 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, appframework: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs + * Upload V1 apps to GCS * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ############# INITIAL VERIFICATION ########## @@ -1902,23 +1857,23 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster + // Upload V1 apps to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1963,20 +1918,20 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) //############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete V1 apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil - // Upload V2 apps to Gcs + // Upload V2 apps to GCS appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS", appVersion)) appFileList = testenv.GetAppFileList(appListV2) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered @@ -2053,13 +2008,13 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps ################## SETUP #################### * Split Applist into clusterlist and local list - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster for local and cluster scope * Create app sources for Cluster Manager and Deployer with local and cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -2071,7 +2026,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs + * Upload V2 apps on GCS * Wait for all C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -2089,37 +2044,37 @@ var _ = Describe("c3appfw test", func() { appListLocal := appListV1[len(appListV1)/2:] appListCluster := appListV1[:len(appListV1)/2] - // Upload appListLocal list of apps to Gcs (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Idxc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListLocal list of apps to Gcs (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Shc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2129,7 +2084,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -2140,7 +2095,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -2190,8 +2145,8 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil @@ -2199,24 +2154,24 @@ var _ = Describe("c3appfw test", func() { appListLocal = appListV1[len(appListV1)/2:] appListCluster = appListV1[:len(appListV1)/2] - // Upload appListLocal list of V2 apps to Gcs (to be used for local install) + // Upload appListLocal list of V2 apps to GCS (to be used for local install) appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) + // Upload appListCluster list of V2 apps to GCS (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // ############ ENABLE MANUAL POLL ############ @@ -2282,27 +2237,27 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster + * Upload big-size app to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework ############## VERIFICATIONS ################ * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Gcs during bigger app install + * Upload more apps from GCS during bigger app install * Wait for polling interval to pass * Verify all apps are installed on Cluster Manager and Deployer */ //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source @@ -2330,29 +2285,29 @@ var _ = Describe("c3appfw test", func() { // Verify Monitoring Console is ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload big-size app to Gcs for Cluster Manager + // Upload big-size app to GCS for Cluster Manager appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Cluster Manager") gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload big-size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Search Head Cluster") + // Upload big-size app to GCS for Search Head Cluster + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Search Head Cluster") gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2369,18 +2324,18 @@ var _ = Describe("c3appfw test", func() { // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - // Upload more apps to Gcs for Cluster Manager + // Upload more apps to GCS for Cluster Manager appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload more apps to GCS for Cluster Manager") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload more apps to Gcs for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Deployer") + // Upload more apps to GCS for Deployer + testcaseEnvInst.Log.Info("Upload more apps to GCS for Deployer") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Deployer") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase @@ -2411,27 +2366,27 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster + * Upload big-size app to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ############## VERIFICATIONS ################ * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Gcs during bigger app install + * Upload more apps from GCS during bigger app install * Wait for polling interval to pass * Verify all apps are installed on Cluster Manager and Deployer */ //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source @@ -2459,29 +2414,29 @@ var _ = Describe("c3appfw test", func() { // Verify Monitoring Console is ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload big-size app to Gcs for Cluster Manager + // Upload big-size app to GCS for Cluster Manager appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Cluster Manager") gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload big-size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Search Head Cluster") + // Upload big-size app to GCS for Search Head Cluster + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Search Head Cluster") gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2498,18 +2453,18 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - // Upload more apps to Gcs for Cluster Manager + // Upload more apps to GCS for Cluster Manager appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload more apps to GCS for Cluster Manager") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload more apps to Gcs for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Deployer") + // Upload more apps to GCS for Deployer + testcaseEnvInst.Log.Info("Upload more apps to GCS for Deployer") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Deployer") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase @@ -2547,7 +2502,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * While app install is in progress, restart the operator @@ -2562,28 +2517,28 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2641,7 +2596,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * While app download is in progress, restart the operator @@ -2656,28 +2611,28 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2735,7 +2690,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### VERIFICATIONS ############# @@ -2747,28 +2702,28 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods * Disable the app - * Delete the app from Gcs + * Delete the app from GCS * Check for repo state in App Deployment Info */ //################## SETUP #################### appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2808,7 +2763,7 @@ var _ = Describe("c3appfw test", func() { allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - // Verify repo state on App to be disabled to be 1 (i.e app present on Gcs bucket) + // Verify repo state on App to be disabled to be 1 (i.e app present on GCS bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) @@ -2829,12 +2784,12 @@ var _ = Describe("c3appfw test", func() { // Wait for App state to update after config file change testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - // Delete the file from Gcs + // Delete the file from GCS gcsFilepath := filepath.Join(gcsTestDirIdxc, appFileName[0]) err = testenv.DeleteFileOnGCP(testGcsBucket, gcsFilepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Gcs test directory", appFileName[0])) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCS test directory", appFileName[0])) - // Verify repo state is set to 2 (i.e app deleted from Gcs bucket) + // Verify repo state is set to 2 (i.e app deleted from GCS bucket) testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) }) @@ -2845,7 +2800,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * While app download is completed, upload new versions of the apps @@ -2868,28 +2823,28 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download all apps from Gcs + // Download all apps from GCS appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2907,19 +2862,19 @@ var _ = Describe("c3appfw test", func() { // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - // Upload V2 apps to Gcs for Indexer Cluster + // Upload V2 apps to GCS for Indexer Cluster appVersion = "V2" appListV2 := []string{appListV2[0]} appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V2 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Get Pod age to check for pod resets later @@ -2960,12 +2915,12 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, appframework: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps ################## SETUP #################### - * Upload 15 apps of 100MB size each to Gcs for Indexer Cluster and Search Head Cluster for cluster scope + * Upload 15 apps of 100MB size each to GCS for Indexer Cluster and Search Head Cluster for cluster scope * Create app sources for Cluster Master and Deployer with cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -2989,27 +2944,27 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") - // Upload apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search head Cluster", appVersion)) + // Upload apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3060,7 +3015,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster + * Upload big-size app to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * When app download is complete, delete apps from app directory @@ -3075,28 +3030,28 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download big size apps from Gcs + // Download big size apps from GCS appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload big size app to Gcs for Indexer Cluster + // Upload big size app to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Gcs for Indexer Cluster") + testcaseEnvInst.Log.Info("Upload big size app to GCS for Indexer Cluster") gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to Gcs test directory for Indexer Cluster") + Expect(err).To(Succeed(), "Unable to upload big size to GCS test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload big size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Gcs for Search Head Cluster") + // Upload big size app to GCS for Search Head Cluster + testcaseEnvInst.Log.Info("Upload big size app to GCS for Search Head Cluster") gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload big size to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3158,7 +3113,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP ################## - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Prepare and deploy C3 CRD with app framework * Verify IsDeploymentInProgress is set * Wait for the pods to be ready @@ -3168,21 +3123,21 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3218,7 +3173,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { /* Test Steps @@ -3279,12 +3234,12 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, appframework: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps ################## SETUP #################### - * Upload ES app to Gcs + * Upload ES app to GCS * Upload TA add-on app to location for Indexer cluster * Create App Source with 'ScopeClusterWithPreConfig' scope for C3 SVA * Prepare and deploy C3 CRD with app framework and wait for pods to be ready @@ -3292,43 +3247,43 @@ var _ = Describe("c3appfw test", func() { * Verify ES app is installed on Deployer and on Search Heads * Verify TA add-on app is installed on indexers ################## UPGRADE VERIFICATION ############# - * Update ES app on Gcs location + * Update ES app on GCS location * Verify updated ES app is installed on Deployer and on Search Heads */ //################## SETUP #################### - // Download ES app from Gcs + // Download ES app from GCS appVersion := "V1" - testcaseEnvInst.Log.Info("Download ES app from Gcs") + testcaseEnvInst.Log.Info("Download ES app from GCS") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download ES app file from GCS") - // Download Technology add-on app from Gcs - testcaseEnvInst.Log.Info("Download Technology add-on app from Gcs") + // Download Technology add-on app from GCS + testcaseEnvInst.Log.Info("Download Technology add-on app from GCS") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileListIdxc) - Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileListIdxc) + Expect(err).To(Succeed(), "Unable to download ES app file from GCS") - // Create directory for file upload to Gcs + // Create directory for file upload to GCS gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - // Upload ES app to Gcs - testcaseEnvInst.Log.Info("Upload ES app to Gcs") + // Upload ES app to GCS + testcaseEnvInst.Log.Info("Upload ES app to GCS") uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload ES app to Gcs test directory") + Expect(err).To(Succeed(), "Unable to upload ES app to GCS test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload Technology add-on apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Technology add-on app to Gcs for Indexer Cluster", appVersion)) + // Upload Technology add-on apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Technology add-on app to GCS for Indexer Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileListIdxc, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, gcsTestDirShc, 180) @@ -3339,7 +3294,7 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 180) @@ -3414,16 +3369,16 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") // //############### UPGRADE APPS ################ - // // Download ES App from Gcs + // // Download ES App from GCS // appVersion = "V2" - // testcaseEnvInst.Log.Info("Download updated ES app from Gcs") + // testcaseEnvInst.Log.Info("Download updated ES app from GCS") // err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") - // // Upload V2 ES app to Gcs for Search Head Cluster - // testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Gcs for Search Head Cluster", appVersion)) + // // Upload V2 ES app to GCS for Search Head Cluster + // testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to GCS for Search Head Cluster", appVersion)) // uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - // Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to Gcs test directory for Search Head Cluster", appVersion)) + // Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to GCS test directory for Search Head Cluster", appVersion)) // uploadedApps = append(uploadedApps, uploadedFiles...) // // Check for changes in App phase to determine if next poll has been triggered diff --git a/test/appframework_gcp/m4/appframework_gcs_suite_test.go b/test/appframework_gcp/m4/appframework_gcs_suite_test.go index 8f4a28249..b2c039d50 100644 --- a/test/appframework_gcp/m4/appframework_gcs_suite_test.go +++ b/test/appframework_gcp/m4/appframework_gcs_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataGcsBucket = os.Getenv("TEST_BUCKET") testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) @@ -52,52 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - testenvInstance.Log.Info("logging download details", "bucket", testDataGcsBucket, "gcsAppDirV1", gcsAppDirV1, "downloadDirV1", downloadDirV1, "appFileList", appFileList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupGCPAppsSuite(testSuiteName, testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_gcp/m4/appframework_gcs_test.go b/test/appframework_gcp/m4/appframework_gcs_test.go index 96759b180..b3d7f39b4 100644 --- a/test/appframework_gcp/m4/appframework_gcs_test.go +++ b/test/appframework_gcp/m4/appframework_gcs_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -53,16 +52,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") gcsTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) gcsTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -71,30 +61,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It(" m4gcp, masterappframeworkm4gcp, m4_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps @@ -135,7 +105,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -171,7 +141,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -187,19 +157,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -270,17 +231,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -313,7 +265,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps @@ -354,7 +306,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -390,7 +342,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -403,19 +355,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -472,17 +415,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -512,7 +446,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps @@ -567,7 +501,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -582,17 +516,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -634,9 +559,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -654,8 +577,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -724,9 +646,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -742,8 +662,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -791,7 +710,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -833,7 +752,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -848,14 +767,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -897,14 +810,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -924,7 +831,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -968,7 +875,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) @@ -1004,7 +911,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) @@ -1015,19 +922,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1079,17 +977,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1182,7 +1071,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1228,7 +1117,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) @@ -1242,14 +1131,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1291,14 +1174,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" @@ -1365,7 +1242,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, masterappframeworkm4, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1431,7 +1308,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1441,7 +1318,7 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} @@ -1453,7 +1330,7 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} @@ -1473,17 +1350,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1553,17 +1421,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1651,7 +1510,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Master @@ -1670,7 +1529,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1683,7 +1542,7 @@ var _ = Describe("m4appfw test", func() { siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1788,7 +1647,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Master @@ -1807,7 +1666,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1821,7 +1680,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1870,7 +1729,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { /* Test Steps @@ -1892,7 +1751,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -1908,7 +1767,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -1920,7 +1779,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) @@ -1928,17 +1787,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1962,7 +1812,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { /* Test Steps @@ -1984,7 +1834,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -2000,7 +1850,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -2012,7 +1862,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -2020,17 +1870,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2054,7 +1895,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { /* Test Steps @@ -2090,7 +1931,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -2102,19 +1943,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2166,7 +1998,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2208,7 +2040,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) @@ -2222,14 +2054,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2256,8 +2082,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Master") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2267,8 +2092,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2351,7 +2175,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { /* Test Steps @@ -2383,7 +2207,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to GCP for Indexer Cluster @@ -2398,7 +2222,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) @@ -2410,7 +2234,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2470,7 +2294,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, masterappframeworkm4, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2499,7 +2323,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to GCP for Indexer Cluster @@ -2510,7 +2334,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) gcsTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) @@ -2519,7 +2343,7 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2537,14 +2361,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2562,7 +2380,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { /* Test Steps @@ -2584,7 +2402,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from GCP appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to GCP for Indexer Cluster @@ -2600,7 +2418,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -2612,7 +2430,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2623,17 +2441,8 @@ var _ = Describe("m4appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2657,7 +2466,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It(" m4gcp, masterappframeworkm4gcp, m4_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { /* Test Steps @@ -2684,7 +2493,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -2696,7 +2505,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") @@ -2709,14 +2518,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_gcp/m4/manager_appframework_test.go b/test/appframework_gcp/m4/manager_appframework_test.go index bc6837431..36d3f0f89 100644 --- a/test/appframework_gcp/m4/manager_appframework_test.go +++ b/test/appframework_gcp/m4/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -52,16 +51,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -70,30 +60,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It(" m4gcp, m4_mgr_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps @@ -134,7 +104,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -170,7 +140,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -186,7 +156,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -312,7 +282,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps @@ -353,7 +323,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -389,7 +359,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -402,7 +372,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -511,7 +481,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps @@ -566,7 +536,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -633,9 +603,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -653,8 +621,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -723,9 +690,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -741,8 +706,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -790,7 +754,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -832,7 +796,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -923,7 +887,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -967,7 +931,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) @@ -1003,7 +967,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -1014,7 +978,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -1181,7 +1145,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1227,7 +1191,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -1364,7 +1328,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1430,7 +1394,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1440,7 +1404,7 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultGCPRegion())} @@ -1452,7 +1416,7 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultGCPRegion())} @@ -1650,7 +1614,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Manager @@ -1669,7 +1633,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1682,7 +1646,7 @@ var _ = Describe("m4appfw test", func() { siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1786,7 +1750,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Manager @@ -1805,7 +1769,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1819,7 +1783,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1868,7 +1832,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { /* Test Steps @@ -1890,7 +1854,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -1906,7 +1870,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -1918,7 +1882,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) @@ -1960,7 +1924,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { /* Test Steps @@ -1982,7 +1946,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -1998,7 +1962,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2010,7 +1974,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -2052,7 +2016,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { /* Test Steps @@ -2088,7 +2052,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2100,7 +2064,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2164,7 +2128,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2206,7 +2170,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -2254,8 +2218,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Manager CR with latest config cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Manager") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Manager") // Set AppsRepoPollInterval for Cluster Manager to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Manager to 180 seconds") @@ -2265,8 +2228,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2349,7 +2311,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { /* Test Steps @@ -2381,7 +2343,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to GCP for Indexer Cluster @@ -2396,7 +2358,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) @@ -2408,7 +2370,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2468,7 +2430,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2497,7 +2459,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to GCP for Indexer Cluster @@ -2508,7 +2470,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) @@ -2517,7 +2479,7 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2560,7 +2522,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { /* Test Steps @@ -2582,7 +2544,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from GCP appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to GCP for Indexer Cluster @@ -2598,7 +2560,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2610,7 +2572,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2655,7 +2617,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It(" m4gcp, m4_mgr_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { /* Test Steps @@ -2682,7 +2644,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2694,7 +2656,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") diff --git a/test/appframework_gcp/s1/appframework_gcs_suite_test.go b/test/appframework_gcp/s1/appframework_gcs_suite_test.go index af2fab4c2..90a237aa0 100644 --- a/test/appframework_gcp/s1/appframework_gcs_suite_test.go +++ b/test/appframework_gcp/s1/appframework_gcs_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataGcsBucket = os.Getenv("TEST_BUCKET") testGCSBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) @@ -52,47 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupGCPAppsSuite(testSuiteName, testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_gcp/s1/appframework_gcs_test.go b/test/appframework_gcp/s1/appframework_gcs_test.go index edfd8019f..bd5346e7d 100644 --- a/test/appframework_gcp/s1/appframework_gcs_test.go +++ b/test/appframework_gcp/s1/appframework_gcs_test.go @@ -23,7 +23,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -46,43 +45,14 @@ var _ = Describe("s1appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") gcsTestDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCS - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGCSBucket, uploadedApps), filePresentOnOperator) }) Context("Standalone deployment (S1) with App Framework", func() { @@ -130,7 +100,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -164,7 +134,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -185,7 +155,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -310,7 +280,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -334,7 +304,7 @@ var _ = Describe("s1appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -354,7 +324,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -481,7 +451,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -511,7 +481,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to gcs test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -531,7 +501,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -566,8 +536,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -599,8 +568,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale down Standalone") scaledReplicaCount = 1 standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone after scaling down") standalone.Spec.Replicas = int32(scaledReplicaCount) err = deployment.UpdateCR(ctx, standalone) @@ -669,7 +637,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to gcs test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -686,7 +654,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -706,8 +674,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -773,7 +740,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from gcs") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload ES app to gcs @@ -782,7 +749,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload ES app to gcs test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, gcsTestDir, 60) appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ @@ -805,7 +772,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone with App Framework") // Ensure Standalone goes to Ready phase testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -829,7 +796,7 @@ var _ = Describe("s1appfw test", func() { // Download ES App from gcs testcaseEnvInst.Log.Info("Download updated ES app from gcs") - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV2, downloadDirV2, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload V2 apps to gcs for Standalone @@ -884,7 +851,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from gcs testcaseEnvInst.Log.Info("Download bigger amount of apps from gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -894,7 +861,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -974,7 +941,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) @@ -1003,7 +970,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 0) @@ -1023,7 +990,7 @@ var _ = Describe("s1appfw test", func() { // Create Standalone Deployment with App Framework standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1136,7 +1103,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from GCS testcaseEnvInst.Log.Info("Download the extra apps from GCS for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to GCS for first Standalone @@ -1154,7 +1121,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1168,7 +1135,7 @@ var _ = Describe("s1appfw test", func() { AppFrameworkConfig: appFrameworkSpec, } - // Create App framework Spec + // Create App Framework Spec appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, gcsTestDirStandalone2, 60) @@ -1238,7 +1205,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -1266,7 +1233,7 @@ var _ = Describe("s1appfw test", func() { // Download all test apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCS for Standalone @@ -1277,7 +1244,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1297,7 +1264,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1345,7 +1312,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCS for Standalone @@ -1354,7 +1321,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1371,7 +1338,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -1435,7 +1402,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCS for Standalone @@ -1444,7 +1411,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1461,7 +1428,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -1517,7 +1484,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1535,7 +1502,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1597,7 +1564,7 @@ var _ = Describe("s1appfw test", func() { * Verify no pod resets triggered due to app install * Verify App enabled and version by running splunk cmd // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec + * Create App Framework volume with random credentials and apply to Spec * Check for changes in App phase to determine if next poll has been triggered ############ UPGRADE V2 APPS ########### * Upload V2 apps to GCS App Source @@ -1628,7 +1595,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1647,7 +1614,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) Expect(err).To(Succeed(), "Unable to obtain secret object") @@ -1667,7 +1634,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec + // Create App Framework volume with invalid credentials and apply to Spec testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) Expect(err).To(Succeed(), "Unable to update secret Object") @@ -1740,7 +1707,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload apps to GCS for Standalone @@ -1749,7 +1716,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 120) spec := enterpriseApi.StandaloneSpec{ @@ -1766,7 +1733,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -1832,7 +1799,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to GCS @@ -1844,7 +1811,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 15 - // Create App framework Spec + // Create App Framework Spec appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1899,7 +1866,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big app") // Upload big-size app to GCS for Standalone @@ -1908,7 +1875,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1925,7 +1892,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App Download is completed on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -1979,7 +1946,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -2017,7 +1984,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -2038,7 +2005,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Standalone CR testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind) diff --git a/test/custom_resource_crud/crud_test_shared.go b/test/custom_resource_crud/crud_test_shared.go new file mode 100644 index 000000000..50e1d1248 --- /dev/null +++ b/test/custom_resource_crud/crud_test_shared.go @@ -0,0 +1,290 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package crcrud + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// RunS1CPUUpdateTest runs the standard S1 CPU limit update test workflow +func RunS1CPUUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, defaultCPULimits string, newCPULimits string) { + // Deploy and verify Standalone + standalone := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, deployment.GetName(), deployment.GetName(), "") + + // Verify telemetry + prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) + testcaseEnvInst.TriggerAndVerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) + + // Deploy and verify Monitoring Console + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + + // Verify CPU limits before updating the CR + standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, defaultCPULimits) + + // Change CPU limits to trigger CR update + standalone.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + err := deployment.UpdateCR(ctx, standalone) + Expect(err).To(Succeed(), "Unable to deploy standalone instance with updated CR ") + + // Verify Standalone is updating + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) + + // Verify Standalone goes to ready state + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Verify CPU limits after updating the CR + testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, newCPULimits) +} + +// RunC3CPUUpdateTest runs the standard C3 CPU limit update test workflow +func RunC3CPUUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, defaultCPULimits string, newCPULimits string) { + // Deploy Single site Cluster and Search Head Clusters + mcRef := deployment.GetName() + prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) + err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + // Verify cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, func(ctx context.Context, d *testenv.Deployment) { + config.ClusterManagerReady(ctx, d, testcaseEnvInst) + }) + + // Verify telemetry + testcaseEnvInst.TriggerAndVerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) + + // Deploy and verify Monitoring Console, RF/SF + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + testcaseEnvInst.StandardC3Verification(ctx, deployment, deployment.GetName(), mc) + + // Verify CPU limits on Indexers before updating the CR + indexerCount := 3 + testcaseEnvInst.VerifyIndexerCPULimits(deployment, deployment.GetName(), indexerCount, defaultCPULimits) + + // Change CPU limits to trigger CR update + idxc := &enterpriseApi.IndexerCluster{} + instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) + testenv.GetInstanceWithExpect(ctx, deployment, idxc, instanceName, "Unable to get instance of Indexer Cluster") + idxc.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + testenv.UpdateCRWithExpect(ctx, deployment, idxc, "Unable to deploy Indexer Cluster with updated CR") + + // Verify Indexer Cluster is updating + idxcName := deployment.GetName() + "-idxc" + testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) + + // Verify Indexers go to ready state + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + + // Verify CPU limits on Indexers after updating the CR + testcaseEnvInst.VerifyIndexerCPULimits(deployment, deployment.GetName(), indexerCount, newCPULimits) + + // Verify CPU limits on Search Heads before updating the CR + searchHeadCount := 3 + testcaseEnvInst.VerifySearchHeadCPULimits(deployment, deployment.GetName(), searchHeadCount, defaultCPULimits) + + // Change CPU limits to trigger CR update + shc := &enterpriseApi.SearchHeadCluster{} + instanceName = fmt.Sprintf("%s-shc", deployment.GetName()) + testenv.GetInstanceWithExpect(ctx, deployment, shc, instanceName, "Unable to fetch Search Head Cluster deployment") + + shc.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + testenv.UpdateCRWithExpect(ctx, deployment, shc, "Unable to deploy Search Head Cluster with updated CR") + + // Verify Search Head Cluster is updating + testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) + + // Verify Search Head go to ready state + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Verify CPU limits on Search Heads after updating the CR + testcaseEnvInst.VerifySearchHeadCPULimits(deployment, deployment.GetName(), searchHeadCount, newCPULimits) +} + +// RunC3PVCDeletionTest runs the standard C3 PVC deletion test workflow +func RunC3PVCDeletionTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, verificationTimeout time.Duration) { + // Deploy Single site Cluster and Search Head Clusters + mcRef := deployment.GetName() + err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + // Verify cluster is ready and RF/SF is met + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, func(ctx context.Context, d *testenv.Deployment) { + config.ClusterManagerReady(ctx, d, testcaseEnvInst) + }) + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + // Deploy and verify Monitoring Console + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, "") + + clusterManagerType := config.ClusterManagerPVCType() + verifyC3ClusterPVCs(testcaseEnvInst, deployment, clusterManagerType, true, verificationTimeout) + + // Delete the Search Head Cluster + shc := &enterpriseApi.SearchHeadCluster{} + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Unable to GET SHC instance") + err = deployment.DeleteCR(ctx, shc) + Expect(err).To(Succeed(), "Unable to delete SHC instance", "SHC Name", shc) + + // Delete the Indexer Cluster + idxc := &enterpriseApi.IndexerCluster{} + testenv.GetInstanceWithExpect(ctx, deployment, idxc, deployment.GetName()+"-idxc", "Unable to GET IDXC instance") + err = deployment.DeleteCR(ctx, idxc) + Expect(err).To(Succeed(), "Unable to delete IDXC instance", "IDXC Name", idxc) + + // Delete the Cluster Manager (v3 or v4) + config.DeleteClusterManager(ctx, deployment) + + // Delete Monitoring Console + testenv.GetInstanceWithExpect(ctx, deployment, mc, mcRef, "Unable to GET Monitoring Console instance") + err = deployment.DeleteCR(ctx, mc) + Expect(err).To(Succeed(), "Unable to delete Monitoring Console instance", "Monitoring Console Name", mcRef) + + verifyC3ClusterPVCs(testcaseEnvInst, deployment, clusterManagerType, false, verificationTimeout) + + // Verify Monitoring Console PVCs (etc and var) have been deleted + testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "monitoring-console", 1, false, verificationTimeout) +} + +func verifyC3ClusterPVCs(testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment, clusterManagerType string, exists bool, timeout time.Duration) { + testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, exists, timeout) + testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, exists, timeout) + testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, exists, timeout) + testcaseEnvInst.VerifyPVCsPerDeployment(deployment, clusterManagerType, 1, exists, timeout) +} + +// RunSHCDeployerResourceSpecTest deploys a Search Head Cluster, verifies default CPU limits, +// updates the deployer resource spec, and verifies the deployer is reconfigured while search heads retain defaults. +func RunSHCDeployerResourceSpecTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, defaultCPULimits string) { + shcName := fmt.Sprintf("%s-shc", deployment.GetName()) + _, err := deployment.DeploySearchHeadCluster(ctx, shcName, "", "", "", "") + if err != nil { + Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster", "shc", shcName) + } + + // Verify CPU limits on Search Heads and deployer before updating CR + searchHeadCount := 3 + testcaseEnvInst.VerifySearchHeadCPULimits(deployment, deployment.GetName(), searchHeadCount, defaultCPULimits) + + DeployerPodName := fmt.Sprintf(testenv.DeployerPod, deployment.GetName()) + testcaseEnvInst.VerifyCPULimits(deployment, DeployerPodName, defaultCPULimits) + + shc := &enterpriseApi.SearchHeadCluster{} + testenv.GetInstanceWithExpect(ctx, deployment, shc, shcName, "Unable to fetch Search Head Cluster deployment") + + // Assign new resources for deployer pod only + newCPULimits := "4" + newCPURequests := "2" + newMemoryLimits := "14Gi" + newMemoryRequests := "12Gi" + + depResSpec := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "cpu": resource.MustParse(newCPURequests), + "memory": resource.MustParse(newMemoryRequests), + }, + Limits: corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + "memory": resource.MustParse(newMemoryLimits), + }, + } + shc.Spec.DeployerResourceSpec = depResSpec + + testenv.UpdateCRWithExpect(ctx, deployment, shc, "Unable to deploy Search Head Cluster with updated CR") + + // Verify Search Head go to ready state + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + // Verify CPU limits on Search Heads - Should be same as before + testcaseEnvInst.VerifySearchHeadCPULimits(deployment, deployment.GetName(), searchHeadCount, defaultCPULimits) + + // Verify modified deployer spec + testcaseEnvInst.VerifyResourceConstraints(deployment, DeployerPodName, depResSpec) +} + +// RunM4CPUUpdateTest runs the standard M4 CPU limit update test workflow +func RunM4CPUUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, defaultCPULimits string, newCPULimits string) { + // Deploy Multisite Cluster and Search Head Clusters + mcRef := deployment.GetName() + prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) + siteCount := 3 + var err error + + err = config.DeployMultisiteCluster(ctx, deployment, deployment.GetName(), 1, siteCount, mcRef) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + // Ensure that the cluster-manager goes to Ready phase + config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) + + testcaseEnvInst.TriggerAndVerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) + + // Deploy and verify Monitoring Console + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, "") + + // Verify RF SF is met + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + // Verify CPU limits on Indexers before updating the CR + testcaseEnvInst.VerifyCPULimitsOnAllSites(deployment, deployment.GetName(), siteCount, defaultCPULimits) + + // Change CPU limits to trigger CR update + idxc := &enterpriseApi.IndexerCluster{} + for i := 1; i <= siteCount; i++ { + siteName := fmt.Sprintf("site%d", i) + instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) + testenv.GetInstanceWithExpect(ctx, deployment, idxc, instanceName, "Unable to fetch Indexer Cluster deployment") + idxc.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + testenv.UpdateCRWithExpect(ctx, deployment, idxc, "Unable to deploy Indexer Cluster with updated CR") + } + + // Verify Indexer Cluster is updating + idxcName := deployment.GetName() + "-" + "site1" + testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) + + // Verify Indexers go to ready state + testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Verify RF SF is met + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + // Verify CPU limits after updating the CR + testcaseEnvInst.VerifyCPULimitsOnAllSites(deployment, deployment.GetName(), siteCount, newCPULimits) +} diff --git a/test/custom_resource_crud/custom_resource_crud_c3_test.go b/test/custom_resource_crud/custom_resource_crud_c3_test.go deleted file mode 100644 index 8c47e63b2..000000000 --- a/test/custom_resource_crud/custom_resource_crud_c3_test.go +++ /dev/null @@ -1,253 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var verificationTimeout time.Duration - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - verificationTimeout = 150 * time.Second - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastercrcrud, integration, c3: can deploy indexer and search head cluster, change their CR, update the instances", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify telemetry - testcaseEnvInst.TriggerTelemetrySubmission(ctx, deployment) - testcaseEnvInst.VerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - indexerCount := 3 - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to get instance of indexer cluster") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-idxc" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify CPU limits on Indexers after updating the CR - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, newCPULimits) - } - - // Verify CPU limits on Search Heads before updating the CR - searchHeadCount := 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - shc := &enterpriseApi.SearchHeadCluster{} - instanceName = fmt.Sprintf("%s-shc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, shc) - Expect(err).To(Succeed(), "Unable to fetch Search Head Cluster deployment") - - shc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster with updated CR") - - // Verify Search Head Cluster is updating - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits on Search Heads after updating the CR - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, newCPULimits) - } - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastercrcrud, integration, c3: can verify IDXC, CM and SHC PVCs are correctly deleted after the CRs deletion", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Search Heads PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, true, verificationTimeout) - - // Verify Deployer PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, true, verificationTimeout) - - // Verify Indexers PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, true, verificationTimeout) - - // Verify Cluster Master PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, splcommon.ClusterManager, 1, true, verificationTimeout) - - // Delete the Search Head Cluster - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Unable to GET SHC instance", "SHC Name", shc) - err = deployment.DeleteCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to delete SHC instance", "SHC Name", shc) - - // Delete the Indexer Cluster - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) - Expect(err).To(Succeed(), "Unable to GET IDXC instance", "IDXC Name", idxc) - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to delete IDXC instance", "IDXC Name", idxc) - - // Delete the Cluster Master - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Unable to GET Cluster Manager instance", "Cluster Manager Name", cm) - err = deployment.DeleteCR(ctx, cm) - Expect(err).To(Succeed(), "Unable to delete Cluster Manager instance", "Cluster Manger Name", cm) - - // Delete Monitoring Console - err = deployment.GetInstance(ctx, mcRef, mc) - Expect(err).To(Succeed(), "Unable to GET Monitoring Console instance", "Monitoring Console Name", mcRef) - err = deployment.DeleteCR(ctx, mc) - Expect(err).To(Succeed(), "Unable to delete Monitoring Console instance", "Monitoring Console Name", mcRef) - - // Verify Search Heads PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, false, verificationTimeout) - - // Verify Deployer PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, false, verificationTimeout) - - // Verify Indexers PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, false, verificationTimeout) - - // Verify Cluster Master PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, splcommon.ClusterManager, 1, false, verificationTimeout) - - // Verify Monitoring Console PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "monitoring-console", 1, false, verificationTimeout) - }) - }) -}) diff --git a/test/custom_resource_crud/custom_resource_crud_m4_test.go b/test/custom_resource_crud/custom_resource_crud_m4_test.go deleted file mode 100644 index 4c00a07dc..000000000 --- a/test/custom_resource_crud/custom_resource_crud_m4_test.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA M4", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var ctx context.Context - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - ctx = context.TODO() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("mastercrcrud, integration, m4: can deploy can deploy multisite indexer and search head clusters, change their CR, update the instances", func() { - - // Deploy Multisite Cluster and Search Head Clusters - mcRef := deployment.GetName() - prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) - siteCount := 3 - err := deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify telemetry - testcaseEnvInst.TriggerTelemetrySubmission(ctx, deployment) - testcaseEnvInst.VerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - for i := 1; i <= siteCount; i++ { - siteName := fmt.Sprintf("site%d", i) - instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to fetch Indexer Cluster deployment") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - } - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-" + "site1" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits after updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, newCPULimits) - } - }) - }) -}) diff --git a/test/custom_resource_crud/custom_resource_crud_s1_test.go b/test/custom_resource_crud/custom_resource_crud_s1_test.go deleted file mode 100644 index deb487628..000000000 --- a/test/custom_resource_crud/custom_resource_crud_s1_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA S1", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var ctx context.Context - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - ctx = context.TODO() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1)", func() { - It("managercrcrud, integration, s1: can deploy a standalone instance, change its CR, update the instance", func() { - - // Deploy Standalone - mcRef := deployment.GetName() - prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Verify Standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify telemetry - testcaseEnvInst.TriggerTelemetrySubmission(ctx, deployment) - testcaseEnvInst.VerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits before updating the CR - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, defaultCPULimits) - - // Change CPU limits to trigger CR update - standalone.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with updated CR ") - - // Verify Standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify Standalone goes to ready state - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits after updating the CR - testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, newCPULimits) - }) - }) -}) diff --git a/test/custom_resource_crud/custom_resource_crud_suite_test.go b/test/custom_resource_crud/custom_resource_crud_suite_test.go index 8972ac76e..ce753a736 100644 --- a/test/custom_resource_crud/custom_resource_crud_suite_test.go +++ b/test/custom_resource_crud/custom_resource_crud_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,12 +24,12 @@ import ( ) const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond + // DefaultCPULimits is the default CPU limit + DefaultCPULimits = "4" + // UpdatedCPULimits is the updated CPU limit + UpdatedCPULimits = "2" + // DefaultVerificationTimeout is the default timeout for CRUD verification steps + DefaultVerificationTimeout = 150 * time.Second ) var ( @@ -39,7 +39,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/custom_resource_crud/custom_resource_crud_test.go b/test/custom_resource_crud/custom_resource_crud_test.go new file mode 100644 index 000000000..f01200b58 --- /dev/null +++ b/test/custom_resource_crud/custom_resource_crud_test.go @@ -0,0 +1,132 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package crcrud + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + "github.com/splunk/splunk-operator/test/testenv" +) + +var _ = Describe("Custom Resource CRUD test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + var defaultCPULimits string + var newCPULimits string + var verificationTimeout time.Duration + + ctx := context.TODO() + + // S1 test — single variant (manager, V4) + Context("Standalone deployment (S1)", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") + defaultCPULimits = DefaultCPULimits + newCPULimits = UpdatedCPULimits + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It("managercrcrud, integration, s1: can deploy a standalone instance, change its CR, update the instance", func() { + RunS1CPUUpdateTest(ctx, deployment, testcaseEnvInst, defaultCPULimits, newCPULimits) + }) + }) + + // C3 tests — V3 (master) and V4 (manager) variants + c3CrudConfigs := []struct { + namePrefix string + label string + newConfig func() *testenv.ClusterReadinessConfig + }{ + {"master", "mastercrcrud", testenv.NewClusterReadinessConfigV3}, + {"", "managercrcrud", testenv.NewClusterReadinessConfigV4}, + } + + for _, tc := range c3CrudConfigs { + tc := tc + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { + BeforeEach(func() { + defaultCPULimits = DefaultCPULimits + newCPULimits = UpdatedCPULimits + verificationTimeout = DefaultVerificationTimeout + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.namePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(tc.label+", integration, c3: can deploy Indexer and Search Head Cluster, change their CR, update the instances", func() { + config := tc.newConfig() + RunC3CPUUpdateTest(ctx, deployment, testcaseEnvInst, config, defaultCPULimits, newCPULimits) + }) + + It(tc.label+", integration, c3: can verify IDXC, CM and SHC PVCs are correctly deleted after the CRs deletion", func() { + config := tc.newConfig() + RunC3PVCDeletionTest(ctx, deployment, testcaseEnvInst, config, verificationTimeout) + }) + }) + } + + // CSPL-3256 - SHC deployer resource spec test (IDXC is irrelevant for this test case) + Context("Search Head Cluster", func() { + BeforeEach(func() { + defaultCPULimits = DefaultCPULimits + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It("managercrcrud, integration, shc: can deploy Search Head Cluster with Deployer resource spec configured", func() { + RunSHCDeployerResourceSpecTest(ctx, deployment, testcaseEnvInst, defaultCPULimits) + }) + }) + + // M4 tests — V3 (master) and V4 (manager) variants + m4CrudConfigs := []struct { + namePrefix string + label string + newConfig func() *testenv.ClusterReadinessConfig + }{ + {"master", "mastercrcrud", testenv.NewClusterReadinessConfigV3}, + {"", "managercrcrud", testenv.NewClusterReadinessConfigV4}, + } + + for _, tc := range m4CrudConfigs { + tc := tc + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster)", func() { + BeforeEach(func() { + defaultCPULimits = DefaultCPULimits + newCPULimits = UpdatedCPULimits + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.namePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(tc.label+", integration, m4: can deploy multisite Indexer and Search Head Clusters, change their CR, update the instances", func() { + config := tc.newConfig() + RunM4CPUUpdateTest(ctx, deployment, testcaseEnvInst, config, defaultCPULimits, newCPULimits) + }) + }) + } +}) diff --git a/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go b/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go deleted file mode 100644 index 86bcea539..000000000 --- a/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go +++ /dev/null @@ -1,308 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - //splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var verificationTimeout time.Duration - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - verificationTimeout = 150 * time.Second - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managercrcrud, integration, c3: can deploy indexer and search head cluster, change their CR, update the instances", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - indexerCount := 3 - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to get instance of indexer cluster") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-idxc" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify CPU limits on Indexers after updating the CR - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, newCPULimits) - } - - // Verify CPU limits on Search Heads before updating the CR - searchHeadCount := 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - shc := &enterpriseApi.SearchHeadCluster{} - instanceName = fmt.Sprintf("%s-shc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, shc) - Expect(err).To(Succeed(), "Unable to fetch Search Head Cluster deployment") - - shc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster with updated CR") - - // Verify Search Head Cluster is updating - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits on Search Heads after updating the CR - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, newCPULimits) - } - }) - }) - - Context("Search Head Cluster", func() { - // CSPL-3256 - Adding the SHC only test case under c3 as IDXC is irrelevant for this test case - It("managercrcrud, integration, shc: can deploy Search Head Cluster with Deployer resource spec configured", func() { - shcName := fmt.Sprintf("%s-shc", deployment.GetName()) - _, err := deployment.DeploySearchHeadCluster(ctx, shcName, "", "", "", "") - if err != nil { - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster", "Shc", shcName) - } - - // Verify CPU limits on Search Heads and deployer before updating CR - searchHeadCount := 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - DeployerPodName := fmt.Sprintf(testenv.DeployerPod, deployment.GetName()) - testcaseEnvInst.VerifyCPULimits(deployment, DeployerPodName, defaultCPULimits) - - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Unable to fetch Search Head Cluster deployment") - - // Assign new resources for deployer pod only - newCPULimits = "4" - newCPURequests := "2" - newMemoryLimits := "14Gi" - newMemoryRequests := "12Gi" - - depResSpec := corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - "cpu": resource.MustParse(newCPURequests), - "memory": resource.MustParse(newMemoryRequests), - }, - Limits: corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - "memory": resource.MustParse(newMemoryLimits), - }, - } - shc.Spec.DeployerResourceSpec = depResSpec - - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster with updated CR") - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify CPU limits on Search Heads - Should be same as before - searchHeadCount = 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - // Verify modified deployer spec - testcaseEnvInst.VerifyResourceConstraints(deployment, DeployerPodName, depResSpec) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managercrcrud, integration, c3: can verify IDXC, CM and SHC PVCs are correctly deleted after the CRs deletion", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Search Heads PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, true, verificationTimeout) - - // Verify Deployer PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, true, verificationTimeout) - - // Verify Indexers PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, true, verificationTimeout) - - // Verify Cluster Manager PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "cluster-manager", 1, true, verificationTimeout) - - // Delete the Search Head Cluster - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Unable to GET SHC instance", "SHC Name", shc) - err = deployment.DeleteCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to delete SHC instance", "SHC Name", shc) - - // Delete the Indexer Cluster - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) - Expect(err).To(Succeed(), "Unable to GET IDXC instance", "IDXC Name", idxc) - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to delete IDXC instance", "IDXC Name", idxc) - - // Delete the Cluster Manager - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Unable to GET Cluster Manager instance", "Cluster Manager Name", cm) - err = deployment.DeleteCR(ctx, cm) - Expect(err).To(Succeed(), "Unable to delete Cluster Manager instance", "Cluster Manger Name", cm) - - // Delete Monitoring Console - err = deployment.GetInstance(ctx, mcRef, mc) - Expect(err).To(Succeed(), "Unable to GET Monitoring Console instance", "Monitoring Console Name", mcRef) - err = deployment.DeleteCR(ctx, mc) - Expect(err).To(Succeed(), "Unable to delete Monitoring Console instance", "Monitoring Console Name", mcRef) - - // Verify Search Heads PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, false, verificationTimeout) - - // Verify Deployer PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, false, verificationTimeout) - - // Verify Indexers PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, false, verificationTimeout) - - // Verify Cluster Manager PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "cluster-manager", 1, false, verificationTimeout) - - // Verify Monitoring Console PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "monitoring-console", 1, false, verificationTimeout) - }) - }) -}) diff --git a/test/custom_resource_crud/manager_custom_resource_crud_m4_test.go b/test/custom_resource_crud/manager_custom_resource_crud_m4_test.go deleted file mode 100644 index 89fadcbb8..000000000 --- a/test/custom_resource_crud/manager_custom_resource_crud_m4_test.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA M4", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var ctx context.Context - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - ctx = context.TODO() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("managercrcrud, integration, m4: can deploy can deploy multisite indexer and search head clusters, change their CR, update the instances", func() { - - // Deploy Multisite Cluster and Search Head Clusters - mcRef := deployment.GetName() - siteCount := 3 - err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - for i := 1; i <= siteCount; i++ { - siteName := fmt.Sprintf("site%d", i) - instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to fetch Indexer Cluster deployment") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - } - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-" + "site1" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits after updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, newCPULimits) - } - }) - }) -}) diff --git a/test/delete_cr/deletecr_suite_test.go b/test/delete_cr/deletecr_suite_test.go index 49b34af39..83d4a3c11 100644 --- a/test/delete_cr/deletecr_suite_test.go +++ b/test/delete_cr/deletecr_suite_test.go @@ -1,23 +1,20 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package deletecr import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "deletecr-" + testenv.RandomDNSName(3) @@ -41,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/delete_cr/deletecr_test.go b/test/delete_cr/deletecr_test.go index 2f3667ba1..167d4ea24 100644 --- a/test/delete_cr/deletecr_test.go +++ b/test/delete_cr/deletecr_test.go @@ -1,138 +1,49 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package deletecr import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" + "github.com/splunk/splunk-operator/test/testenv" ) -var _ = Describe("DeleteCR test", func() { +var _ = Describe("Delete CR test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) - Context("Standalone deployment (S1 - Standalone Pod)", func() { + Context("Standalone deployment (S1)", func() { It("integration, managerdeletecr: can deploy standalone and delete", func() { - - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Delete Standalone CR - err = deployment.DeleteCR(ctx, standalone) - Expect(err).To(Succeed(), "Unable to Delete Standalone") - + testenv.RunDeleteStandaloneWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName()) }) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3)", func() { It("integration, managerdeletecr: can deploy C3 and delete search head, clustermanager", func() { - - // Deploy C3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, true, "") - Expect(err).To(Succeed(), "Unable to deploy C3 instance") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - idxc := &enterpriseApi.IndexerCluster{} - idxcName := deployment.GetName() + "-idxc" - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Unable to get Indexer instance") - - // Delete Indexer Cluster CR - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to Delete Indexer Cluster") - - sh := &enterpriseApi.SearchHeadCluster{} - shcName := deployment.GetName() + "-shc" - err = deployment.GetInstance(ctx, shcName, sh) - Expect(err).To(Succeed(), "Unable to get Search Head instance") - - // Delete Search Head Cluster CR - err = deployment.DeleteCR(ctx, sh) - Expect(err).To(Succeed(), "Unable to Delete Search Head Cluster") - - cm := &enterpriseApi.ClusterManager{} - cmName := deployment.GetName() - err = deployment.GetInstance(ctx, cmName, cm) - Expect(err).To(Succeed(), "Unable to get Cluster Manager instance") - - // Delete Cluster Manager CR - err = deployment.DeleteCR(ctx, cm) - Expect(err).To(Succeed(), "Unable to Delete Cluster Manager") - + testenv.RunDeleteC3Workflow(ctx, deployment, testcaseEnvInst, deployment.GetName(), 3) }) }) }) diff --git a/test/example/example1_test.go b/test/example/example1_test.go index bba0023d1..37e214084 100644 --- a/test/example/example1_test.go +++ b/test/example/example1_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,12 +14,10 @@ package example import ( - "fmt" "math/rand" "time" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" ) @@ -31,20 +29,11 @@ var _ = XDescribe("Example1", func() { // This is invoke for each "It" spec below BeforeEach(func() { - var err error - // Create a deployment for this test - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - deployment.Teardown() - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) // "It" spec diff --git a/test/example/example2_test.go b/test/example/example2_test.go index 3988e0976..9ca6b9959 100644 --- a/test/example/example2_test.go +++ b/test/example/example2_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,12 +14,10 @@ package example import ( - "fmt" "math/rand" "time" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" ) @@ -31,20 +29,11 @@ var _ = XDescribe("Example2", func() { // This is invoke for each "It" spec below BeforeEach(func() { - var err error - // Create a deployment for this test - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - deployment.Teardown() - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) // "It" spec diff --git a/test/example/example_suite_test.go b/test/example/example_suite_test.go index 78f198f31..2e5710526 100644 --- a/test/example/example_suite_test.go +++ b/test/example/example_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,9 +14,7 @@ package example import ( - "math/rand" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -29,12 +27,7 @@ var ( testSuiteName = "example-" + testenv.RandomDNSName(3) ) -func init() { - rand.Seed(time.Now().UnixNano()) -} - func TestExampleSuite(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) @@ -47,5 +40,7 @@ var _ = BeforeSuite(func() { }) var _ = AfterSuite(func() { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + if testenvInstance != nil { + Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + } }) diff --git a/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go b/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go index d2c4be9f1..3a57cf228 100644 --- a/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go +++ b/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go @@ -11,13 +11,12 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package indingsep +package indexingestionsep import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,18 +25,9 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv - testSuiteName = "indingsep-" + testenv.RandomDNSName(3) + testSuiteName = "indexingestionsep-" + testenv.RandomDNSName(3) queue = enterpriseApi.QueueSpec{ Provider: "sqs", @@ -100,7 +90,6 @@ var ( appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) s3TestDir = "icappfw-" + testenv.RandomDNSName(4) appListV1 = testenv.BasicApps - s3AppDirV1 = testenv.AppLocationV1 ) // TestBasic is the main entry point @@ -119,7 +108,7 @@ var _ = BeforeSuite(func() { appFileList := testenv.GetAppFileList(appListV1) // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download V1 app files") }) diff --git a/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go b/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go index 9142c9882..db2169a99 100644 --- a/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go +++ b/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package indingsep +package indexingestionsep import ( "context" @@ -21,38 +21,23 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/onsi/ginkgo/types" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" "github.com/splunk/splunk-operator/test/testenv" ) -var _ = Describe("indingsep test", func() { +var _ = Describe("Index and Ingestion Separation test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment - var cmSpec enterpriseApi.ClusterManagerSpec - ctx := context.TODO() BeforeEach(func() { - var err error - - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") cmSpec = enterpriseApi.ClusterManagerSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -65,101 +50,22 @@ var _ = Describe("indingsep test", func() { }) AfterEach(func() { - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) Context("Ingestor and Indexer deployment", func() { - It("indingsep, smoke, indingsep: Splunk Operator can deploy Ingestors and Indexers", func() { + It("indexingestionsep, smoke: Splunk Operator can deploy Ingestors and Indexers", func() { // TODO: Remove secret reference and uncomment serviceAccountName part once IRSA fixed for Splunk and EKS 1.34+ // Create Service Account // testcaseEnvInst.Log.Info("Create Service Account") // testcaseEnvInst.CreateServiceAccount(serviceAccountName) - // Secret reference - volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( - "queue-secret-ref-volume", - testcaseEnvInst.GetIndexIngestSepSecretName(), - )} - queue.SQS.VolList = volumeSpec - - // Deploy Queue - testcaseEnvInst.Log.Info("Deploy Queue") - q, err := deployment.DeployQueue(ctx, "queue", queue) - Expect(err).To(Succeed(), "Unable to deploy Queue") - - // Deploy ObjectStorage - testcaseEnvInst.Log.Info("Deploy ObjectStorage") - objStorage, err := deployment.DeployObjectStorage(ctx, "os", objectStorage) - Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") - - // Deploy Ingestor Cluster - testcaseEnvInst.Log.Info("Deploy Ingestor Cluster") - _, err = deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster") - - // Deploy Cluster Manager - testcaseEnvInst.Log.Info("Deploy Cluster Manager") - _, err = deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Indexer Cluster") - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster") - - // Ensure that Ingestor Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Ingestor Cluster is in Ready phase") - testcaseEnvInst.VerifyIngestorReady(ctx, deployment) - - // Ensure that Cluster Manager is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Cluster Manager is in Ready phase") - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure that Indexer Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Indexer Cluster is in Ready phase") - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + setupIngestorStack(ctx, deployment, testcaseEnvInst, queue, objectStorage, cmSpec) - // Delete the Indexer Cluster - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) - Expect(err).To(Succeed(), "Unable to get Indexer Cluster instance", "Indexer Cluster Name", idxc) - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to delete Indexer Cluster instance", "Indexer Cluster Name", idxc) - - // Delete the Ingestor Cluster - ingest := &enterpriseApi.IngestorCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-ingest", ingest) - Expect(err).To(Succeed(), "Unable to get Ingestor Cluster instance", "Ingestor Cluster Name", ingest) - err = deployment.DeleteCR(ctx, ingest) - Expect(err).To(Succeed(), "Unable to delete Ingestor Cluster instance", "Ingestor Cluster Name", ingest) - - // Delete the Queue - q = &enterpriseApi.Queue{} - err = deployment.GetInstance(ctx, "queue", q) - Expect(err).To(Succeed(), "Unable to get Queue instance", "Queue Name", q) - err = deployment.DeleteCR(ctx, q) - Expect(err).To(Succeed(), "Unable to delete Queue", "Queue Name", q) - - // Delete the ObjectStorage - objStorage = &enterpriseApi.ObjectStorage{} - err = deployment.GetInstance(ctx, "os", objStorage) - Expect(err).To(Succeed(), "Unable to get ObjectStorage instance", "ObjectStorage Name", objStorage) - err = deployment.DeleteCR(ctx, objStorage) - Expect(err).To(Succeed(), "Unable to delete ObjectStorage", "ObjectStorage Name", objStorage) + deleteIngestorStack(ctx, deployment) }) - }) - Context("Ingestor and Indexer deployment", func() { - It("indingsep, smoke, indingsep: Splunk Operator can deploy Ingestors and Indexers with additional configurations", func() { + It("indexingestionsep, smoke: Splunk Operator can deploy Ingestors and Indexers with additional configurations", func() { // TODO: Remove secret reference and uncomment serviceAccountName part once IRSA fixed for Splunk and EKS 1.34+ // Create Service Account // testcaseEnvInst.Log.Info("Create Service Account") @@ -172,15 +78,8 @@ var _ = Describe("indingsep test", func() { )} queue.SQS.VolList = volumeSpec - // Deploy Queue - testcaseEnvInst.Log.Info("Deploy Queue") - q, err := deployment.DeployQueue(ctx, "queue", queue) - Expect(err).To(Succeed(), "Unable to deploy Queue") - - // Deploy ObjectStorage - testcaseEnvInst.Log.Info("Deploy ObjectStorage") - objStorage, err := deployment.DeployObjectStorage(ctx, "os", objectStorage) - Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") + // Deploy Queue and ObjectStorage + q, objStorage := deployQueueAndObjectStorage(ctx, deployment, queue, objectStorage) // Deploy Ingestor Cluster with additional configurations (similar to standalone app framework test) appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) @@ -228,7 +127,7 @@ var _ = Describe("indingsep test", func() { } testcaseEnvInst.Log.Info("Deploy Ingestor Cluster with additional configurations") - _, err = deployment.DeployIngestorClusterWithAdditionalConfiguration(ctx, ic) + _, err := deployment.DeployIngestorClusterWithAdditionalConfiguration(ctx, ic) Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster") // Ensure that Ingestor Cluster is in Ready phase @@ -260,73 +159,21 @@ var _ = Describe("indingsep test", func() { testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") // Verify probe configuration - testcaseEnvInst.Log.Info("Get config map for probes") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for probes", "ConfigMap", ConfigMapName) - testcaseEnvInst.Log.Info("Verify probe configurations on Ingestor pods") - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + testcaseEnvInst.VerifyProbeConfigAndScripts(ctx, deployment, true) }) - }) - Context("Ingestor and Indexer deployment", func() { - It("indingsep, integration, indingsep: Splunk Operator can deploy Ingestors and Indexers with correct setup", func() { + It("indexingestionsep, integration: Splunk Operator can deploy Ingestors and Indexers with correct setup", func() { // TODO: Remove secret reference and uncomment serviceAccountName part once IRSA fixed for Splunk and EKS 1.34+ // Create Service Account // testcaseEnvInst.Log.Info("Create Service Account") // testcaseEnvInst.CreateServiceAccount(serviceAccountName) - // Secret reference - volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( - "queue-secret-ref-volume", - testcaseEnvInst.GetIndexIngestSepSecretName(), - )} - queue.SQS.VolList = volumeSpec - - // Deploy Queue - testcaseEnvInst.Log.Info("Deploy Queue") - q, err := deployment.DeployQueue(ctx, "queue", queue) - Expect(err).To(Succeed(), "Unable to deploy Queue") - - // Deploy ObjectStorage - testcaseEnvInst.Log.Info("Deploy ObjectStorage") - objStorage, err := deployment.DeployObjectStorage(ctx, "os", objectStorage) - Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") - - // Deploy Ingestor Cluster - testcaseEnvInst.Log.Info("Deploy Ingestor Cluster") - _, err = deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster") - - // Deploy Cluster Manager - testcaseEnvInst.Log.Info("Deploy Cluster Manager") - _, err = deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Indexer Cluster") - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster") - - // Ensure that Ingestor Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Ingestor Cluster is in Ready phase") - testcaseEnvInst.VerifyIngestorReady(ctx, deployment) - - // Ensure that Cluster Manager is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Cluster Manager is in Ready phase") - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure that Indexer Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Indexer Cluster is in Ready phase") - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + setupIngestorStack(ctx, deployment, testcaseEnvInst, queue, objectStorage, cmSpec) // Get instance of current Ingestor Cluster CR with latest config testcaseEnvInst.Log.Info("Get instance of current Ingestor Cluster CR with latest config") ingest := &enterpriseApi.IngestorCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-ingest", ingest) - Expect(err).To(Succeed(), "Failed to get instance of Ingestor Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, ingest, deployment.GetName()+"-ingest", "Failed to get instance of Ingestor Cluster") // Verify Ingestor Cluster Status testcaseEnvInst.Log.Info("Verify Ingestor Cluster Status") @@ -336,8 +183,7 @@ var _ = Describe("indingsep test", func() { // Get instance of current Indexer Cluster CR with latest config testcaseEnvInst.Log.Info("Get instance of current Indexer Cluster CR with latest config") index := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", index) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, index, deployment.GetName()+"-idxc", "Failed to get instance of Indexer Cluster") // Verify Indexer Cluster Status testcaseEnvInst.Log.Info("Verify Indexer Cluster Status") @@ -352,18 +198,10 @@ var _ = Describe("indingsep test", func() { if strings.Contains(pod, "ingest") || strings.Contains(pod, "idxc") { // Verify outputs.conf - testcaseEnvInst.Log.Info("Verify outputs.conf") - outputsPath := "opt/splunk/etc/system/local/outputs.conf" - outputsConf, err := testenv.GetConfFile(pod, outputsPath, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get outputs.conf from Ingestor Cluster pod") - testenv.ValidateContent(outputsConf, outputs, true) + verifyConfFileContent(pod, "opt/splunk/etc/system/local/outputs.conf", deployment.GetName(), outputs, "Failed to get outputs.conf from Ingestor Cluster pod") // Verify default-mode.conf - testcaseEnvInst.Log.Info("Verify default-mode.conf") - defaultsPath := "opt/splunk/etc/system/local/default-mode.conf" - defaultsConf, err := testenv.GetConfFile(pod, defaultsPath, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get default-mode.conf from Ingestor Cluster pod") - testenv.ValidateContent(defaultsConf, defaultsAll, true) + verifyConfFileContent(pod, "opt/splunk/etc/system/local/default-mode.conf", deployment.GetName(), defaultsAll, "Failed to get default-mode.conf from Ingestor Cluster pod") // Verify AWS env variables testcaseEnvInst.Log.Info("Verify AWS env variables") @@ -378,13 +216,79 @@ var _ = Describe("indingsep test", func() { testenv.ValidateContent(defaultsConf, defaultsIngest, true) } else if strings.Contains(pod, "idxc") { // Verify inputs.conf - testcaseEnvInst.Log.Info("Verify inputs.conf") - inputsPath := "opt/splunk/etc/system/local/inputs.conf" - inputsConf, err := testenv.GetConfFile(pod, inputsPath, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get inputs.conf from Indexer Cluster pod") - testenv.ValidateContent(inputsConf, inputs, true) + verifyConfFileContent(pod, "opt/splunk/etc/system/local/inputs.conf", deployment.GetName(), inputs, "Failed to get inputs.conf from Indexer Cluster pod") } } }) }) }) + +// verifyConfFileContent retrieves a conf file from a pod and validates its content. +func verifyConfFileContent(pod, confPath, deploymentName string, expectedContent []string, errorMsg string) { + conf, err := testenv.GetConfFile(pod, confPath, deploymentName) + Expect(err).To(Succeed(), errorMsg) + testenv.ValidateContent(conf, expectedContent, true) +} + +// deployQueueAndObjectStorage deploys a Queue and ObjectStorage CR and returns both. +func deployQueueAndObjectStorage(ctx context.Context, deployment *testenv.Deployment, qSpec enterpriseApi.QueueSpec, osSpec enterpriseApi.ObjectStorageSpec) (*enterpriseApi.Queue, *enterpriseApi.ObjectStorage) { + q, err := deployment.DeployQueue(ctx, "queue", qSpec) + Expect(err).To(Succeed(), "Unable to deploy Queue") + + objStorage, err := deployment.DeployObjectStorage(ctx, "os", osSpec) + Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") + + return q, objStorage +} + +// setupIngestorStack deploys the full Queue/ObjectStorage/IngestorCluster/ClusterManager/IndexerCluster stack +// and verifies each component reaches the Ready phase. +func setupIngestorStack(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, qSpec enterpriseApi.QueueSpec, osSpec enterpriseApi.ObjectStorageSpec, cmSpec enterpriseApi.ClusterManagerSpec) { + volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( + "queue-secret-ref-volume", + testcaseEnvInst.GetIndexIngestSepSecretName(), + )} + qSpec.SQS.VolList = volumeSpec + + q, objStorage := deployQueueAndObjectStorage(ctx, deployment, qSpec, osSpec) + + _, err := deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") + Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster") + + _, err = deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) + Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") + + _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") + Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster") + + testcaseEnvInst.VerifyIngestorReady(ctx, deployment) + testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) +} + +// deleteIngestorStack tears down the full Queue/ObjectStorage/IngestorCluster/IndexerCluster stack. +func deleteIngestorStack(ctx context.Context, deployment *testenv.Deployment) { + // Delete the Indexer Cluster + idxc := &enterpriseApi.IndexerCluster{} + testenv.GetInstanceWithExpect(ctx, deployment, idxc, deployment.GetName()+"-idxc", "Unable to get Indexer Cluster instance") + err := deployment.DeleteCR(ctx, idxc) + Expect(err).To(Succeed(), "Unable to delete Indexer Cluster instance", "Indexer Cluster Name", idxc) + + // Delete the Ingestor Cluster + ingest := &enterpriseApi.IngestorCluster{} + testenv.GetInstanceWithExpect(ctx, deployment, ingest, deployment.GetName()+"-ingest", "Unable to get Ingestor Cluster instance") + err = deployment.DeleteCR(ctx, ingest) + Expect(err).To(Succeed(), "Unable to delete Ingestor Cluster instance", "Ingestor Cluster Name", ingest) + + // Delete the Queue + q := &enterpriseApi.Queue{} + testenv.GetInstanceWithExpect(ctx, deployment, q, "queue", "Unable to get Queue instance") + err = deployment.DeleteCR(ctx, q) + Expect(err).To(Succeed(), "Unable to delete Queue", "Queue Name", q) + + // Delete the ObjectStorage + objStorage := &enterpriseApi.ObjectStorage{} + testenv.GetInstanceWithExpect(ctx, deployment, objStorage, "os", "Unable to get ObjectStorage instance") + err = deployment.DeleteCR(ctx, objStorage) + Expect(err).To(Succeed(), "Unable to delete ObjectStorage", "ObjectStorage Name", objStorage) +} diff --git a/test/ingest_search/ingest_search_suite_test.go b/test/ingest_search/ingest_search_suite_test.go index 301f11611..6bce14483 100644 --- a/test/ingest_search/ingest_search_suite_test.go +++ b/test/ingest_search/ingest_search_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package ingestsearchtest import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "ingest-" + testenv.RandomDNSName(3) @@ -39,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/ingest_search/ingest_search_test.go b/test/ingest_search/ingest_search_test.go index 090f034a9..4fdb8d888 100644 --- a/test/ingest_search/ingest_search_test.go +++ b/test/ingest_search/ingest_search_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,253 +14,34 @@ package ingestsearchtest import ( - "bufio" "context" - "encoding/json" - "fmt" - "io" - "os" - "strings" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" ) -var _ = Describe("Ingest and Search Test", func() { +var _ = Describe("Ingest and Search test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment - var firstLine string ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) Context("Standalone deployment (S1)", func() { It("ingest_search, integration, s1: can search internal logs for standalone instance", func() { - - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - Eventually(func() enterpriseApi.Phase { - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - - searchString := "index=_internal | stats count by host" - searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) - if err != nil { - testcaseEnvInst.Log.Error(err, "Failed to execute search on pod", "pod", podName, "searchString", searchString) - return enterpriseApi.PhaseError - } - testcaseEnvInst.Log.Info("Performed a search", "searchString", searchString) - - var searchResults map[string]interface{} - unmarshalErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) - if unmarshalErr != nil { - testcaseEnvInst.Log.Error(unmarshalErr, "Failed to unmarshal JSON response") - } - - prettyResults, jsonErr := json.MarshalIndent(searchResults, "", " ") - if jsonErr != nil { - testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") - } else { - testcaseEnvInst.Log.Info("Sync Search results:", "prettyResults", string(prettyResults)) - } - - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(Equal(enterpriseApi.PhaseReady)) - - Eventually(func() enterpriseApi.Phase { - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - searchString := "index=_internal GUID component=ServerConfig" - - // Perform a simple search - sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString, deployment) - if reqErr != nil { - testcaseEnvInst.Log.Error(reqErr, "Failed to execute search on pod", "pod", podName, "searchString", searchString) - return enterpriseApi.PhaseError - } - testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) - - // Check SID status until done - searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) - if statusErr != nil { - testcaseEnvInst.Log.Error(statusErr, "Failed to get search status on pod", "pod", podName, "sid", sid) - return enterpriseApi.PhaseError - } - testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) - - // Get SID results - searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) - if resErr != nil { - testcaseEnvInst.Log.Error(resErr, "Failed to get search results on pod", "pod", podName, "sid", sid) - return enterpriseApi.PhaseError - } - - // Display results for debug purposes - prettyResults, jsonErr := json.MarshalIndent(searchResultsResp, "", " ") - if jsonErr != nil { - testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") - } else { - testcaseEnvInst.Log.Info("Search results:", "prettyResults", string(prettyResults)) - } - - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(Equal(enterpriseApi.PhaseReady)) + RunS1InternalLogSearchTest(ctx, deployment, testcaseEnvInst) }) - }) - Context("Standalone deployment (S1)", func() { It("ingest_search, integration, s1: can ingest custom data to new index and search", func() { - - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify splunk status is up - Eventually(func() enterpriseApi.Phase { - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - - splunkBin := "/opt/splunk/bin/splunk" - username := "admin" - password := "$(cat /mnt/splunk-secrets/password)" - splunkCmd := "status" - - statusCmd := fmt.Sprintf("%s %s -auth %s:%s", splunkBin, splunkCmd, username, password) - command := []string{"/bin/bash"} - statusCmdResp, stderr, err := deployment.PodExecCommand(ctx, podName, command, statusCmd, false) - if err != nil { - testcaseEnvInst.Log.Error(err, "Failed to execute command on pod", "pod", podName, "statusCmd", statusCmd, "statusCmdResp", statusCmdResp, "stderr", stderr) - return enterpriseApi.PhaseError - } - - if !strings.Contains(strings.ToLower(statusCmdResp), strings.ToLower("splunkd is running")) { - testcaseEnvInst.Log.Error(err, "Failed to find splunkd running", "pod", podName, "statusCmdResp", statusCmdResp) - return enterpriseApi.PhaseError - } - - testcaseEnvInst.Log.Info("Waiting for standalone splunkd status to be ready", "instance", standalone.ObjectMeta.Name, "Phase", standalone.Status.Phase) - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(Equal(enterpriseApi.PhaseReady)) - - // Create an index - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - indexName := "myTestIndex" - - // Create an index on a standalone instance - err = testenv.CreateAnIndexStandalone(ctx, indexName, podName, deployment) - Expect(err).To(Succeed(), "Failed response to add index to splunk") - - // Create a mock logfile to ingest - logFile := "/tmp/test.log" - err = testenv.CreateMockLogfile(logFile, 1) - Expect(err).To(Succeed(), "Failed response to add index to splunk logfile %s", logFile) - - // Copy log file and ingest it - err = testenv.IngestFileViaOneshot(ctx, logFile, indexName, podName, deployment) - Expect(err).To(Succeed(), "Failed to ingest logfile %s on pod %s", logFile, podName) - - // Read first line to find a search token - var file, openErr = os.Open(logFile) - Expect(openErr).To(Succeed(), "Failed to open newly created logfile %s on pod %s", logFile, podName) - - reader := bufio.NewReader(file) - var readErr error - firstLine, readErr = reader.ReadString('\n') - Expect(readErr).Should(Or(BeNil(), Equal(io.EOF)), "Failed to read first line of logfile %s on pod ", logFile, podName) - - tokens := strings.Fields(firstLine) - Expect(len(tokens)).To(BeNumerically(">=", 2), "Incorrect tokens (%s) in first logline %s for logfile %s", tokens, firstLine, logFile) - - searchToken := tokens[len(tokens)-1] - testcaseEnvInst.Log.Info("Got search token successfully", "logFile", logFile, "searchToken", searchToken) - - searchString := fmt.Sprintf("index=%s | stats count by host", indexName) - - // Wait for search results to be available instead of fixed sleep - err = testenv.WaitForSearchResultsNonEmpty(ctx, deployment, podName, searchString, 2*time.Second) - Expect(err).To(Succeed(), "Timed out waiting for search results") - - searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", podName, searchString) - - // Verify result. Should get count 1. result:{count:1} - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, podName) - - hostCount := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", hostCount["count"].(string), "host", hostCount["host"].(string)) - testHostCnt := strings.Compare(hostCount["count"].(string), "1") - testHostname := strings.Compare(hostCount["host"].(string), podName) - Expect(testHostCnt).To(Equal(0), "Incorrect search results for count. Expect: 1 Got: %d", hostCount["count"].(string)) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", podName, hostCount["host"].(string)) - - searchString2 := fmt.Sprintf("index=%s %s", indexName, searchToken) - sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString2, deployment) - Expect(reqErr).To(Succeed(), "Failed to execute search '%s' on pod %s", searchString, podName) - testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) - - // Check SID status until done - searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) - Expect(statusErr).To(Succeed(), "Failed to get search status on pod %s for sid %s", podName, sid) - testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) - - // Get SID results - searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) - Expect(resErr).To(Succeed(), "Failed to get search results on pod %s for sid %s", podName, sid) - - testcaseEnvInst.Log.Info("Raw Search results:", "searchResultsResp", searchResultsResp) - var searchResults2 testenv.SearchJobResultsResponse - jsonErr = json.Unmarshal([]byte(searchResultsResp), &searchResults2) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - found := false - for key, elem := range searchResults2.Results { - testcaseEnvInst.Log.Info("Search results _raw and host:", "_raw", elem.Raw, "host", elem.SplunkServer, "firstLine", firstLine) - trimFirstLine := strings.TrimSuffix(firstLine, "\n") - if strings.Compare(elem.Raw, trimFirstLine) == 0 { - testcaseEnvInst.Log.Info("Found search results in _raw and splunk_server", "key", key, "podName", podName, "elem", elem) - found = true - } - } - Expect(found).To(Equal(true), "Incorrect search results %s", searchResults) + RunS1IngestAndSearchTest(ctx, deployment, testcaseEnvInst) }) }) }) diff --git a/test/ingest_search/ingest_search_test_shared.go b/test/ingest_search/ingest_search_test_shared.go new file mode 100644 index 000000000..6873c3558 --- /dev/null +++ b/test/ingest_search/ingest_search_test_shared.go @@ -0,0 +1,204 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ingestsearchtest + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" +) + +// RunS1InternalLogSearchTest deploys a Standalone instance and verifies internal log searches +// using both synchronous and asynchronous search APIs. +func RunS1InternalLogSearchTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) { + standalone := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, deployment.GetName(), "", "") + + Eventually(func() enterpriseApi.Phase { + podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) + + searchString := "index=_internal | stats count by host" + searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) + if err != nil { + testcaseEnvInst.Log.Error(err, "Failed to execute search on pod", "pod", podName, "searchString", searchString) + return enterpriseApi.PhaseError + } + testcaseEnvInst.Log.Info("Performed a search", "searchString", searchString) + + var searchResults map[string]interface{} + unmarshalErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) + if unmarshalErr != nil { + testcaseEnvInst.Log.Error(unmarshalErr, "Failed to unmarshal JSON response") + } + + prettyResults, jsonErr := json.MarshalIndent(searchResults, "", " ") + if jsonErr != nil { + testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") + } else { + testcaseEnvInst.Log.Info("Sync Search results:", "prettyResults", string(prettyResults)) + } + + return standalone.Status.Phase + }, deployment.GetTimeout(), testenv.PollInterval).Should(Equal(enterpriseApi.PhaseReady)) + + Eventually(func() enterpriseApi.Phase { + podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) + searchString := "index=_internal GUID component=ServerConfig" + + sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString, deployment) + if reqErr != nil { + testcaseEnvInst.Log.Error(reqErr, "Failed to execute search on pod", "pod", podName, "searchString", searchString) + return enterpriseApi.PhaseError + } + testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) + + searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) + if statusErr != nil { + testcaseEnvInst.Log.Error(statusErr, "Failed to get search status on pod", "pod", podName, "sid", sid) + return enterpriseApi.PhaseError + } + testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) + + searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) + if resErr != nil { + testcaseEnvInst.Log.Error(resErr, "Failed to get search results on pod", "pod", podName, "sid", sid) + return enterpriseApi.PhaseError + } + + prettyResults, jsonErr := json.MarshalIndent(searchResultsResp, "", " ") + if jsonErr != nil { + testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") + } else { + testcaseEnvInst.Log.Info("Search results:", "prettyResults", string(prettyResults)) + } + + return standalone.Status.Phase + }, deployment.GetTimeout(), testenv.PollInterval).Should(Equal(enterpriseApi.PhaseReady)) +} + +// RunS1IngestAndSearchTest deploys a Standalone instance, ingests a custom log file into a new +// index, and verifies the ingested data is searchable via both sync and async search APIs. +func RunS1IngestAndSearchTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) { + standalone := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, deployment.GetName(), "", "") + + Eventually(func() enterpriseApi.Phase { + podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) + + splunkBin := "/opt/splunk/bin/splunk" + username := "admin" + password := "$(cat /mnt/splunk-secrets/password)" + splunkCmd := "status" + + statusCmd := fmt.Sprintf("%s %s -auth %s:%s", splunkBin, splunkCmd, username, password) + command := []string{"/bin/bash"} + statusCmdResp, stderr, err := deployment.PodExecCommand(ctx, podName, command, statusCmd, false) + if err != nil { + testcaseEnvInst.Log.Error(err, "Failed to execute command on pod", "pod", podName, "statusCmd", statusCmd, "statusCmdResp", statusCmdResp, "stderr", stderr) + return enterpriseApi.PhaseError + } + + if !strings.Contains(strings.ToLower(statusCmdResp), strings.ToLower("splunkd is running")) { + testcaseEnvInst.Log.Error(err, "Failed to find splunkd running", "pod", podName, "statusCmdResp", statusCmdResp) + return enterpriseApi.PhaseError + } + + testcaseEnvInst.Log.Info("Waiting for standalone splunkd status to be ready", "instance", standalone.ObjectMeta.Name, "phase", standalone.Status.Phase) + return standalone.Status.Phase + }, deployment.GetTimeout(), testenv.PollInterval).Should(Equal(enterpriseApi.PhaseReady)) + + podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) + indexName := "myTestIndex" + + err := testenv.CreateAnIndexStandalone(ctx, indexName, podName, deployment) + Expect(err).To(Succeed(), "Failed response to add index to splunk") + + logFile := "/tmp/test.log" + err = testenv.CreateMockLogfile(logFile, 1) + Expect(err).To(Succeed(), "Failed response to add index to splunk logfile %s", logFile) + + err = testenv.IngestFileViaOneshot(ctx, logFile, indexName, podName, deployment) + Expect(err).To(Succeed(), "Failed to ingest logfile %s on pod %s", logFile, podName) + + file, openErr := os.Open(logFile) + Expect(openErr).To(Succeed(), "Failed to open newly created logfile %s on pod %s", logFile, podName) + + reader := bufio.NewReader(file) + firstLine, readErr := reader.ReadString('\n') + Expect(readErr).Should(Or(BeNil(), Equal(io.EOF)), "Failed to read first line of logfile %s on pod ", logFile, podName) + + tokens := strings.Fields(firstLine) + Expect(len(tokens)).To(BeNumerically(">=", 2), "Incorrect tokens (%s) in first logline %s for logfile %s", tokens, firstLine, logFile) + + searchToken := tokens[len(tokens)-1] + testcaseEnvInst.Log.Info("Got search token successfully", "logFile", logFile, "searchToken", searchToken) + + searchString := fmt.Sprintf("index=%s | stats count by host", indexName) + + err = testenv.WaitForSearchResultsNonEmpty(ctx, deployment, podName, searchString, 2*time.Second) + Expect(err).To(Succeed(), "Timed out waiting for search results") + + searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) + Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", podName, searchString) + + var searchResults map[string]interface{} + jsonErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) + Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) + + testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) + Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, podName) + + hostCount := searchResults["result"].(map[string]interface{}) + testcaseEnvInst.Log.Info("Sync Search results host count:", "count", hostCount["count"].(string), "host", hostCount["host"].(string)) + testHostCnt := strings.Compare(hostCount["count"].(string), "1") + testHostname := strings.Compare(hostCount["host"].(string), podName) + Expect(testHostCnt).To(Equal(0), "Incorrect search results for count. Expect: 1 Got: %d", hostCount["count"].(string)) + Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", podName, hostCount["host"].(string)) + + searchString2 := fmt.Sprintf("index=%s %s", indexName, searchToken) + sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString2, deployment) + Expect(reqErr).To(Succeed(), "Failed to execute search '%s' on pod %s", searchString, podName) + testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) + + searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) + Expect(statusErr).To(Succeed(), "Failed to get search status on pod %s for sid %s", podName, sid) + testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) + + searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) + Expect(resErr).To(Succeed(), "Failed to get search results on pod %s for sid %s", podName, sid) + + testcaseEnvInst.Log.Info("Raw Search results:", "searchResultsResp", searchResultsResp) + var searchResults2 testenv.SearchJobResultsResponse + jsonErr = json.Unmarshal([]byte(searchResultsResp), &searchResults2) + Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) + + found := false + for key, elem := range searchResults2.Results { + testcaseEnvInst.Log.Info("Search results _raw and host:", "_raw", elem.Raw, "host", elem.SplunkServer, "firstLine", firstLine) + trimFirstLine := strings.TrimSuffix(firstLine, "\n") + if strings.Compare(elem.Raw, trimFirstLine) == 0 { + testcaseEnvInst.Log.Info("Found search results in _raw and splunk_server", "key", key, "podName", podName, "elem", elem) + found = true + } + } + Expect(found).To(Equal(true), "Incorrect search results %s", searchResults) +} diff --git a/test/licensemanager/manager_suite_test.go b/test/licensemanager/licensemanager_suite_test.go similarity index 77% rename from test/licensemanager/manager_suite_test.go rename to test/licensemanager/licensemanager_suite_test.go index d504c3685..8a58bf6da 100644 --- a/test/licensemanager/manager_suite_test.go +++ b/test/licensemanager/licensemanager_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package licensemanager import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "lmanager-" + testenv.RandomDNSName(3) diff --git a/test/licensemanager/lm_appfw_test.go b/test/licensemanager/lm_appfw_test.go new file mode 100644 index 000000000..4089e1a54 --- /dev/null +++ b/test/licensemanager/lm_appfw_test.go @@ -0,0 +1,45 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package licensemanager + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + "github.com/splunk/splunk-operator/test/testenv" +) + +var _ = Describe("Licensemanager App Framework test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + var config *LicenseTestConfig + ctx := context.TODO() + + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") + + config = NewLicenseManagerConfig() + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster) with License Manager", func() { + It("licensemanager, integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", func() { + RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, config) + }) + }) +}) diff --git a/test/licensemanager/lm_s1_test.go b/test/licensemanager/lm_test.go similarity index 53% rename from test/licensemanager/lm_s1_test.go rename to test/licensemanager/lm_test.go index 035ab730f..634d880e5 100644 --- a/test/licensemanager/lm_s1_test.go +++ b/test/licensemanager/lm_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,11 +15,8 @@ package licensemanager import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" ) @@ -31,35 +28,13 @@ var _ = Describe("Licensemanager test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") config = NewLicenseManagerConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) Context("Standalone deployment (S1) with License Manager", func() { @@ -67,4 +42,16 @@ var _ = Describe("Licensemanager test", func() { RunLMS1Test(ctx, deployment, testcaseEnvInst, config) }) }) + + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster) with License Manager", func() { + It("licensemanager, integration, c3: Splunk Operator can configure License Manager with Indexers and Search Heads in C3 SVA", func() { + RunLMC3Test(ctx, deployment, testcaseEnvInst, config) + }) + }) + + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster) with License Manager", func() { + It("licensemanager, integration, m4: Splunk Operator can configure License Manager with Indexers and Search Heads in M4 SVA", func() { + RunLMM4Test(ctx, deployment, testcaseEnvInst, config) + }) + }) }) diff --git a/test/licensemanager/lm_test_shared.go b/test/licensemanager/lm_test_shared.go index 205fd2c74..31d2c17f0 100644 --- a/test/licensemanager/lm_test_shared.go +++ b/test/licensemanager/lm_test_shared.go @@ -20,106 +20,84 @@ import ( "path/filepath" "time" + . "github.com/onsi/gomega" + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" "github.com/splunk/splunk-operator/test/testenv" - - . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" ) type LicenseTestConfig struct { - DeployStandaloneWithLM func(ctx context.Context, deployment *testenv.Deployment, name string, mcRef string) (*enterpriseApi.Standalone, error) - LicenseManagerReady func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) - ClusterManagerReady func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) - DeployMultisiteClusterWithSearchHead func(ctx context.Context, deployment *testenv.Deployment, name string, indexerReplicas, siteCount int, mcRef string) error - DeployLicenseManagerWithGivenSpec func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) - LicenseManagerPodName string - LicenseManagerSpecType string + *testenv.ClusterReadinessConfig + DeployLicenseManagerWithGivenSpec func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) + BuildLMAppFrameworkSpec func(testcaseEnvInst *testenv.TestCaseEnv, appFrameworkSpec enterpriseApi.AppFrameworkSpec) interface{} + LicenseManagerPodName string +} + +func newLicenseCommonSplunkSpec(testcaseEnvInst *testenv.TestCaseEnv) enterpriseApi.CommonSplunkSpec { + return enterpriseApi.CommonSplunkSpec{ + Volumes: []corev1.Volume{{ + Name: "licenses", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: testcaseEnvInst.GetLMConfigMap(), + }, + }, + }, + }}, + LicenseURL: "/mnt/licenses/enterprise.lic", + Spec: enterpriseApi.Spec{ImagePullPolicy: "Always", Image: testcaseEnvInst.GetSplunkImage()}, + } } func NewLicenseMasterConfig() *LicenseTestConfig { return &LicenseTestConfig{ - DeployStandaloneWithLM: func(ctx context.Context, deployment *testenv.Deployment, name string, mcRef string) (*enterpriseApi.Standalone, error) { - return deployment.DeployStandaloneWithLMaster(ctx, name, mcRef) - }, - LicenseManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyLicenseMasterReady(ctx, deployment) - }, - ClusterManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyClusterMasterReady(ctx, deployment) - }, - DeployMultisiteClusterWithSearchHead: func(ctx context.Context, deployment *testenv.Deployment, name string, indexerReplicas int, siteCount int, mcRef string) error { - return deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) - }, + ClusterReadinessConfig: testenv.NewClusterReadinessConfigV3(), DeployLicenseManagerWithGivenSpec: func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) { return deployment.DeployLicenseMasterWithGivenSpec(ctx, name, spec.(enterpriseApiV3.LicenseMasterSpec)) }, - LicenseManagerPodName: testenv.LicenseMasterPod, - LicenseManagerSpecType: "v3", + BuildLMAppFrameworkSpec: func(testcaseEnvInst *testenv.TestCaseEnv, appFrameworkSpec enterpriseApi.AppFrameworkSpec) interface{} { + return enterpriseApiV3.LicenseMasterSpec{ + CommonSplunkSpec: newLicenseCommonSplunkSpec(testcaseEnvInst), + AppFrameworkConfig: appFrameworkSpec, + } + }, + LicenseManagerPodName: testenv.LicenseMasterPod, } } func NewLicenseManagerConfig() *LicenseTestConfig { return &LicenseTestConfig{ - DeployStandaloneWithLM: func(ctx context.Context, deployment *testenv.Deployment, name string, mcRef string) (*enterpriseApi.Standalone, error) { - return deployment.DeployStandaloneWithLM(ctx, name, mcRef) - }, - LicenseManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyLicenseManagerReady(ctx, deployment) - }, - ClusterManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyClusterManagerReady(ctx, deployment) - }, - DeployMultisiteClusterWithSearchHead: func(ctx context.Context, deployment *testenv.Deployment, name string, indexerReplicas int, siteCount int, mcRef string) error { - return deployment.DeployMultisiteClusterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) - }, + ClusterReadinessConfig: testenv.NewClusterReadinessConfigV4(), DeployLicenseManagerWithGivenSpec: func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) { return deployment.DeployLicenseManagerWithGivenSpec(ctx, name, spec.(enterpriseApi.LicenseManagerSpec)) }, - LicenseManagerPodName: testenv.LicenseManagerPod, - LicenseManagerSpecType: "v4", + BuildLMAppFrameworkSpec: func(testcaseEnvInst *testenv.TestCaseEnv, appFrameworkSpec enterpriseApi.AppFrameworkSpec) interface{} { + return enterpriseApi.LicenseManagerSpec{ + CommonSplunkSpec: newLicenseCommonSplunkSpec(testcaseEnvInst), + AppFrameworkConfig: appFrameworkSpec, + } + }, + LicenseManagerPodName: testenv.LicenseManagerPod, } } -func downloadLicenseAndCreateConfigMap(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv) { - downloadDir := "licenseFolder" - +func downloadAppFiles(ctx context.Context, testDataS3Bucket, azureDataContainer, appDir, downloadDir string, appFileList []string, version string) { var err error - var licenseFilePath string - - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err = testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - case "azure": - licenseFilePath, err = testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - case "gcp": - licenseFilePath, err = testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) -} -func downloadAppFiles(ctx context.Context, testDataS3Bucket, azureDataContainer, appDir, downloadDir string, appFileList []string, version string) { switch testenv.ClusterProvider { case "eks": - err := testenv.DownloadFilesFromS3(testDataS3Bucket, appDir, downloadDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to download %s app files", version)) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, appDir, downloadDir, appFileList) case "azure": containerName := "/" + azureDataContainer + "/" + appDir - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDir, containerName, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to download %s app files", version)) + err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDir, containerName, appFileList) case "gcp": - err := testenv.DownloadFilesFromGCP(testDataS3Bucket, appDir, downloadDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to download %s app files", version)) + err = testenv.DownloadFilesFromGCP(testDataS3Bucket, appDir, downloadDir, appFileList) } + + Expect(err).To(Succeed(), fmt.Sprintf("Unable to download %s app files", version)) } func uploadAppFiles(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, testS3Bucket, testDir, downloadDir string, appFileList []string, version string) []string { @@ -157,37 +135,27 @@ func deleteUploadedFiles(ctx context.Context, testS3Bucket string, uploadedApps } func RunLMS1Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *LicenseTestConfig) { - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) + // Download license file + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) - // Create standalone Deployment with License Manager/Master + // Create Standalone deployment with License Manager/Master mcRef := deployment.GetName() standalone, err := config.DeployStandaloneWithLM(ctx, deployment, deployment.GetName(), mcRef) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with LM") // Wait for License Manager/Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - // Deploy Monitoring Console - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + // Deploy and verify Monitoring Console + _ = testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, deployment.GetName()) - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + // Verify livenessProbe and readinessProbe config object and scripts + testcaseEnvInst.VerifyProbeConfigAndScripts(ctx, deployment, false) - // Verify License Manager/Master is configured on standalone instance + // Verify License Manager/Master is configured on Standalone instance standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) testenv.VerifyLMConfiguredOnPod(ctx, deployment, standalonePodName) @@ -197,52 +165,26 @@ func RunLMS1Test(ctx context.Context, deployment *testenv.Deployment, testcaseEn } func RunLMC3Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *LicenseTestConfig) { - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) + // Download license file + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) - // Deploy Single site Cluster with License Manager/Master + // Deploy single site Cluster with License Manager/Master mcRef := deployment.GetName() err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) Expect(err).To(Succeed(), "Unable to deploy cluster") - // Wait for Cluster Manager to be in READY status - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, func(ctx context.Context, d *testenv.Deployment) { + config.ClusterManagerReady(ctx, d, testcaseEnvInst) + }) - // Wait for Search Head Cluster to be in READY status - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + deployMCAndVerifyRFSF(ctx, deployment, testcaseEnvInst, mcRef) - // Wait for Indexers to be in READY status - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + // Verify License Manager/Master is configured on indexers and search heads + indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), 3, false, 0) + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), 3, false, 0) + testenv.VerifyLMConfiguredOnPods(ctx, deployment, append(indexerPods, shPods...)) - // Verify License Manager/Master is configured on indexers - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 1) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 2) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - - // Verify License Manager/Master is configured on SHs - searchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 1) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 2) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - - // Verify License Manager/Master is configured on Monitoring Console - monitoringConsolePodName := fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName()) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) + testenv.VerifyLMConfiguredOnMC(ctx, deployment) } func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, testenvInstance *testenv.TestEnv, config *LicenseTestConfig) { @@ -274,10 +216,10 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment uploadedFiles := uploadAppFiles(ctx, testcaseEnvInst, testS3Bucket, testDir, downloadDirV1, appFileList, appVersion) uploadedApps = append(uploadedApps, uploadedFiles...) - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) + // Download license file + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) - // Create App framework Spec + // Create app framework spec volumeName := "lm-test-volume-" + testenv.RandomDNSName(3) var volumeSpec []enterpriseApi.VolumeSpec switch testenv.ClusterProvider { @@ -289,17 +231,17 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment volumeSpec = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volumeName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "gcp", "blob", testenv.GetDefaultS3Region())} } - // AppSourceDefaultSpec: Remote Storage volume name and Scope of App deployment + // AppSourceDefaultSpec: Remote Storage volume name and scope of app deployment appSourceDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: volumeName, Scope: enterpriseApi.ScopeLocal, } - // appSourceSpec: App source name, location and volume name and scope from appSourceDefaultSpec + // appSourceSpec: app source name, location and volume name and scope from appSourceDefaultSpec appSourceName := "lm-" + testenv.RandomDNSName(3) appSourceSpec := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceName, testDir, appSourceDefaultSpec)} - // appFrameworkSpec: AppSource settings, Poll Interval, volumes, appSources on volumes + // appFrameworkSpec: AppSource settings, poll interval, volumes, appSources on volumes appFrameworkSpec := enterpriseApi.AppFrameworkSpec{ Defaults: appSourceDefaultSpec, AppsRepoPollInterval: 60, @@ -307,62 +249,15 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment AppSources: appSourceSpec, } - var spec interface{} - if config.LicenseManagerSpecType == "v3" { - spec = enterpriseApiV3.LicenseMasterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Volumes: []corev1.Volume{ - { - Name: "licenses", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: testcaseEnvInst.GetLMConfigMap(), - }, - }, - }, - }, - }, - LicenseURL: "/mnt/licenses/enterprise.lic", - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - } else { - spec = enterpriseApi.LicenseManagerSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Volumes: []corev1.Volume{ - { - Name: "licenses", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: testcaseEnvInst.GetLMConfigMap(), - }, - }, - }, - }, - }, - LicenseURL: "/mnt/licenses/enterprise.lic", - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - } + spec := config.BuildLMAppFrameworkSpec(testcaseEnvInst, appFrameworkSpec) // Deploy the License Manager/Master with App Framework var err error _, err = config.DeployLicenseManagerWithGivenSpec(ctx, deployment, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy LM with App framework") + Expect(err).To(Succeed(), "Unable to deploy LM with App Framework") // Wait for License Manager/Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) // Verify apps are copied at the correct location on License Manager/Master (/etc/apps/) podName := []string{fmt.Sprintf(config.LicenseManagerPodName, deployment.GetName(), 0)} @@ -391,7 +286,7 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment Expect(err).To(Succeed(), "Timed out waiting for LicenseManager to reach Ready phase") // Verify LM stays in ready state - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) // Verify apps are copied at the correct location on License Manager/Master (/etc/apps/) testcaseEnvInst.VerifyAppsCopied(ctx, deployment, testenvInstance.GetName(), podName, appListV2, true, enterpriseApi.ScopeLocal) @@ -408,54 +303,29 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment } func RunLMM4Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *LicenseTestConfig) { - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) + // Download license file + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) // Deploy Multisite Cluster with License Manager/Master and Search Head siteCount := 3 mcRef := deployment.GetName() - err := config.DeployMultisiteClusterWithSearchHead(ctx, deployment, deployment.GetName(), 1, siteCount, mcRef) + err := config.DeployMultisiteCluster(ctx, deployment, deployment.GetName(), 1, siteCount, mcRef) Expect(err).To(Succeed(), "Unable to deploy cluster") - // Wait for Cluster Manager to be in READY status - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for Indexers to be in READY status - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) - // Verify Multisite Indexer Cluster status - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + deployMCAndVerifyRFSF(ctx, deployment, testcaseEnvInst, mcRef) - // Wait for Search Head Cluster to be in READY status - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Verify License Manager/Master is configured on indexers and search heads + indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), 3, false, 0) + testenv.VerifyLMConfiguredOnPods(ctx, deployment, append(indexerPods, shPods...)) - // Deploy Monitoring Console - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testenv.VerifyLMConfiguredOnMC(ctx, deployment) +} - // Verify RF SF is met +func deployMCAndVerifyRFSF(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, mcRef string) { + _ = testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, deployment.GetName()) testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify License Manager/Master is configured on indexers - indexerPodName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 2, 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 3, 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - - // Verify License Manager/Master is configured on SHs - searchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 1) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 2) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - - // Verify License Manager/Master is configured on Monitoring Console - monitoringConsolePodName := fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName()) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) } diff --git a/test/licensemanager/manager_lm_c3_test.go b/test/licensemanager/manager_lm_c3_test.go deleted file mode 100644 index 8e44449c9..000000000 --- a/test/licensemanager/manager_lm_c3_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemanager - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemanager test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = NewLicenseManagerConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Manager", func() { - It("licensemanager, integration, c3: Splunk Operator can configure License Manager with Indexers and Search Heads in C3 SVA", func() { - RunLMC3Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Manager", func() { - It("licensemanager, integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", func() { - RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, config) - }) - }) -}) diff --git a/test/licensemanager/manager_lm_m4_test.go b/test/licensemanager/manager_lm_m4_test.go deleted file mode 100644 index aedcd4dee..000000000 --- a/test/licensemanager/manager_lm_m4_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemanager - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemanager test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = NewLicenseManagerConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster) with License Manager", func() { - It("licensemanager, integration, m4: Splunk Operator can configure License Manager with indexers and search head in M4 SVA", func() { - RunLMM4Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - -}) diff --git a/test/licensemaster/lm_appfw_test.go b/test/licensemaster/lm_appfw_test.go new file mode 100644 index 000000000..0536c05aa --- /dev/null +++ b/test/licensemaster/lm_appfw_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package licensemaster + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + + "github.com/splunk/splunk-operator/test/licensemanager" + "github.com/splunk/splunk-operator/test/testenv" +) + +var _ = Describe("Licensemaster App Framework test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + var config *licensemanager.LicenseTestConfig + ctx := context.TODO() + + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") + + config = licensemanager.NewLicenseMasterConfig() + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster) with License Master", func() { + It("licensemaster, integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", func() { + licensemanager.RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, config) + }) + }) +}) diff --git a/test/licensemaster/lm_c3_test.go b/test/licensemaster/lm_c3_test.go deleted file mode 100644 index ed2f83f7f..000000000 --- a/test/licensemaster/lm_c3_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemaster - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/licensemanager" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("licensemaster test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *licensemanager.LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = licensemanager.NewLicenseMasterConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Master", func() { - It("licensemaster, integration, c3: Splunk Operator can configure License Master with Indexers and Search Heads in C3 SVA", func() { - licensemanager.RunLMC3Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Master", func() { - It("licensemaster, integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", func() { - licensemanager.RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, config) - }) - }) -}) diff --git a/test/licensemaster/lm_m4_test.go b/test/licensemaster/lm_m4_test.go deleted file mode 100644 index 1e4c6fb65..000000000 --- a/test/licensemaster/lm_m4_test.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemaster - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/licensemanager" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemaster test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *licensemanager.LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = licensemanager.NewLicenseMasterConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster) with License Master", func() { - It("licensemaster, integration, m4: Splunk Operator can configure License Master with indexers and search head in M4 SVA", func() { - licensemanager.RunLMM4Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - -}) diff --git a/test/licensemaster/lm_suite_test.go b/test/licensemaster/lm_suite_test.go index 9df3f4576..ceba436ed 100644 --- a/test/licensemaster/lm_suite_test.go +++ b/test/licensemaster/lm_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package licensemaster import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "lm-" + testenv.RandomDNSName(3) diff --git a/test/licensemaster/lm_s1_test.go b/test/licensemaster/lm_test.go similarity index 55% rename from test/licensemaster/lm_s1_test.go rename to test/licensemaster/lm_test.go index cc5c92e6b..55c5010af 100644 --- a/test/licensemaster/lm_s1_test.go +++ b/test/licensemaster/lm_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,11 +15,8 @@ package licensemaster import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/licensemanager" "github.com/splunk/splunk-operator/test/testenv" @@ -33,33 +30,13 @@ var _ = Describe("Licensemaster test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") config = licensemanager.NewLicenseMasterConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) Context("Standalone deployment (S1) with License Master", func() { @@ -67,4 +44,16 @@ var _ = Describe("Licensemaster test", func() { licensemanager.RunLMS1Test(ctx, deployment, testcaseEnvInst, config) }) }) + + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster) with License Master", func() { + It("licensemaster, integration, c3: Splunk Operator can configure License Master with Indexers and Search Heads in C3 SVA", func() { + licensemanager.RunLMC3Test(ctx, deployment, testcaseEnvInst, config) + }) + }) + + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster) with License Master", func() { + It("licensemaster, integration, m4: Splunk Operator can configure License Master with Indexers and Search Heads in M4 SVA", func() { + licensemanager.RunLMM4Test(ctx, deployment, testcaseEnvInst, config) + }) + }) }) diff --git a/test/monitoring_console/manager_monitoring_console_test.go b/test/monitoring_console/manager_monitoring_console_test.go deleted file mode 100644 index 1cc3b89f3..000000000 --- a/test/monitoring_console/manager_monitoring_console_test.go +++ /dev/null @@ -1,1101 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package monitoringconsoletest - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Monitoring Console test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Deploy Monitoring Console", func() { - It("smoke, monitoringconsole: can deploy MC CR and can be configured standalone", func() { - /* - Test Steps - 1. Deploy Monitoring Console - 2. Deploy Standalone - 3. Wait for Monitoring Console status to go back to READY - 4. Verify Standalone configured in Monitoring Console Config Map - 5. Verify Monitoring Console Pod has correct peers in Peer List - --------------- RECONFIG WITH NEW MC -------------------------- - 6. Reconfig S1 with 2nd Monitoring Console Name - 7. Check 2nd Monitoring Console Config Map to verify s1 - 8. Deploy 2nd Monitoring Console Pod - 9. Verify Standalone pod is configured on Monitoring Console Pod - 10. Verify 1st Monitoring Console Config Map is not configured with S1 - 11. Verify 1st Monitoring Console Pod is not configured with S1 - */ - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Create Standalone Spec and apply - standaloneOneName := deployment.GetName() - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standaloneOne) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // ######################### RECONFIGURE STANDALONE WITH SECOND MC ####################################### - - // Reconfig S1 with 2nd Monitoring Console Name - mcTwoName := deployment.GetName() + "-two" - err = deployment.GetInstance(ctx, standaloneOneName, standaloneOne) - Expect(err).To(Succeed(), "Unable to get instance of Standalone") - standaloneOne.Spec.MonitoringConsoleRef.Name = mcTwoName - - // Update Standalone with 2nd MC - err = deployment.UpdateCR(ctx, standaloneOne) - Expect(err).To(Succeed(), "Unable to update Standalone with new MC Name") - - // Deploy 2nd MC Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Second Monitoring Console Pod") - - // Verify 2nd Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // Check Standalone is configure in MC Config Map - testcaseEnvInst.Log.Info("Checking for Standalone Pod on SECOND MC Config Map after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcTwoName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in SECOND MC Peer list after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcTwoName, true, false) - - // Verify Monitoring Console One is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is not configured in MC ONE Config Map - testcaseEnvInst.Log.Info("Checking for Standalone Pod NOT ON FIRST MC Config Map after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, false) - - // Check Standalone Pod Not in MC ONE Peer List - testcaseEnvInst.Log.Info("Check standalone NOT ON FIRST MC Peer list after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, false, false) - - }) - }) - - Context("Standalone deployment (S1)", func() { - It("managermc1, integration: can deploy a MC with standalone instance and update MC with new standalone deployment", func() { - /* - Test Steps - 1. Deploy Standalone - 2. Wait for Standalone to go to READY - 3. Deploy Monitoring Console - 4. Wait for Monitoring Console status to be READY - 5. Verify Standalone configured in Monitoring Console Config Map - 6. Verify Monitoring Console Pod has correct peers in Peer List - 7. Deploy 2nd Standalone - 8. Wait for Second Standalone to be READY - 9. Wait for Monitoring Console status to go UPDATING then READY - 10. Verify both Standalone configured in Monitoring Console Config Map - 11. Verify both Standalone configured in Monitoring Console Pod Peers String - 12. Delete 2nd Standalone - 13. Wait for Monitoring Console to go to UPDATING then READY - 14. Verify only first Standalone configured in Monitoring Console Config Map - 15. Verify only first Standalone configured in Monitoring Console Pod Peers String - */ - - standaloneOneName := deployment.GetName() - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standaloneOne) - - // Deploy MC and wait for MC to be READY - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - // Add another standalone instance in namespace - testcaseEnvInst.Log.Info("Adding second standalone deployment to namespace") - // CSPL-901 standaloneTwoName := deployment.GetName() + "-two" - standaloneTwoName := "standalone-" + testenv.RandomDNSName(3) - // Configure Resources on second standalone CSPL-555 - standaloneTwoSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("2"), - "memory": resource.MustParse("4Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("0.2"), - "memory": resource.MustParse("256Mi"), - }, - }, - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneTwo, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneTwoName, standaloneTwoSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone two to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneTwoName, standaloneTwo) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to Updating Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Vrify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods = append(standalonePods, fmt.Sprintf(testenv.StandalonePod, standaloneTwoName, 0)) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Delete Standlone TWO of the standalone and ensure MC is updated - testcaseEnvInst.Log.Info("Deleting second standalone deployment to namespace", "Standalone Name", standaloneTwoName) - deployment.GetInstance(ctx, standaloneTwoName, standaloneTwo) - err = deployment.DeleteCR(ctx, standaloneTwo) - Expect(err).To(Succeed(), "Unable to delete standalone instance", "Standalone Name", standaloneTwo) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to Updating Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone One Pod in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check Standalone One Pod in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // Check Standalone TWO NOT configured in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneTwoName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Two Pod NOT in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, false) - - // Check Standalone Pod TWO NOT configured MC Peer List - testcaseEnvInst.Log.Info("Check Standalone Two Pod NOT in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, false, false) - - }) - }) - - Context("Standalone deployment with Scale up", func() { - It("managermc1, integration: can deploy a MC with standalone instance and update MC when standalone is scaled up", func() { - /* - Test Steps - 1. Deploy Standalone - 2. Wait for Standalone to go to READY - 3. Deploy Monitoring Console - 4. Wait for Monitoring Console status to be READY - 5. Verify Standalone configured in Monitoring Console Config Map - 6. Verify Monitoring Console Pod has correct peers in Peer List - 7. Scale Standalone to 2 REPLICAS - 8. Wait for Second Standalone POD to come up and PHASE to be READY - 9. Wait for Monitoring Console status to go UPDATING then READY - 10. Verify both Standalone PODS configured in Monitoring Console Config Map - 11. Verify both Standalone configured in Monitoring Console Pod Peers String - */ - - standaloneName := deployment.GetName() - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy MC and wait for MC to be READY - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 1, false, 0) - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Scale Standalone instance - testcaseEnvInst.Log.Info("Scaling Standalone CR") - scaledReplicaCount := 2 - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale Standalone") - - // Ensure standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to Updating Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 2, false, 0) - - // Check Standalone is configure in MC Config Map - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managermc, smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() { - /* - Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. VerifyMonitoring Console Pod has Search Heads in Peer strings - 6. Verify Monitoring Console Pod has peers(indexers) in Peer string - 7. Scale SH Cluster - 8. Scale Indexer Count - 9. Add a standalone - 10. Verify Standalone is configured in MC Config Map and Peer String - 11. Verify SH are configured in MC Config Map and Peers String - 12. Verify Indexers are configured in Peer String - */ - - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 - mcName := deployment.GetName() - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Wait for Cluster Manager to appear in Monitoring Console Config Map - err = testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Cluster Manager in MC ConfigMap") - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - // Wait for Monitoring console Pod to be configured with all search head - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads in MC config") - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // Scale Search Head Cluster - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - shcName := deployment.GetName() + "-shc" - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update Replicas of SHC - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale Search Head Cluster") - - // Ensure Search Head cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) - - // Scale indexers - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - idxcName := deployment.GetName() + "-idxc" - - // Get instance of current Indexer CR with latest config - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale Indxer Cluster") - - // Ensure Indxer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy Standalone Pod - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready Phase - // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to PENDING Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhasePending) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone configured on Monitoring Console - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, "SPLUNK_STANDALONE_URL", mcName, true) - - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, mcName, true, false) - - // Verify all Search Head Members are configured on Monitoring Console - shPods = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Config Map after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC after Scale Up") - indexerPods = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managermc1, integration: MC can configure SHC, indexer instances and reconfigure to new MC", func() { - /* - Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Monitoring Console Pod has peers in Peer string on MC Pod - --------------- RECONFIG CLUSTER MANAGER WITH NEW MC -------------------------- - 13. Reconfigure CM with Second MC - 14. Verify CM in config map of Second MC - 15. Create Second MC Pod - 16. Verify Indexers in second MC Pod - 17. Verify SHC not in second MC - 18. Verify SHC still present in first MC d - 19. Veirfy CM and Indexers not in first MC Pod - ---------------- RECONFIG SHC WITH NEW MC - 20. Configure SHC with Second MC - 21. Verify CM, DEPLOYER, Search Heads in config map of seconds MC - 22. Verify SHC in second MC pod - 23. Verify Indexers in Second MC Pod - 24. Verify CM and Deployer not in first MC CONFIG MAP - 24. Verify SHC, Indexers not in first MC Pod - */ - - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 - mcName := deployment.GetName() - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Manager in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Wait for Monitoring console Pod to be configured with all search head - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads in MC config") - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ################# Update Monitoring Console In Cluster Manager CR ################################## - - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Manager Goes to Updating Phase - //testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### - - // Check Cluster Manager in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager in Monitoring Console Two Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcTwoName, true) - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Verify Indexers in Monitoring Console Pod TWO Config String after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Deployer NOT in Monitoring Console TWO Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, false) - - // Check Monitoring Console TWO is NOT configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### - - // Verify Monitoring Console One Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Manager Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Verify Indexers NOT in Monitoring Console One Pod Config String after Cluster Manager Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - - // Check Monitoring Console One is still configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // ################# Update Monitoring Console In SHC CR ################################## - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - shcName := deployment.GetName() + "-shc" - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update SHC to use 2nd Montioring Console - shc.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to get update Monitoring Console in Search Head Cluster CRD") - - // Ensure Search Head Cluster go to Ready Phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify MC is Ready and stays in ready state - // testenv.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo, testcaseEnvInst) - - // ############################ VERIFICATION FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### - - // Check Cluster Manager in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcTwoName, true) - - // Check Deployer in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Deployer on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, true) - - // Verify all Search Head Members are configured on Monitoring Console Two - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after SHC Reconfig") - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads in MC two config after SHC reconfig") - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC TWO after SHC Reconfig") - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for indexers in MC two config after SHC reconfig") - - // ############################ VERIFICATION FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### - - // Verify MC ONE is Ready and stays in ready state before running verfications - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) - - // Check Cluster Manager Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, false) - - // Check DEPLOYER Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, false) - - // Verify all Search Head Members are Not configured on Monitoring Console One - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Pod after Search Head Reconfig") - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads to be removed from MC one config after SHC reconfig") - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pod NOT on MC One after SHC Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - }) - }) - - Context("Multisite Clustered deployment (M4 - 3 Site clustered indexer, search head cluster)", func() { - It("managermc2, integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", func() { - /* - Test Steps - 1. Deploy Multisite Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Indexers are configured in MC Config Map - 7. Verify Monitoring Console Pod has peers in Peer string on MC Pod - ############ CLUSTER MANAGER MC RECONFIG ################################# - 8. Configure Cluster Manager to use 2nd Monitoring Console - 9. Verify Cluster Manager is configured Config Maps of Second MC - 10. Deploy 2nd MC pod - 11. Verify Indexers in 2nd MC Pod - 12. Verify SHC not in 2nd MC CM - 13. Verify SHC not in 2nd MC Pod - 14. Verify Cluster Manager not 1st MC Config Map - 15. Verify Indexers not in 1st MC Pod - */ - defaultSHReplicas := 3 - defaultIndexerReplicas := 1 - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterWithMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure indexer clustered is configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Check Cluster Manager in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Monitoring console is configured with all search head instances in namespace - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Pod") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, 3) - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC POD") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ############ CLUSTER MANAGER MC RECONFIG ################################# - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Manager Goes to Updating Phase - //testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console Two instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // Check Cluster Manager in Monitoring Console Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager on MC TWO CONFIG MAP after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcTwoName, true) - - // Check Monitoring Console TWO is configured with all Indexers in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC TWO POD after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Monitoring console Two is NOT configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // Verify Monitoring Console One is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Manager NOT configured on Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager NOT in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pods Not on MC one POD after Cluster Manager RECONFIG") - //testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - - // Check Deployer in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Deployer in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Monitoring console One is configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - }) - }) - - Context("Standalone deployment (S1)", func() { - It("managermc2, integration: can deploy a MC with standalone instance and update MC with new standalone deployment of similar names", func() { - /* - Test Steps - 1. Deploy Standalone with name "search-head-adhoc" - 2. Wait for Standalone to go to READY - 3. Deploy Monitoring Console - 4. Wait for Monitoring Console status to be READY - 5. Verify Standalone configured in Monitoring Console Config Map - 6. Verify Monitoring Console Pod has correct peers in Peer List - 7. Deploy 2nd Standalone with name "search-head" - 8. Wait for Second Standalone to be READY - 9. Wait for Monitoring Console status to go UPDATING then READY - 10. Verify both Standalone configured in Monitoring Console Config Map - 11. Verify both Standalone configured in Monitoring Console Pod Peers String - 12. Delete 2nd Standalone - 13. Wait for Monitoring Console to go to UPDATING then READY - 14. Verify only first Standalone configured in Monitoring Console Config Map - 15. Verify only first Standalone configured in Monitoring Console Pod Peers String - */ - - standaloneOneName := "search-head-adhoc" - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standaloneOne to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneOneName, standaloneOne) - - // Deploy MC and wait for MC to be READY - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check standaloneOne is configured in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check standaloneOne Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Add another standalone instance in namespace - testcaseEnvInst.Log.Info("Adding second standalone deployment to namespace") - standaloneTwoName := "search-head" - // Configure Resources on second standalone CSPL-555 - standaloneTwoSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("2"), - "memory": resource.MustParse("4Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("0.2"), - "memory": resource.MustParse("256Mi"), - }, - }, - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneTwo, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneTwoName, standaloneTwoSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone two to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneTwoName, standaloneTwo) - - // wait for custom resource(CR) resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Vrify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check standaloneTwo is configure in MC Config Map - standalonePods = append(standalonePods, fmt.Sprintf(testenv.StandalonePod, standaloneTwoName, 0)) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check standaloneTwo Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Delete standaloneTwo and ensure MC is updated - testcaseEnvInst.Log.Info("Deleting second standalone deployment to namespace", "Standalone Name", standaloneTwoName) - deployment.GetInstance(ctx, standaloneTwoName, standaloneTwo) - err = deployment.DeleteCR(ctx, standaloneTwo) - Expect(err).To(Succeed(), "Unable to delete standalone instance", "Standalone Name", standaloneTwo) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone One Pod in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check Standalone One Pod in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // Check Standalone TWO NOT configured in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneTwoName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Two Pod NOT in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, false) - - // Check Standalone Pod TWO NOT configured MC Peer List - testcaseEnvInst.Log.Info("Check Standalone Two Pod NOT in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, false, false) - - }) - }) - -}) diff --git a/test/monitoring_console/monitoring_console_suite_test.go b/test/monitoring_console/monitoring_console_suite_test.go index 83bf2060d..167041418 100644 --- a/test/monitoring_console/monitoring_console_suite_test.go +++ b/test/monitoring_console/monitoring_console_suite_test.go @@ -1,23 +1,20 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package monitoringconsoletest import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "mc-" + testenv.RandomDNSName(3) @@ -41,13 +29,9 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) - sc, _ := GinkgoConfiguration() - sc.Timeout = 240 * time.Minute - - RunSpecs(t, "Running "+testSuiteName, sc) + RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { diff --git a/test/monitoring_console/monitoring_console_test.go b/test/monitoring_console/monitoring_console_test.go index 87605c825..67104889b 100644 --- a/test/monitoring_console/monitoring_console_test.go +++ b/test/monitoring_console/monitoring_console_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,49 +19,30 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" + splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" ) -var _ = Describe("Monitoring Console test", func() { +// Master (V3) Monitoring Console tests +var _ = Describe("Monitoring Console test (master)", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("mastermc, smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() { /* Test Steps @@ -69,7 +50,7 @@ var _ = Describe("Monitoring Console test", func() { 2. Deploy Monitoring Console 3. Wait for Monitoring Console status to go back to READY 4. Verify SH are configured in MC Config Map - 5. VerifyMonitoring Console Pod has Search Heads in Peer strings + 5. Verify Monitoring Console Pod has Search Heads in Peer strings 6. Verify Monitoring Console Pod has peers(indexers) in Peer string 7. Scale SH Cluster 8. Scale Indexer Count @@ -83,37 +64,422 @@ var _ = Describe("Monitoring Console test", func() { defaultIndexerReplicas := 3 mcName := deployment.GetName() + // Deploy and verify Monitoring Console + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") + + // Deploy and verify C3 cluster with MC + testcaseEnvInst.DeployAndVerifyC3WithMC(ctx, deployment, deployment.GetName(), defaultIndexerReplicas, mcName) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Wait for Cluster Master to appear in Monitoring Console Config Map + err := testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for Cluster Master in MC ConfigMap") + + // Verify MC configuration for C3 cluster + testcaseEnvInst.VerifyMCConfigForC3Cluster(ctx, deployment, deployment.GetName(), mcName, defaultSHReplicas, defaultIndexerReplicas, true) + + // Scale Search Head Cluster + scaledSHReplicas := defaultSHReplicas + 1 + testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) + testcaseEnvInst.ScaleSearchHeadCluster(ctx, deployment, deployment.GetName(), scaledSHReplicas) + + // Scale indexers + scaledIndexerReplicas := defaultIndexerReplicas + 1 + testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) + testcaseEnvInst.ScaleIndexerCluster(ctx, deployment, deployment.GetName(), scaledIndexerReplicas) + + // get revision number of the resource + resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Deploy Standalone with MC reference + testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, deployment.GetName(), mcName) + + // Ensure Indexer Cluster goes to Ready phase + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + + // Ensure Search Head Cluster goes to Ready Phase + // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + // wait for custom resource resource version to change + testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Verify Standalone configured on Monitoring Console + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, deployment.GetName(), mcName, true) + + // Verify MC configuration after scale up + testcaseEnvInst.Log.Info("Verify MC configuration after Scale Up") + testcaseEnvInst.VerifyMCConfigForC3Cluster(ctx, deployment, deployment.GetName(), mcName, scaledSHReplicas, scaledIndexerReplicas, true) + }) + + It("mastermc, integration: MC can configure SHC, indexer instances and reconfigure to new MC", func() { + /* + Test Steps + 1. Deploy Single Site Indexer Cluster + 2. Deploy Monitoring Console + 3. Wait for Monitoring Console status to go back to READY + 4. Verify SH are configured in MC Config Map + 5. Verify SH are configured in peer strings on MC Pod + 6. Verify Monitoring Console Pod has peers in Peer string on MC Pod + --------------- RECONFIG CLUSTER MANAGER WITH NEW MC -------------------------- + 13. Reconfigure CM with Second MC + 14. Verify CM in config map of Second MC + 15. Create Second MC Pod + 16. Verify Indexers in second MC Pod + 17. Verify SHC not in second MC + 18. Verify SHC still present in first MC + 19. Verify CM and Indexers not in first MC Pod + ---------------- RECONFIG SHC WITH NEW MC + 20. Configure SHC with Second MC + 21. Verify CM, DEPLOYER, Search Heads in config map of second MC + 22. Verify SHC in second MC pod + 23. Verify Indexers in Second MC Pod + 24. Verify CM and Deployer not in first MC CONFIG MAP + 24. Verify SHC, Indexers not in first MC Pod + */ + + defaultSHReplicas := 3 + defaultIndexerReplicas := 3 + mcName := deployment.GetName() + // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") + + err := deployment.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) + Expect(err).To(Succeed(), "Unable to deploy Cluster Master") + + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Verify MC configuration for C3 cluster + testcaseEnvInst.VerifyMCConfigForC3Cluster(ctx, deployment, deployment.GetName(), mcName, defaultSHReplicas, defaultIndexerReplicas, true) // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + // Generate pod name slices for later verification + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) + indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) + + // ################# Update Monitoring Console In Cluster Master CR ################################## + + mcTwoName := deployment.GetName() + "-two" + cm := &enterpriseApiV3.ClusterMaster{} + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to get instance of Cluster Manager") + // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - err = deployment.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") + cm.Spec.MonitoringConsoleRef.Name = mcTwoName + err = deployment.UpdateCR(ctx, cm) + + Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") + + // wait for custom resource resource version to change + testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) // Ensure that the cluster-master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase + // Ensure indexers go to Ready phase + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + + // Deploy and verify Monitoring Console Two + mcTwo := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") + + // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### + masterParams := MCReconfigParams{CMServiceNameFmt: testenv.ClusterMasterServiceName, CMURLKey: "SPLUNK_CLUSTER_MASTER_URL"} + VerifyMCTwoAfterCMReconfig(ctx, deployment, testcaseEnvInst, masterParams, mcTwoName, shPods, indexerPods, true) + + // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### + VerifyMCOneAfterCMReconfig(ctx, deployment, testcaseEnvInst, masterParams, mcName, mc, shPods, false) + + // ################# Update Monitoring Console In SHC CR ################################## + + // Update SHC to use 2nd Monitoring Console + shc := &enterpriseApi.SearchHeadCluster{} + shcName := deployment.GetName() + "-shc" + testenv.UpdateMonitoringConsoleRefAndVerify(ctx, deployment, testcaseEnvInst, shc, shcName, mcTwoName) + + // Ensure Search Head Cluster goes to Ready Phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Verify MC is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) + + // ############################ VERIFICATOIN FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### + VerifyMCTwoAfterSHCReconfig(ctx, deployment, testcaseEnvInst, masterParams, mcTwoName, shPods, indexerPods, 0) + + // ############################ VERIFICATOIN FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### + VerifyMCOneAfterSHCReconfig(ctx, deployment, testcaseEnvInst, masterParams, mcName, mc, shPods, 0) + }) + }) + + Context("Multisite Clustered deployment (M4 - 3 Site Clustered Indexer, Search Head Cluster)", func() { + It("mastermc, integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", func() { + /* + Test Steps + 1. Deploy Multisite Indexer Cluster + 2. Deploy Monitoring Console + 3. Wait for Monitoring Console status to go back to READY + 4. Verify SH are configured in MC Config Map + 5. Verify SH are configured in peer strings on MC Pod + 6. Verify Indexers are configured in MC Config Map + 7. Verify Monitoring Console Pod has peers in Peer string on MC Pod + ############ CLUSTER MANAGER MC RECONFIG ################################# + 8. Configure Cluster Master to use 2nd Monitoring Console + 9. Verify Cluster Master is configured Config Maps of Second MC + 10. Deploy 2nd MC pod + 11. Verify Indexers in 2nd MC Pod + 12. Verify SHC not in 2nd MC CM + 13. Verify SHC not in 2nd MC Pod + 14. Verify Cluster Master not 1st MC Config Map + 15. Verify Indexers not in 1st MC Pod + */ + defaultSHReplicas := 3 + defaultIndexerReplicas := 1 + siteCount := 3 + mcName := deployment.GetName() + err := deployment.DeployMultisiteClusterMasterWithMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) + Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with SHC") + + // Ensure that the cluster-master goes to Ready phase + testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) + + // Deploy and verify Monitoring Console + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + + // Verify MC configuration for M4 cluster + testcaseEnvInst.VerifyMCConfigForM4Cluster(ctx, deployment, deployment.GetName(), mcName, defaultSHReplicas, defaultIndexerReplicas, siteCount, true) + + // Generate pod name slices for later verification + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) + indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), defaultIndexerReplicas, true, siteCount) + + // ############ CLUSTER MANAGER MC RECONFIG ################################# + mcTwoName := deployment.GetName() + "-two" + // Update Cluster Manager to use 2nd Monitoring Console + cm := &enterpriseApiV3.ClusterMaster{} + testenv.UpdateMonitoringConsoleRefAndVerify(ctx, deployment, testcaseEnvInst, cm, deployment.GetName(), mcTwoName) + + // Ensure that the cluster-master goes to Ready phase + testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + + // Deploy and verify Monitoring Console Two + testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") + + masterParams := MCReconfigParams{CMServiceNameFmt: testenv.ClusterMasterServiceName, CMURLKey: "SPLUNK_CLUSTER_MASTER_URL"} + VerifyMCTwoAfterCMReconfig(ctx, deployment, testcaseEnvInst, masterParams, mcTwoName, shPods, indexerPods, false) + VerifyMCOneAfterCMReconfig(ctx, deployment, testcaseEnvInst, masterParams, mcName, mc, shPods, true) + + }) + }) +}) + +// Manager (V4) Monitoring Console tests +var _ = Describe("Monitoring Console test (manager)", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + ctx := context.TODO() + + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + Context("Deploy Monitoring Console", func() { + It("smoke, monitoringconsole: can deploy MC CR and can be configured standalone", func() { + /* + Test Steps + 1. Deploy Monitoring Console + 2. Deploy Standalone + 3. Wait for Monitoring Console status to go back to READY + 4. Verify Standalone configured in Monitoring Console Config Map + 5. Verify Monitoring Console Pod has correct peers in Peer List + --------------- RECONFIG WITH NEW MC -------------------------- + 6. Reconfig S1 with 2nd Monitoring Console Name + 7. Check 2nd Monitoring Console Config Map to verify s1 + 8. Deploy 2nd Monitoring Console Pod + 9. Verify Standalone pod is configured on Monitoring Console Pod + 10. Verify 1st Monitoring Console Config Map is not configured with S1 + 11. Verify 1st Monitoring Console Pod is not configured with S1 + */ + + // Deploy Monitoring Console CRD + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") + + // Create Standalone Spec and apply + standaloneOneName := deployment.GetName() + mcName := deployment.GetName() + spec := testenv.NewStandaloneSpecWithMCRef(testcaseEnvInst.GetSplunkImage(), mcName) + standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + // Wait for standalone to be in READY Status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standaloneOne) // wait for custom resource resource version to change testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - // Verify Monitoring Console is Ready and stays in ready state + // Verify MC is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Wait for Cluster Master to appear in Monitoring Console Config Map - err = testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Cluster Master in MC ConfigMap") + // Check Standalone is configured in MC + standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + + // ######################### RECONFIGURE STANDALONE WITH SECOND MC ####################################### + + // Reconfig S1 with 2nd Monitoring Console Name + mcTwoName := deployment.GetName() + "-two" + testenv.GetInstanceWithExpect(ctx, deployment, standaloneOne, standaloneOneName, "Unable to get instance of Standalone") + standaloneOne.Spec.MonitoringConsoleRef.Name = mcTwoName + + // Update Standalone with 2nd MC + err = deployment.UpdateCR(ctx, standaloneOne) + Expect(err).To(Succeed(), "Unable to update Standalone with new MC Name") + + // Deploy 2nd MC Pod + testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") + + // Check Standalone is configured in MC Two + testcaseEnvInst.Log.Info("Checking for Standalone on SECOND MC after Standalone RECONFIG") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcTwoName, true) + + // Verify Monitoring Console One is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Check Standalone is NOT configured in MC One + testcaseEnvInst.Log.Info("Checking for Standalone NOT ON FIRST MC after Standalone RECONFIG") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, false) + + }) + }) + + Context("Standalone deployment (S1)", func() { + It("managermc1, integration: can deploy a MC with standalone instance and update MC with new standalone deployment", func() { + RunS1StandaloneAddDeleteMCTest(ctx, deployment, testcaseEnvInst, deployment.GetName(), "standalone-"+testenv.RandomDNSName(3)) + }) + }) + + Context("Standalone deployment with Scale up", func() { + It("managermc1, integration: can deploy a MC with standalone instance and update MC when standalone is scaled up", func() { + /* + Test Steps + 1. Deploy Standalone + 2. Wait for Standalone to go to READY + 3. Deploy Monitoring Console + 4. Wait for Monitoring Console status to be READY + 5. Verify Standalone configured in Monitoring Console Config Map + 6. Verify Monitoring Console Pod has correct peers in Peer List + 7. Scale Standalone to 2 REPLICAS + 8. Wait for Second Standalone POD to come up and PHASE to be READY + 9. Wait for Monitoring Console status to go UPDATING then READY + 10. Verify both Standalone PODS configured in Monitoring Console Config Map + 11. Verify both Standalone configured in Monitoring Console Pod Peers String + */ + + standaloneName := deployment.GetName() + mcName := deployment.GetName() + spec := testenv.NewStandaloneSpecWithMCRef(testcaseEnvInst.GetSplunkImage(), mcName) + + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneName, spec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + // Wait for standalone to be in READY Status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + // Deploy MC and wait for MC to be READY + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + + // Check Standalone is configured in MC + standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 1, false, 0) + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + + // get revision number of the resource + resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Scale Standalone instance + testcaseEnvInst.Log.Info("Scaling Standalone CR") + scaledReplicaCount := 2 + standalone = &enterpriseApi.Standalone{} + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") + + standalone.Spec.Replicas = int32(scaledReplicaCount) + + err = deployment.UpdateCR(ctx, standalone) + Expect(err).To(Succeed(), "Failed to scale Standalone") + + // Ensure standalone is scaling up + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) + + // Wait for Standalone to be in READY status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + // wait for custom resource resource version to change + testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + + // Verify MC is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 2, false, 0) + + // Check both Standalone pods are configured in MC after scale up + testcaseEnvInst.Log.Info("Checking for Standalone Pods on MC after scale up") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + }) + }) + + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { + It("managermc, smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() { + /* + Test Steps + 1. Deploy Single Site Indexer Cluster + 2. Deploy Monitoring Console + 3. Wait for Monitoring Console status to go back to READY + 4. Verify SH are configured in MC Config Map + 5. Verify Monitoring Console Pod has Search Heads in Peer strings + 6. Verify Monitoring Console Pod has peers(indexers) in Peer string + 7. Scale SH Cluster + 8. Scale Indexer Count + 9. Add a standalone + 10. Verify Standalone is configured in MC Config Map and Peer String + 11. Verify SH are configured in MC Config Map and Peers String + 12. Verify Indexers are configured in Peer String + */ + + defaultSHReplicas := 3 + defaultIndexerReplicas := 3 + mcName := deployment.GetName() + + // Deploy Monitoring Console Pod + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") + + err := deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) + Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") + + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Wait for Cluster Manager to appear in Monitoring Console Config Map + err = testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for Cluster Manager in MC ConfigMap") // Check Deployer in Monitoring Console Config Map testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) @@ -121,8 +487,10 @@ var _ = Describe("Monitoring Console test", func() { // Check Search Head Pods in Monitoring Console Config Map shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - // Check Monitoring console Pod is configured with all search head - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + + // Wait for Monitoring console Pod to be configured with all search head + err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false, 5*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for search heads in MC config") // Check Monitoring console is configured with all Indexer in Name Space indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) @@ -131,83 +499,41 @@ var _ = Describe("Monitoring Console test", func() { // Scale Search Head Cluster scaledSHReplicas := defaultSHReplicas + 1 testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - shcName := deployment.GetName() + "-shc" - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update Replicas of SHC - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale Search Head Cluster") - - // Ensure Search Head cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) + testcaseEnvInst.ScaleSearchHeadCluster(ctx, deployment, deployment.GetName(), scaledSHReplicas) // Scale indexers scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - idxcName := deployment.GetName() + "-idxc" - - // Get instance of current Indexer CR with latest config - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale Indxer Cluster") - - // Ensure Indxer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) + testcaseEnvInst.ScaleIndexerCluster(ctx, deployment, deployment.GetName(), scaledIndexerReplicas) // get revision number of the resource resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) // Deploy Standalone Pod - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } + spec := testenv.NewStandaloneSpecWithMCRef(testcaseEnvInst.GetSplunkImage(), mcName) standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) Expect(err).To(Succeed(), "Unable to deploy standalone instance") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - // Ensure Indexer cluster go to Ready phase + // Ensure Indexer Cluster goes to Ready phase testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready Phase + // Ensure Search Head Cluster goes to Ready Phase // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) // wait for custom resource resource version to change testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - // Wait for MC to go to PENDING Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhasePending) - // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) // Check Standalone configured on Monitoring Console - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, "SPLUNK_STANDALONE_URL", mcName, true) - - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, mcName, true, false) + standalonePods := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) // Verify all Search Head Members are configured on Monitoring Console shPods = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas, false, 0) @@ -223,10 +549,8 @@ var _ = Describe("Monitoring Console test", func() { indexerPods = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas, false, 0) testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) }) - }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastermc, integration: MC can configure SHC, indexer instances and reconfigure to new MC", func() { + It("managermc1, integration: MC can configure SHC, indexer instances and reconfigure to new MC", func() { /* Test Steps 1. Deploy Single Site Indexer Cluster @@ -241,11 +565,11 @@ var _ = Describe("Monitoring Console test", func() { 15. Create Second MC Pod 16. Verify Indexers in second MC Pod 17. Verify SHC not in second MC - 18. Verify SHC still present in first MC d - 19. Veirfy CM and Indexers not in first MC Pod + 18. Verify SHC still present in first MC + 19. Verify CM and Indexers not in first MC Pod ---------------- RECONFIG SHC WITH NEW MC 20. Configure SHC with Second MC - 21. Verify CM, DEPLOYER, Search Heads in config map of seconds MC + 21. Verify CM, DEPLOYER, Search Heads in config map of second MC 22. Verify SHC in second MC pod 23. Verify Indexers in Second MC Pod 24. Verify CM and Deployer not in first MC CONFIG MAP @@ -257,35 +581,18 @@ var _ = Describe("Monitoring Console test", func() { mcName := deployment.GetName() // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) + err := deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Master in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true) + // Check Cluster Manager in Monitoring Console Config Map + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true) // Check Deployer in Monitoring Console Config Map testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) @@ -297,19 +604,19 @@ var _ = Describe("Monitoring Console test", func() { // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Check Monitoring console Pod is configured with all search head - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + // Wait for Monitoring console Pod to be configured with all search head + err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false, 5*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for search heads in MC config") // Check Monitoring console is configured with all Indexer in Name Space indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - // ################# Update Monitoring Console In Cluster Master CR ################################## + // ################# Update Monitoring Console In Cluster Manager CR ################################## mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") + cm := &enterpriseApi.ClusterManager{} + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to get instance of Cluster Manager") // get revision number of the resource resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) @@ -322,133 +629,50 @@ var _ = Describe("Monitoring Console test", func() { // wait for custom resource resource version to change testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - // Ensure Cluster Master Goes to Updating Phase - //testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + // Ensure that the cluster-manager goes to Ready phase + testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) // Ensure indexers go to Ready phase testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) + testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### - - // Check Cluster Master in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager in Monitoring Console Two Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcTwoName, true) - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Verify Indexers in Monitoring Console Pod TWO Config String after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Deployer NOT in Monitoring Console TWO Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, false) - - // Check Monitoring Console TWO is NOT configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) + managerParams := MCReconfigParams{CMServiceNameFmt: testenv.ClusterManagerServiceName, CMURLKey: splcommon.ClusterManagerURL} + VerifyMCTwoAfterCMReconfig(ctx, deployment, testcaseEnvInst, managerParams, mcTwoName, shPods, indexerPods, true) // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### - - // Verify Monitoring Console One Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Master Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Verify Indexers NOT in Monitoring Console One Pod Config String after Cluster Master Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - - // Check Monitoring Console One is still configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + VerifyMCOneAfterCMReconfig(ctx, deployment, testcaseEnvInst, managerParams, mcName, mc, shPods, false) // ################# Update Monitoring Console In SHC CR ################################## // Get instance of current SHC CR with latest config shc := &enterpriseApi.SearchHeadCluster{} shcName := deployment.GetName() + "-shc" - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, shcName, "Failed to get instance of Search Head Cluster") - // Update SHC to use 2nd Montioring Console + // Update SHC to use 2nd Monitoring Console shc.Spec.MonitoringConsoleRef.Name = mcTwoName err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to get update Monitoring Console in Search Head Cluster CRD") + Expect(err).To(Succeed(), "Failed to update Monitoring Console in Search Head Cluster CRD") - // Ensure Search Head Cluster go to Ready Phase + // Ensure Search Head Cluster goes to Ready Phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // ############################ VERIFICATOIN FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### - - // Check Cluster Master in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcTwoName, true) + // testenv.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo, testcaseEnvInst) - // Check Deployer in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Deployer on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, true) + // ############################ VERIFICATION FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### + VerifyMCTwoAfterSHCReconfig(ctx, deployment, testcaseEnvInst, managerParams, mcTwoName, shPods, indexerPods, 5*time.Minute) - // Verify all Search Head Members are configured on Monitoring Console Two - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false) - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC TWO after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // ############################ VERIFICATOIN FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### - - // Verify MC ONE is Ready and stays in ready state before running verfications - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) - - // Check Cluster Master Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, false) - - // Check DEPLOYER Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, false) - - // Verify all Search Head Members are Not configured on Monitoring Console One - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Pod after Search Head Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pod NOT on MC One after SHC Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) + // ############################ VERIFICATION FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### + VerifyMCOneAfterSHCReconfig(ctx, deployment, testcaseEnvInst, managerParams, mcName, mc, shPods, 5*time.Minute) }) }) - Context("Multisite Clustered deployment (M4 - 3 Site clustered indexer, search head cluster)", func() { - It("mastermc, integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", func() { + Context("Multisite Clustered deployment (M4 - 3 Site Clustered Indexer, Search Head Cluster)", func() { + It("managermc2, integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", func() { /* Test Steps 1. Deploy Multisite Indexer Cluster @@ -459,24 +683,24 @@ var _ = Describe("Monitoring Console test", func() { 6. Verify Indexers are configured in MC Config Map 7. Verify Monitoring Console Pod has peers in Peer string on MC Pod ############ CLUSTER MANAGER MC RECONFIG ################################# - 8. Configure Cluster Master to use 2nd Monitoring Console - 9. Verify Cluster Master is configured Config Maps of Second MC + 8. Configure Cluster Manager to use 2nd Monitoring Console + 9. Verify Cluster Manager is configured Config Maps of Second MC 10. Deploy 2nd MC pod 11. Verify Indexers in 2nd MC Pod 12. Verify SHC not in 2nd MC CM 13. Verify SHC not in 2nd MC Pod - 14. Verify Cluster Master not 1st MC Config Map + 14. Verify Cluster Manager not 1st MC Config Map 15. Verify Indexers not in 1st MC Pod */ defaultSHReplicas := 3 defaultIndexerReplicas := 1 siteCount := 3 mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterMasterWithMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with SHC") + err := deployment.DeployMultisiteClusterWithMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) + Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + // Ensure that the cluster-manager goes to Ready phase + testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) // Ensure indexers go to Ready phase testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) @@ -487,18 +711,14 @@ var _ = Describe("Monitoring Console test", func() { // Ensure search head cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - // Check Cluster Master in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true) + // Check Cluster Manager in Monitoring Console Config Map + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true) // Check Deployer in Monitoring Console Config Map testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") // Check Monitoring console is configured with all search head instances in namespace shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) @@ -516,9 +736,8 @@ var _ = Describe("Monitoring Console test", func() { // ############ CLUSTER MANAGER MC RECONFIG ################################# mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") + cm := &enterpriseApi.ClusterManager{} + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to get instance of Cluster Manager") // get revision number of the resource resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) @@ -530,57 +749,23 @@ var _ = Describe("Monitoring Console test", func() { // wait for custom resource resource version to change testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - // Ensure Cluster Master Goes to Updating Phase - //testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + // Ensure that the cluster-manager goes to Ready phase + testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console Two instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // Check Cluster Master in Monitoring Console Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager on MC TWO CONFIG MAP after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcTwoName, true) - - // Check Monitoring Console TWO is configured with all Indexers in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC TWO POD after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Monitoring console Two is NOT configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // Verify Monitoring Console One is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Master NOT configured on Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager NOT in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, false) + testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pods Not on MC one POD after Cluster Master RECONFIG") - //testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) + managerParams := MCReconfigParams{CMServiceNameFmt: testenv.ClusterManagerServiceName, CMURLKey: splcommon.ClusterManagerURL} + VerifyMCTwoAfterCMReconfig(ctx, deployment, testcaseEnvInst, managerParams, mcTwoName, shPods, indexerPods, false) + VerifyMCOneAfterCMReconfig(ctx, deployment, testcaseEnvInst, managerParams, mcName, mc, shPods, true) - // Check Deployer in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Deployer in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Monitoring console One is configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + }) + }) + Context("Standalone deployment (S1)", func() { + It("managermc2, integration: can deploy a MC with standalone instance and update MC with new standalone deployment of similar names", func() { + RunS1StandaloneAddDeleteMCTest(ctx, deployment, testcaseEnvInst, "search-head-adhoc", "search-head") }) }) + }) diff --git a/test/monitoring_console/monitoring_console_test_shared.go b/test/monitoring_console/monitoring_console_test_shared.go new file mode 100644 index 000000000..7cf06c586 --- /dev/null +++ b/test/monitoring_console/monitoring_console_test_shared.go @@ -0,0 +1,245 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package monitoringconsoletest + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// RunS1StandaloneAddDeleteMCTest deploys two standalone instances with a Monitoring Console, +// verifies both are registered, then deletes the second standalone and verifies the MC +// config map and peer list are updated correctly. +func RunS1StandaloneAddDeleteMCTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, standaloneOneName, standaloneTwoName string) { + mcName := deployment.GetName() + + // Deploy Standalone one with MCRef + spec := testenv.NewStandaloneSpecWithMCRef(testcaseEnvInst.GetSplunkImage(), mcName) + standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + // Wait for Standalone to be in READY Status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneOneName, standaloneOne) + + // Deploy MC and wait for MC to be READY + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + + // Check Standalone is configured in MC Config Map + standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) + + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + + // Get revision number of the resource + resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Add another standalone instance in namespace + testcaseEnvInst.Log.Info("Adding second standalone deployment to namespace") + standaloneTwoSpec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseEnvInst.GetSplunkImage(), + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("2"), + "memory": resource.MustParse("4Gi"), + }, + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("0.2"), + "memory": resource.MustParse("256Mi"), + }, + }, + }, + Volumes: []corev1.Volume{}, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: mcName, + }, + }, + } + standaloneTwo, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneTwoName, standaloneTwoSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + // Wait for standalone two to be in READY status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneTwoName, standaloneTwo) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Check both standalones are configured in MC Config Map + standalonePods = append(standalonePods, fmt.Sprintf(testenv.StandalonePod, standaloneTwoName, 0)) + + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map after adding new standalone") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + + // get revision number of the resource + resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Delete standalone two and ensure MC is updated + testcaseEnvInst.Log.Info("Deleting second standalone deployment from namespace", "Standalone Name", standaloneTwoName) + deployment.GetInstance(ctx, standaloneTwoName, standaloneTwo) + err = deployment.DeleteCR(ctx, standaloneTwo) + Expect(err).To(Succeed(), "Unable to delete standalone instance", "Standalone Name", standaloneTwo) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Check standalone one is still configured in MC Config Map + standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) + + testcaseEnvInst.Log.Info("Checking for Standalone One Pod in MC Config Map after deleting second standalone") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + + // Check Standalone Two NOT configured in MC Config Map + standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneTwoName, 1, false, 0) + + testcaseEnvInst.Log.Info("Checking for Standalone Two Pod NOT in MC Config Map after deleting second standalone") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, false) +} + +func verifyStandaloneInMC(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, pods []string, mcName string, shouldExist bool) { + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, pods, "SPLUNK_STANDALONE_URL", mcName, shouldExist) + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, pods, mcName, shouldExist, false) +} + +// MCReconfigParams holds the service name and URL parameters that differ between +// V3 (master) and V4 (manager) Monitoring Console tests. +type MCReconfigParams struct { + CMServiceNameFmt string // format string for CM service name (e.g., testenv.ClusterMasterServiceName) + CMURLKey string // config map URL key (e.g., "SPLUNK_CLUSTER_MASTER_URL" or splcommon.ClusterManagerURL) +} + +// VerifyMCTwoAfterCMReconfig verifies that MC Two is correctly configured after the Cluster Manager +// has been reconfigured to point to it: CM and indexers should be present, SH should be absent. +// If checkDeployerAbsent is true, also verifies deployer is absent on MC Two (used in C3 tests). +func VerifyMCTwoAfterCMReconfig(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, + params MCReconfigParams, mcTwoName string, shPods, indexerPods []string, checkDeployerAbsent bool) { + + testcaseEnvInst.Log.Info("Verify CM in MC Two Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcTwoName, true) + + testcaseEnvInst.Log.Info("Verify Indexers in MC Two Config String after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) + + if checkDeployerAbsent { + testcaseEnvInst.Log.Info("Verify Deployer NOT in MC Two Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, false) + } + + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC Two Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) + + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC Two Config String after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) +} + +// VerifyMCOneAfterCMReconfig verifies that MC One is correctly configured after the Cluster Manager +// has been reconfigured away from it: CM should be absent, SH should still be present. +// If checkDeployerPresent is true, also verifies deployer is still present on MC One (used in M4 tests). +func VerifyMCOneAfterCMReconfig(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, + params MCReconfigParams, mcName string, mc *enterpriseApi.MonitoringConsole, shPods []string, checkDeployerPresent bool) { + + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) + + testcaseEnvInst.Log.Info("Verify CM NOT in MC One Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcName, false) + + // CSPL-619: Indexer verification on MC One is commented out in all test variants + + if checkDeployerPresent { + testcaseEnvInst.Log.Info("Verify Deployer still in MC One Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) + } + + testcaseEnvInst.Log.Info("Verify SH Pods still in MC One Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) + + testcaseEnvInst.Log.Info("Verify SH Pods still in MC One Config String after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) +} + +// VerifyMCTwoAfterSHCReconfig verifies that MC Two has all components (CM, deployer, SH, indexers) +// after the SHC has been reconfigured to point to it. +// If timeout > 0, uses WaitForPodsInMCConfigString; otherwise uses direct VerifyPodsInMCConfigString. +func VerifyMCTwoAfterSHCReconfig(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, + params MCReconfigParams, mcTwoName string, shPods, indexerPods []string, timeout time.Duration) { + + testcaseEnvInst.Log.Info("Verify CM in MC Two Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcTwoName, true) + + testcaseEnvInst.Log.Info("Verify Deployer in MC Two Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, true) + + testcaseEnvInst.Log.Info("Verify SH Pods in MC Two Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, true) + + if timeout > 0 { + testcaseEnvInst.Log.Info("Verify SH Pods in MC Two Config String after SHC Reconfig (with wait)") + err := testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false, timeout) + Expect(err).To(Succeed(), "Timed out waiting for search heads in MC two config after SHC reconfig") + + testcaseEnvInst.Log.Info("Verify Indexers in MC Two Config String after SHC Reconfig (with wait)") + err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true, timeout) + Expect(err).To(Succeed(), "Timed out waiting for indexers in MC two config after SHC reconfig") + } else { + testcaseEnvInst.Log.Info("Verify SH Pods in MC Two Config String after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false) + + testcaseEnvInst.Log.Info("Verify Indexers in MC Two Config String after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) + } +} + +// VerifyMCOneAfterSHCReconfig verifies that MC One has lost all components (CM, deployer, SH) +// after the SHC has been reconfigured away from it. +// If timeout > 0, uses WaitForPodsInMCConfigString; otherwise uses direct VerifyPodsInMCConfigString. +func VerifyMCOneAfterSHCReconfig(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, + params MCReconfigParams, mcName string, mc *enterpriseApi.MonitoringConsole, shPods []string, timeout time.Duration) { + + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) + + testcaseEnvInst.Log.Info("Verify CM NOT in MC One Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcName, false) + + testcaseEnvInst.Log.Info("Verify Deployer NOT in MC One Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, false) + + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC One Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, false) + + if timeout > 0 { + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC One Config String after SHC Reconfig (with wait)") + err := testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false, timeout) + Expect(err).To(Succeed(), "Timed out waiting for search heads to be removed from MC one config after SHC reconfig") + } else { + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC One Config String after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false) + } + + // CSPL-619: Indexer verification on MC One is commented out in all test variants +} diff --git a/test/secret/manager_secret_c3_test.go b/test/secret/manager_secret_c3_test.go deleted file mode 100644 index 6944755b4..000000000 --- a/test/secret/manager_secret_c3_test.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managersecret, smoke, c3: secret update on indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Manager goes to update phase - testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/manager_secret_m4_test.go b/test/secret/manager_secret_m4_test.go deleted file mode 100644 index a51dac966..000000000 --- a/test/secret/manager_secret_m4_test.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for M4 SVA", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - // SpecifiedTestTimeout override default timeout for m4 test cases as we have seen - // it takes more than 3000 seconds for one of the test case - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 40000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("managersecret, integration, m4: secret update on multisite indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcName) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Test 1 - // Update Secrets Data and - // Verify New versioned secret are created with correct value. - // Verify new secrets are mounted on pods. - // Verify New Secrets are present in server.conf (Pass4SymmKey) - // Verify New Secrets via api access (password) - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Manager goes to update phase - testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each versioned secret for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/manager_secret_s1_test.go b/test/secret/manager_secret_s1_test.go deleted file mode 100644 index 58a671b67..000000000 --- a/test/secret/manager_secret_s1_test.go +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA S1", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() - var deployment *testenv.Deployment - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with LM and MC", func() { - It("managersecret, integration, s1: Secret update on a standalone instance with LM and MC", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Manager - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - - }) - }) - - Context("Standalone deployment (S1) with LM amd MC", func() { - It("managersecret, integration, s1: Secret Object is recreated on delete and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - //1. Delete Secret Object - //2. Verify New versioned secret are created with new values. - //3. Verify New secrets are mounted on pods. - //4. Verify New Secrets are present in server.conf (Pass4SymmKey) - //5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Manager - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret Object - err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) - - Context("Standalone deployment (S1)", func() { - It("managersecret, smoke, s1: Secret Object data is repopulated in secret object on passing empty Data map and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - // 1. Delete Secret Passing Empty Data Map to secret Object - // 2. Verify New versioned secret are created with new values. - // 3. Verify New secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Create standalone Deployment with MonitoringConsoleRef - mcName := deployment.GetName() - standaloneSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with MonitoringConsoleRef") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret by passing empty Data Map - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, map[string][]byte{}) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) -}) diff --git a/test/secret/secret_c3_test.go b/test/secret/secret_c3_test.go deleted file mode 100644 index 8baee517c..000000000 --- a/test/secret/secret_c3_test.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastersecret, smoke, c3: secret update on indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Wait for ClusterInitialized event to confirm cluster is fully initialized - idxcName := deployment.GetName() + "-idxc" - err = testcaseEnvInst.WaitForClusterInitialized(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for ClusterInitialized event on IndexerCluster") - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Master goes to update phase - testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for PasswordSyncCompleted event on SearchHeadCluster - shcName := deployment.GetName() + "-shc" - err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), shcName, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on SearchHeadCluster") - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Wait for PasswordSyncCompleted event on IndexerCluster - err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on IndexerCluster") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/secret_m4_test.go b/test/secret/secret_m4_test.go deleted file mode 100644 index 3a1efe591..000000000 --- a/test/secret/secret_m4_test.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for M4 SVA", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - // SpecifiedTestTimeout override default timeout for m4 test cases as we have seen - // it takes more than 3000 seconds for one of the test case - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 40000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("mastersecret, integration, m4: secret update on multisite indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcName) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Test 1 - // Update Secrets Data and - // Verify New versioned secret are created with correct value. - // Verify new secrets are mounted on pods. - // Verify New Secrets are present in server.conf (Pass4SymmKey) - // Verify New Secrets via api access (password) - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Master goes to update phase - testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each versioned secret for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/secret_s1_test.go b/test/secret/secret_s1_test.go deleted file mode 100644 index d680a4661..000000000 --- a/test/secret/secret_s1_test.go +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA S1", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() - var deployment *testenv.Deployment - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with LM and MC", func() { - It("mastersecret, integration, s1: Secret update on a standalone instance with LM and MC", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Master - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - - }) - }) - - Context("Standalone deployment (S1) with LM amd MC", func() { - It("mastersecret, integration, s1: Secret Object is recreated on delete and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - //1. Delete Secret Object - //2. Verify New versioned secret are created with new values. - //3. Verify New secrets are mounted on pods. - //4. Verify New Secrets are present in server.conf (Pass4SymmKey) - //5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Master - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret Object - err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) - - Context("Standalone deployment (S1)", func() { - It("mastersecret, smoke, s1: Secret Object data is repopulated in secret object on passing empty Data map and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - // 1. Delete Secret Passing Empty Data Map to secret Object - // 2. Verify New versioned secret are created with new values. - // 3. Verify New secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Create standalone Deployment with MonitoringConsoleRef - mcName := deployment.GetName() - standaloneSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with MonitoringConsoleRef") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret by passing empty Data Map - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, map[string][]byte{}) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) -}) diff --git a/test/secret/secret_suite_test.go b/test/secret/secret_suite_test.go index c7d40946a..b933fbbd9 100644 --- a/test/secret/secret_suite_test.go +++ b/test/secret/secret_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package secret import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 10 * ConsistentPollInterval -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "secret-" + testenv.RandomDNSName(3) @@ -39,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/secret/secret_test.go b/test/secret/secret_test.go new file mode 100644 index 000000000..f2e6d01d7 --- /dev/null +++ b/test/secret/secret_test.go @@ -0,0 +1,117 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package secret + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + + "github.com/splunk/splunk-operator/test/testenv" +) + +var _ = Describe("Secret test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + ctx := context.TODO() + + // S1 tests — both label variants use identical V4 config + s1SecretLabels := []string{"mastersecret", "managersecret"} + + for _, label := range s1SecretLabels { + label := label + Context("Standalone deployment (S1) with LM and MC", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(label+", integration, s1: Secret update on a standalone instance with LM and MC", func() { + config := testenv.NewClusterReadinessConfigV4() + RunS1SecretUpdateTest(ctx, deployment, testcaseEnvInst, config) + }) + + It(label+", integration, s1: Secret Object is recreated on delete and new secrets are applied to Splunk Pods", func() { + config := testenv.NewClusterReadinessConfigV4() + RunS1SecretDeleteTest(ctx, deployment, testcaseEnvInst, config) + }) + + It(label+", smoke, s1: Secret Object data is repopulated in secret object on passing empty Data map and new secrets are applied to Splunk Pods", func() { + config := testenv.NewClusterReadinessConfigV4() + RunS1SecretDeleteWithMCRefTest(ctx, deployment, testcaseEnvInst, config) + }) + }) + } + + // C3 tests — V3 (master) and V4 (manager) variants + c3SecretConfigs := []struct { + namePrefix string + label string + newConfig func() *testenv.ClusterReadinessConfig + }{ + {"master", "mastersecret", testenv.NewClusterReadinessConfigV3}, + {"", "managersecret", testenv.NewClusterReadinessConfigV4}, + } + + for _, tc := range c3SecretConfigs { + tc := tc + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.namePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(tc.label+", smoke, c3: secret update on Indexers and Search Head Cluster", func() { + config := tc.newConfig() + RunC3SecretUpdateTest(ctx, deployment, testcaseEnvInst, config) + }) + }) + } + + // M4 tests — V3 (master) and V4 (manager) variants + m4SecretConfigs := []struct { + namePrefix string + label string + newConfig func() *testenv.ClusterReadinessConfig + }{ + {"master", "mastersecret", testenv.NewClusterReadinessConfigV3}, + {"", "managersecret", testenv.NewClusterReadinessConfigV4}, + } + + for _, tc := range m4SecretConfigs { + tc := tc + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster)", func() { + BeforeEach(func() { + testenv.SpecifiedTestTimeout = 40000 + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.namePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(tc.label+", integration, m4: secret update on multisite Indexers and Search Head Cluster", func() { + config := tc.newConfig() + RunM4SecretUpdateTest(ctx, deployment, testcaseEnvInst, config) + }) + }) + } +}) diff --git a/test/secret/secret_test_shared.go b/test/secret/secret_test_shared.go new file mode 100644 index 000000000..f575a401e --- /dev/null +++ b/test/secret/secret_test_shared.go @@ -0,0 +1,278 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package secret + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" +) + +// verifySecretsPropagated checks that the given secret data has been propagated to all +// versioned secret objects, pods, server config, input config, and via the API. +func verifySecretsPropagated(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, secretData map[string][]byte, updated bool) { + // Once Pods are READY check each versioned secret for updated secret keys + secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) + + // Verify Secrets on versioned secret objects + testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretData, updated) + + // Once Pods are READY check each pod for updated secret keys + verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) + + // Verify secrets on pods + testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretData, updated) + + // Verify Secrets on ServerConf on Pod + testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretData, updated) + + // Verify Hec token on InputConf on Pod + testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretData, updated) + + // Verify Secrets via api access on Pod + testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretData, updated) +} + +// verifyLMAndStandaloneReady waits for License Manager then Standalone to reach READY status. +func verifyLMAndStandaloneReady(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, standalone *enterpriseApi.Standalone) { + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) +} + +// verifyLMAndClusterManagerReady waits for License Manager then Cluster Manager to reach READY status. +func verifyLMAndClusterManagerReady(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) + config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) +} + +// generateAndApplySecretUpdate creates randomized secret data and applies it to the namespace-scoped +// secret object, returning the updated data map for subsequent verification. +func generateAndApplySecretUpdate(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, namespaceScopedSecretName string) map[string][]byte { + modifiedHecToken := testenv.GetRandomHECToken() + modifiedValue := testenv.RandomDNSName(10) + updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifiedValue, modifiedValue, modifiedValue, modifiedValue) + err := testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) + Expect(err).To(Succeed(), "Unable to update secret Object") + return updatedSecretData +} + +// RunS1SecretUpdateTest runs the standard S1 secret update test workflow +func RunS1SecretUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + // Download License File and create config map + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) + + // Create standalone Deployment with License Manager + mcRef := deployment.GetName() + var standalone *enterpriseApi.Standalone + var err error + + standalone, err = config.DeployStandaloneWithLM(ctx, deployment, deployment.GetName(), mcRef) + Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") + + verifyLMAndStandaloneReady(ctx, deployment, testcaseEnvInst, config, standalone) + + // Deploy and verify Monitoring Console + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), deployment.GetName()) + + // Get Current Secrets Struct + namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + _, err = testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to get secret struct") + + // Update Secret Value on Secret Object + updatedSecretData := generateAndApplySecretUpdate(ctx, deployment, testcaseEnvInst, namespaceScopedSecretName) + + // Ensure standalone is updating + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) + + verifyLMAndStandaloneReady(ctx, deployment, testcaseEnvInst, config, standalone) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + verifySecretsPropagated(ctx, deployment, testcaseEnvInst, updatedSecretData, true) +} + +// RunS1SecretDeleteTest runs the standard S1 secret delete test workflow +func RunS1SecretDeleteTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + // Download License File and create config map + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) + + // Create standalone Deployment with License Manager + mcRef := deployment.GetName() + var standalone *enterpriseApi.Standalone + var err error + + standalone, err = config.DeployStandaloneWithLM(ctx, deployment, deployment.GetName(), mcRef) + Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") + + verifyLMAndStandaloneReady(ctx, deployment, testcaseEnvInst, config, standalone) + + // Deploy and verify Monitoring Console + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), deployment.GetName()) + + // Get Current Secrets Struct + namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to get secret struct") + + // Delete Secret Object + err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to delete secret Object") + + // Ensure standalone is updating + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) + + verifyLMAndStandaloneReady(ctx, deployment, testcaseEnvInst, config, standalone) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + verifySecretsPropagated(ctx, deployment, testcaseEnvInst, secretStruct.Data, false) +} + +// RunS1SecretDeleteWithMCRefTest runs the S1 secret delete test with MC reference workflow +func RunS1SecretDeleteWithMCRefTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + // Create standalone Deployment with MonitoringConsoleRef + var standalone *enterpriseApi.Standalone + var err error + + mcName := deployment.GetName() + standaloneSpec := testenv.NewStandaloneSpecWithMCRef(testcaseEnvInst.GetSplunkImage(), mcName) + standalone, err = deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance with MonitoringConsoleRef") + + // Wait for Standalone to be in READY status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + // Deploy and verify Monitoring Console + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + + // Get revision number of the resource + resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Get Current Secrets Struct + namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to get secret struct") + + // Delete Secret Object + err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to delete secret Object") + + // Ensure standalone is updating + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) + + // Wait for Standalone to be in READY status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + verifySecretsPropagated(ctx, deployment, testcaseEnvInst, secretStruct.Data, false) +} + +// RunC3SecretUpdateTest runs the standard C3 secret update test workflow +func RunC3SecretUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + // Download License File and create config map + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) + + mcRef := deployment.GetName() + err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, func(ctx context.Context, d *testenv.Deployment) { + config.ClusterManagerReady(ctx, d, testcaseEnvInst) + }) + + mc, resourceVersion, namespaceScopedSecretName := deployMCAndVerifyInitialSecret(ctx, deployment, testcaseEnvInst) + + // Update Secret Value on Secret Object + updatedSecretData := generateAndApplySecretUpdate(ctx, deployment, testcaseEnvInst, namespaceScopedSecretName) + + config.VerifyClusterManagerPhaseUpdating(ctx, deployment, testcaseEnvInst) + + verifyLMAndClusterManagerReady(ctx, deployment, testcaseEnvInst, config) + + // Ensure Search Head Cluster goes to Ready phase + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + // Wait for PasswordSyncCompleted event on SearchHeadCluster + shcName := deployment.GetName() + "-shc" + err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), shcName, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on SearchHeadCluster") + + // Ensure Indexers go to Ready phase + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + + // Wait for PasswordSyncCompleted event on IndexerCluster + idxcName := deployment.GetName() + "-idxc" + err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on IndexerCluster") + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Verify RF SF is met + testcaseEnvInst.Log.Info("Checking RF SF after secret change") + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + verifySecretsPropagated(ctx, deployment, testcaseEnvInst, updatedSecretData, true) +} + +func deployMCAndVerifyInitialSecret(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) (*enterpriseApi.MonitoringConsole, string, string) { + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), deployment.GetName()) + testcaseEnvInst.Log.Info("Checking RF SF before secret change") + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + _, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to get secret struct") + return mc, resourceVersion, namespaceScopedSecretName +} + +// RunM4SecretUpdateTest runs the standard M4 secret update test workflow +func RunM4SecretUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + // Download License File and create config map + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) + + siteCount := 3 + mcName := deployment.GetName() + var err error + + err = config.DeployMultisiteCluster(ctx, deployment, deployment.GetName(), 1, siteCount, mcName) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + verifyLMAndClusterManagerReady(ctx, deployment, testcaseEnvInst, config) + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) + + mc, resourceVersion, namespaceScopedSecretName := deployMCAndVerifyInitialSecret(ctx, deployment, testcaseEnvInst) + + // Update Secret Value on Secret Object + updatedSecretData := generateAndApplySecretUpdate(ctx, deployment, testcaseEnvInst, namespaceScopedSecretName) + + config.VerifyClusterManagerPhaseUpdating(ctx, deployment, testcaseEnvInst) + + verifyLMAndClusterManagerReady(ctx, deployment, testcaseEnvInst, config) + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Verify RF SF is met + testcaseEnvInst.Log.Info("Checking RF SF after secret change") + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + verifySecretsPropagated(ctx, deployment, testcaseEnvInst, updatedSecretData, true) +} diff --git a/test/smartstore/manager_smartstore_test.go b/test/smartstore/manager_smartstore_test.go deleted file mode 100644 index f54a98da5..000000000 --- a/test/smartstore/manager_smartstore_test.go +++ /dev/null @@ -1,282 +0,0 @@ -package smartstore - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Smartstore test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() - var deployment *testenv.Deployment - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone Deployment (S1)", func() { - It("managersmartstore, integration: Can configure multiple indexes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexVolumeMap := map[string]string{"test-index-" + testenv.RandomDNSName(3): volName, - "test-index-" + testenv.RandomDNSName(3): volName, - } - testcaseEnvInst.Log.Info("Index secret name ", "secret name ", testcaseEnvInst.GetIndexSecretName()) - - var indexSpec []enterpriseApi.IndexSpec - volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - // Create index volume spec from index volume map - for index, volume := range indexVolumeMap { - indexSpec = append(indexSpec, testenv.GenerateIndexSpec(index, volume)) - } - - // Generate smartstore spec - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volumeSpec, - IndexList: indexSpec, - } - - // Deploy Standalone - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for Standalone to reach Ready phase - err = testcaseEnvInst.WaitForStandalonePhase(ctx, deployment, testcaseEnvInst.GetName(), standalone.Name, enterpriseApi.PhaseReady, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") - - // Verify standalone goes to ready state and stays ready - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - for indexName := range indexVolumeMap { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for indexName := range indexVolumeMap { - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index by restarting splunk and check for index on S3 - for indexName := range indexVolumeMap { - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - }) - }) - - Context("Standalone Deployment (S1)", func() { - It("managersmartstore, integration: Can configure indexes which use default volumes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - specialConfig := map[string]int{"MaxGlobalDataSizeMB": 100, "MaxGlobalRawDataSizeMB": 100} - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - indexSpec := []enterpriseApi.IndexSpec{{Name: indexName, RemotePath: indexName}} - defaultSmartStoreSpec := enterpriseApi.IndexConfDefaultsSpec{IndexAndGlobalCommonSpec: enterpriseApi.IndexAndGlobalCommonSpec{VolName: volName, MaxGlobalDataSizeMB: uint(specialConfig["MaxGlobalDataSizeMB"]), MaxGlobalRawDataSizeMB: uint(specialConfig["MaxGlobalRawDataSizeMB"])}} - cacheManagerSmartStoreSpec := enterpriseApi.CacheManagerSpec{MaxCacheSizeMB: 9900000, EvictionPaddingSizeMB: 1000, MaxConcurrentDownloads: 6, MaxConcurrentUploads: 6, EvictionPolicy: "lru"} - - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - Defaults: defaultSmartStoreSpec, - CacheManagerConf: cacheManagerSmartStoreSpec, - } - - // Deploy Standalone with given smartstore spec - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Verify standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - - // Check special index configs - testcaseEnvInst.VerifyIndexConfigsMatch(ctx, deployment, podName, indexName, specialConfig["MaxGlobalDataSizeMB"], specialConfig["MaxGlobalRawDataSizeMB"]) - - // Ingest data to the index - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - - // Roll Hot Buckets on the test index by restarting splunk - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - - // Check for indexes on S3 - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - - // Verify Cachemanager Values - serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf" - - // Validate MaxCacheSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB)) - - // Validate EvictionPaddingSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB)) - - // Validate MaxConcurrentDownloads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads)) - - // Validate MaxConcurrentUploads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads)) - - // Validate EvictionPolicy - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4)", func() { - It("managersmartstore, smoke: Can configure indexes and volumes on Multisite Indexer Cluster through app", func() { - - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - indexSpec := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexName, volName)} - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - } - - siteCount := 3 - err := deployment.DeployMultisiteClusterWithSearchHeadAndIndexes(ctx, deployment.GetName(), 1, siteCount, testcaseEnvInst.GetIndexSecretName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - - testcaseEnvInst.Log.Info("Adding new index to Cluster Manager CR") - indexNameTwo := "test-index-two" + testenv.RandomDNSName(3) - indexList := []string{indexName, indexNameTwo} - newIndex := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexNameTwo, volName)} - - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Master") - cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to add new index to cluster master") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - for _, index := range indexList { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, index) - } - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenvInstance.Log.Info("Ingesting data on index", "Index Name", indexNameTwo) - testenv.IngestFileViaMonitor(ctx, logFile, indexNameTwo, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexNameTwo) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenvInstance.Log.Info("Checking index on S3", "Index Name", indexNameTwo, "Pod Name", podName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexNameTwo, podName) - } - }) - }) -}) diff --git a/test/smartstore/smartstore_suite_test.go b/test/smartstore/smartstore_suite_test.go index 693ca4cf8..54e837d61 100644 --- a/test/smartstore/smartstore_suite_test.go +++ b/test/smartstore/smartstore_suite_test.go @@ -1,8 +1,20 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package smartstore import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -10,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "smartstore-" + testenv.RandomDNSName(3) @@ -26,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/smartstore/smartstore_test.go b/test/smartstore/smartstore_test.go index bdd100f37..d9a7a7148 100644 --- a/test/smartstore/smartstore_test.go +++ b/test/smartstore/smartstore_test.go @@ -1,363 +1,97 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package smartstore import ( "context" - "fmt" "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" ) var _ = Describe("Smartstore test", func() { var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() var deployment *testenv.Deployment + ctx := context.TODO() - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone Deployment (S1)", func() { - It("mastersmartstore, integration: Can configure multiple indexes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexVolumeMap := map[string]string{"test-index-" + testenv.RandomDNSName(3): volName, - "test-index-" + testenv.RandomDNSName(3): volName, - } - testcaseEnvInst.Log.Info("Index secret name ", "secret name ", testcaseEnvInst.GetIndexSecretName()) - - var indexSpec []enterpriseApi.IndexSpec - volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - // Create index volume spec from index volume map - for index, volume := range indexVolumeMap { - indexSpec = append(indexSpec, testenv.GenerateIndexSpec(index, volume)) - } - - // Generate smartstore spec - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volumeSpec, - IndexList: indexSpec, - } - - // Deploy Standalone - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for Standalone to reach Ready phase - err = testcaseEnvInst.WaitForStandalonePhase(ctx, deployment, testcaseEnvInst.GetName(), standalone.Name, enterpriseApi.PhaseReady, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") - - // Verify standalone goes to ready state and stays ready - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - for indexName := range indexVolumeMap { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for indexName := range indexVolumeMap { - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index by restarting splunk and check for index on S3 - for indexName := range indexVolumeMap { - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - }) - }) - - Context("Standalone Deployment (S1)", func() { - It("mastersmartstore, integration: Can configure indexes which use default volumes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - specialConfig := map[string]int{"MaxGlobalDataSizeMB": 100, "MaxGlobalRawDataSizeMB": 100} - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - indexSpec := []enterpriseApi.IndexSpec{{Name: indexName, RemotePath: indexName}} - defaultSmartStoreSpec := enterpriseApi.IndexConfDefaultsSpec{IndexAndGlobalCommonSpec: enterpriseApi.IndexAndGlobalCommonSpec{VolName: volName, MaxGlobalDataSizeMB: uint(specialConfig["MaxGlobalDataSizeMB"]), MaxGlobalRawDataSizeMB: uint(specialConfig["MaxGlobalRawDataSizeMB"])}} - cacheManagerSmartStoreSpec := enterpriseApi.CacheManagerSpec{MaxCacheSizeMB: 9900000, EvictionPaddingSizeMB: 1000, MaxConcurrentDownloads: 6, MaxConcurrentUploads: 6, EvictionPolicy: "lru"} - - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - Defaults: defaultSmartStoreSpec, - CacheManagerConf: cacheManagerSmartStoreSpec, - } - - // Deploy Standalone with given smartstore spec - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Verify standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - - // Check special index configs - testcaseEnvInst.VerifyIndexConfigsMatch(ctx, deployment, podName, indexName, specialConfig["MaxGlobalDataSizeMB"], specialConfig["MaxGlobalRawDataSizeMB"]) - - // Ingest data to the index - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - - // Roll Hot Buckets on the test index by restarting splunk - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - - // Check for indexes on S3 - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - - // Verify Cachemanager Values - serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf" - - // Validate MaxCacheSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB)) - - // Validate EvictionPaddingSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB)) - - // Validate MaxConcurrentDownloads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads)) - - // Validate MaxConcurrentUploads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads)) - - // Validate EvictionPolicy - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy) - + smartstoreConfigs := []struct { + namePrefix string + label string + s1IndexesTimeout time.Duration + newConfig func() *testenv.ClusterReadinessConfig + }{ + {"master", "mastersmartstore", 2 * time.Minute, testenv.NewClusterReadinessConfigV3}, + {"", "managersmartstore", 5 * time.Minute, testenv.NewClusterReadinessConfigV4}, + } + + for _, tc := range smartstoreConfigs { + tc := tc + Context("Standalone deployment (S1)", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.namePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(tc.label+", integration: Can configure multiple indexes through app", func() { + RunS1MultipleIndexesTest(ctx, deployment, testcaseEnvInst, tc.s1IndexesTimeout) + }) + + It(tc.label+", integration: Can configure indexes which use default volumes through app", func() { + RunS1DefaultVolumesTest(ctx, deployment, testcaseEnvInst) + }) }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4)", func() { - It("mastersmartstore, m4, integration: Can configure indexes and volumes on Multisite Indexer Cluster through app", func() { - - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - indexSpec := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexName, volName)} - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - } - - siteCount := 3 - err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx, deployment.GetName(), 1, siteCount, testcaseEnvInst.GetIndexSecretName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - - oldBundleHash := testenv.GetClusterManagerBundleHash(ctx, deployment, "ClusterMaster") - - testcaseEnvInst.Log.Info("Adding new index to Cluster Manager CR") - indexNameTwo := "test-index-" + testenv.RandomDNSName(3) - indexList := []string{indexName, indexNameTwo} - newIndex := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexNameTwo, volName)} - - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Master") - cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to add new index to cluster master") - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Context("Multisite Indexer Cluster with Search Head Cluster (M4)", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.namePrefix) + }) - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify new bundle is pushed - testcaseEnvInst.VerifyClusterManagerBundlePush(ctx, deployment, testcaseEnvInst.GetName(), 1, oldBundleHash) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - for _, index := range indexList { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, index) - } - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenvInstance.Log.Info("Ingesting data on index", "Index Name", indexNameTwo) - testenv.IngestFileViaMonitor(ctx, logFile, indexNameTwo, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexNameTwo) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenvInstance.Log.Info("Checking index on S3", "Index Name", indexNameTwo, "Pod Name", podName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexNameTwo, podName) - } + It(tc.label+", m4, integration: Can configure indexes and volumes on Multisite Indexer Cluster through app", func() { + config := tc.newConfig() + RunM4MultisiteSmartStoreTest(ctx, deployment, testcaseEnvInst, config) + }) }) - }) + } Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, smartstore: can deploy a Standalone instance with Epehemeral Etc storage", func() { - - /* Test Steps - ################## SETUP #################### - * Create spec for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ############ VERIFICATION FOR STANDALONE ########### - * verify Standalone comes up with Ephemeral for Etc and pvc for Var volume - */ - - // Create App framework spec for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - EtcVolumeStorageConfig: enterpriseApi.StorageClassSpec{ - StorageClassName: "TestStorageEtcEph", - StorageCapacity: "1Gi", - EphemeralStorage: true, - }, - }, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, smartstore: can deploy a Standalone instance with Epehemeral Var storage", func() { - /* Test Steps - ################## SETUP #################### - * Create spec for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ############ VERIFICATION FOR STANDALONE ########### - * verify Standalone comes up with Ephemeral for Var and pvc for Etc volume - */ - - // Create App framework spec for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - VarVolumeStorageConfig: enterpriseApi.StorageClassSpec{ - StorageClassName: "TestStorageVarEph", - StorageCapacity: "1Gi", - EphemeralStorage: true, - }, - }, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + It("integration, s1, smartstore: can deploy a Standalone instance with Ephemeral Etc storage", func() { + storageConfig := enterpriseApi.StorageClassSpec{StorageClassName: "TestStorageEtcEph", StorageCapacity: "1Gi", EphemeralStorage: true} + RunS1EphemeralStorageTest(ctx, deployment, testcaseEnvInst, storageConfig, true) + }) + It("integration, s1, smartstore: can deploy a Standalone instance with Ephemeral Var storage", func() { + storageConfig := enterpriseApi.StorageClassSpec{StorageClassName: "TestStorageVarEph", StorageCapacity: "1Gi", EphemeralStorage: true} + RunS1EphemeralStorageTest(ctx, deployment, testcaseEnvInst, storageConfig, false) }) }) }) diff --git a/test/smartstore/smartstore_test_shared.go b/test/smartstore/smartstore_test_shared.go new file mode 100644 index 000000000..17d1d2eb7 --- /dev/null +++ b/test/smartstore/smartstore_test_shared.go @@ -0,0 +1,228 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package smartstore + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" + corev1 "k8s.io/api/core/v1" +) + +// RunS1MultipleIndexesTest runs the standard S1 multiple indexes SmartStore test workflow +func RunS1MultipleIndexesTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, waitTimeout time.Duration) { + volName := "test-volume-" + testenv.RandomDNSName(3) + indexVolumeMap := map[string]string{ + "test-index-" + testenv.RandomDNSName(3): volName, + "test-index-" + testenv.RandomDNSName(3): volName, + } + testcaseEnvInst.Log.Info("Index secret name ", "secret name ", testcaseEnvInst.GetIndexSecretName()) + + var indexSpec []enterpriseApi.IndexSpec + volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + + // Create index volume spec from index volume map + for index, volume := range indexVolumeMap { + indexSpec = append(indexSpec, testenv.GenerateIndexSpec(index, volume)) + } + + // Generate smartstore spec + smartStoreSpec := enterpriseApi.SmartStoreSpec{ + VolList: volumeSpec, + IndexList: indexSpec, + } + + // Deploy Standalone + standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance ") + + // Wait for Standalone to reach Ready phase + err = testcaseEnvInst.WaitForStandalonePhase(ctx, deployment, testcaseEnvInst.GetName(), standalone.Name, enterpriseApi.PhaseReady, waitTimeout) + Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") + + // Verify standalone goes to ready state and stays ready + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + // Check index on pod + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + for indexName := range indexVolumeMap { + testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) + } + + // Ingest data to the index + for indexName := range indexVolumeMap { + logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) + testenv.CreateMockLogfile(logFile, 2000) + testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) + } + + // Roll Hot Buckets on the test index by restarting splunk and check for index on S3 + for indexName := range indexVolumeMap { + testenv.RollHotToWarm(ctx, deployment, podName, indexName) + testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) + } +} + +// RunS1DefaultVolumesTest runs the standard S1 default volumes SmartStore test workflow +func RunS1DefaultVolumesTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) { + volName := "test-volume-" + testenv.RandomDNSName(3) + indexName := "test-index-" + testenv.RandomDNSName(3) + + specialConfig := map[string]int{"MaxGlobalDataSizeMB": 100, "MaxGlobalRawDataSizeMB": 100} + + volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + + indexSpec := []enterpriseApi.IndexSpec{{Name: indexName, RemotePath: indexName}} + defaultSmartStoreSpec := enterpriseApi.IndexConfDefaultsSpec{IndexAndGlobalCommonSpec: enterpriseApi.IndexAndGlobalCommonSpec{VolName: volName, MaxGlobalDataSizeMB: uint(specialConfig["MaxGlobalDataSizeMB"]), MaxGlobalRawDataSizeMB: uint(specialConfig["MaxGlobalRawDataSizeMB"])}} + cacheManagerSmartStoreSpec := enterpriseApi.CacheManagerSpec{MaxCacheSizeMB: 9900000, EvictionPaddingSizeMB: 1000, MaxConcurrentDownloads: 6, MaxConcurrentUploads: 6, EvictionPolicy: "lru"} + + smartStoreSpec := enterpriseApi.SmartStoreSpec{ + VolList: volSpec, + IndexList: indexSpec, + Defaults: defaultSmartStoreSpec, + CacheManagerConf: cacheManagerSmartStoreSpec, + } + + // Deploy Standalone with given smartstore spec + standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance ") + + // Verify standalone goes to ready state + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + // Check index on pod + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) + + // Check special index configs + testcaseEnvInst.VerifyIndexConfigsMatch(ctx, deployment, podName, indexName, specialConfig["MaxGlobalDataSizeMB"], specialConfig["MaxGlobalRawDataSizeMB"]) + + // Ingest data to the index + logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) + testenv.CreateMockLogfile(logFile, 2000) + testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) + + // Roll Hot Buckets on the test index by restarting splunk + testenv.RollHotToWarm(ctx, deployment, podName, indexName) + + // Check for indexes on S3 + testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) + + // Verify Cachemanager Values + serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf" + + // Validate MaxCacheSizeMB + testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB)) + + // Validate EvictionPaddingSizeMB + testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB)) + + // Validate MaxConcurrentDownloads + testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads)) + + // Validate MaxConcurrentUploads + testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads)) + + // Validate EvictionPolicy + testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy) +} + +// RunS1EphemeralStorageTest deploys a Standalone with one ephemeral storage volume configured and verifies it is ready. +// Pass etcStorage=true to set EtcVolumeStorageConfig, false to set VarVolumeStorageConfig. +func RunS1EphemeralStorageTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, storageConfig enterpriseApi.StorageClassSpec, etcStorage bool) { + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "Always", + Image: testcaseEnvInst.GetSplunkImage(), + }, + Volumes: []corev1.Volume{}, + }, + } + if etcStorage { + spec.CommonSplunkSpec.EtcVolumeStorageConfig = storageConfig + } else { + spec.CommonSplunkSpec.VarVolumeStorageConfig = storageConfig + } + + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") + + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) +} + +// verifyM4ClusterAndRFSF verifies cluster manager and multisite cluster are ready and RF/SF is met. +func verifyM4ClusterAndRFSF(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, siteCount int) { + config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + testcaseEnvInst.VerifyMultisiteClusterReadyAndRFSF(ctx, deployment, siteCount) +} + +// RunM4MultisiteSmartStoreTest runs the standard M4 multisite SmartStore test workflow +func RunM4MultisiteSmartStoreTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + volName := "test-volume-" + testenv.RandomDNSName(3) + indexName := "test-index-" + testenv.RandomDNSName(3) + + volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + indexSpec := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexName, volName)} + smartStoreSpec := enterpriseApi.SmartStoreSpec{ + VolList: volSpec, + IndexList: indexSpec, + } + + siteCount := 3 + var err error + + err = config.DeployMultisiteClusterWithIndexes(ctx, deployment, deployment.GetName(), 1, siteCount, testcaseEnvInst.GetIndexSecretName(), smartStoreSpec) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + verifyM4ClusterAndRFSF(ctx, deployment, testcaseEnvInst, config, siteCount) + + // Use multisite workflow helper to verify index, ingest data, roll to warm, and verify on S3 + testcaseEnvInst.MultisiteIndexerWorkflow(ctx, deployment, deployment.GetName(), siteCount, indexName) + + // Get old bundle hash before adding new index + oldBundleHash := config.GetBundleHash(ctx, deployment) + + testcaseEnvInst.Log.Info("Adding new index to Cluster Manager CR") + indexNameTwo := "test-index-" + testenv.RandomDNSName(3) + indexList := []string{indexName, indexNameTwo} + newIndex := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexNameTwo, volName)} + + // Update CR with new index based on API version + config.AppendSmartStoreIndex(ctx, deployment, newIndex) + + verifyM4ClusterAndRFSF(ctx, deployment, testcaseEnvInst, config, siteCount) + + // Verify new bundle is pushed (only for v3) + if config.APIVersion == "v3" { + testcaseEnvInst.VerifyClusterManagerBundlePush(ctx, deployment, testcaseEnvInst.GetName(), 1, oldBundleHash) + } + + // Verify both indexes on all sites + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) + for _, index := range indexList { + testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, index) + } + } + + // Use multisite workflow helper for the new index + testcaseEnvInst.Log.Info("Ingesting data on index", "Index Name", indexNameTwo) + testcaseEnvInst.MultisiteIndexerWorkflow(ctx, deployment, deployment.GetName(), siteCount, indexNameTwo) +} diff --git a/test/smoke/cluster_manager_sites_response.go b/test/smoke/cluster_manager_sites_response.go deleted file mode 100644 index f9bcdee60..000000000 --- a/test/smoke/cluster_manager_sites_response.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package smoke - -// ClusterManagerSitesResponse is a representation of the sites managed by a Splunk cluster-manager -// Endpoint: /services/cluster/manager/sites -type ClusterManagerSitesResponse struct { - Entries []ClusterManagerSitesEntry `json:"entry"` -} - -// ClusterManagerSitesEntry represents a site of an indexer cluster with its metadata -type ClusterManagerSitesEntry struct { - Name string `json:"name"` - Content ClusterManagerSitesContent `json:"content"` -} - -// ClusterManagerSitesContent represents detailed information about a site -type ClusterManagerSitesContent struct { - Peers map[string]ClusterManagerSitesPeer `json:"peers"` -} - -// ClusterManagerSitesPeer reprensents an indexer peer member of a site -type ClusterManagerSitesPeer struct { - ServerName string `json:"server_name"` -} diff --git a/test/smoke/smoke_suite_test.go b/test/smoke/smoke_suite_test.go index b47f8a984..f0ee154dc 100644 --- a/test/smoke/smoke_suite_test.go +++ b/test/smoke/smoke_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package smoke import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "smoke-" + testenv.RandomDNSName(3) @@ -39,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index d6d969819..66f355c33 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,16 +15,10 @@ package smoke import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - corev1 "k8s.io/api/core/v1" ) var _ = Describe("Smoke test", func() { @@ -34,134 +28,41 @@ var _ = Describe("Smoke test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) Context("Standalone deployment (S1)", func() { It("smoke, basic, s1: can deploy a standalone instance", func() { - - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Verify standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + testenv.RunStandaloneDeploymentWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName()) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("smoke, basic, c3: can deploy indexers and search head cluster", func() { - - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, "") - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + testenv.RunC3DeploymentWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName(), 3, "") }) }) - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster)", func() { It("smoke, basic, m4: can deploy indexers and search head cluster", func() { - - siteCount := 3 - err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, "") - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + testenv.RunM4DeploymentWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName(), 1, 3, "") }) }) - Context("Multisite cluster deployment (M1 - multisite indexer cluster)", func() { + Context("Multisite cluster deployment (M1 - Multisite Indexer Cluster)", func() { It("smoke, basic: can deploy multisite indexers cluster", func() { - - siteCount := 3 - err := deployment.DeployMultisiteCluster(ctx, deployment.GetName(), 1, siteCount, "") - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + testenv.RunM1DeploymentWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName(), 1, 3) }) }) Context("Standalone deployment (S1) with Service Account", func() { It("smoke, basic, s1: can deploy a standalone instance attached to a service account", func() { - // Create Service Account serviceAccountName := "smoke-service-account" - testcaseEnvInst.CreateServiceAccount(serviceAccountName) - - standaloneSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ServiceAccount: serviceAccountName, - }, - } - - // Create standalone Deployment with License Manager - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify serviceAccount is configured on Pod - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyServiceAccountConfiguredOnPod(deployment, testcaseEnvInst.GetName(), standalonePodName, serviceAccountName) + testenv.RunStandaloneWithServiceAccountWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName(), serviceAccountName) }) }) }) diff --git a/test/testenv/appframework_utils.go b/test/testenv/appframework_utils.go index a60554bd6..6b6f08ef5 100644 --- a/test/testenv/appframework_utils.go +++ b/test/testenv/appframework_utils.go @@ -109,7 +109,7 @@ var PVTestAppsLocation = "appframework/100mb_sample_apps/" // AppStagingLocOnPod is the volume on Splunk pod where apps will be copied from operator var AppStagingLocOnPod = "/operator-staging/appframework/" -// AppDownloadVolume location on Operator pod for App framework downloads +// AppDownloadVolume location on Operator pod for App Framework downloads var AppDownloadVolume = "/opt/splunk/appframework/downloadedApps/" // GenerateAppSourceSpec return AppSourceSpec struct with given values @@ -125,7 +125,7 @@ func GenerateAppSourceSpec(appSourceName string, appSourceLocation string, appSo func GetPodAppStatus(ctx context.Context, deployment *Deployment, podName string, ns string, appname string, clusterWideInstall bool) (string, string, error) { // For clusterwide install do not check for versions on deployer and cluster-manager as the apps arent installed there if clusterWideInstall && (strings.Contains(podName, "-cluster-manager-") || strings.Contains(podName, splcommon.TestClusterManagerDashed) || strings.Contains(podName, "-deployer-")) { - logf.Log.Info("Pod skipped as install is Cluter-wide", "PodName", podName) + logf.Log.Info("Pod skipped as install is Cluter-wide", "podName", podName) return "", "", nil } output, err := GetPodAppInstallStatus(ctx, deployment, podName, ns, appname) @@ -154,11 +154,11 @@ func GetPodInstalledAppVersion(deployment *Deployment, podName string, ns string } } filePath := fmt.Sprintf("/opt/splunk/%s/%s/default/app.conf", path, appname) - logf.Log.Info("Check app version", "App", appname, "Conf file", filePath) + logf.Log.Info("Check app version", "app", appname, "confFile", filePath) confline, err := GetConfLineFromPod(podName, filePath, ns, "version", "launcher", true) if err != nil { - logf.Log.Error(err, "Failed to get version from pod", "Pod Name", podName) + logf.Log.Error(err, "Failed to get version from pod", "podName", podName) return "", err } version := strings.TrimSpace(strings.Split(confline, "=")[1]) @@ -181,7 +181,7 @@ func GetPodAppInstallStatus(ctx context.Context, deployment *Deployment, podName return "", err } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout) + logf.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout) return strings.TrimSuffix(stdout, "\n"), nil } @@ -195,7 +195,7 @@ func GetPodAppbtoolStatus(ctx context.Context, deployment *Deployment, podName s logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command, "stdin", stdin) return "", err } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) if len(stdout) > 0 { if strings.Contains(strings.Split(stdout, "\n")[0], "App is disabled") { @@ -218,13 +218,13 @@ func GetAppFileList(appList []string) []string { // GetAppframeworkManualUpdateConfigMap gets config map for given manual update configmap func GetAppframeworkManualUpdateConfigMap(ctx context.Context, deployment *Deployment, ns string) (*corev1.ConfigMap, error) { ConfigMapName := fmt.Sprintf(AppframeworkManualUpdateConfigMap, ns) - logf.Log.Info("Get config map for", "CONFIG MAP NAME", ConfigMapName) + logf.Log.Info("Get config map for", "configMapName", ConfigMapName) ConfigMap, err := GetConfigMap(ctx, deployment, ns, ConfigMapName) if err != nil { logf.Log.Error(err, "Failed to get splunk manual poll Config Map") return ConfigMap, err } - logf.Log.Info("Config Map contents", "CONFIG MAP NAME", ConfigMapName, "Data", ConfigMap.Data) + logf.Log.Info("Config Map contents", "configMapName", ConfigMapName, "data", ConfigMap.Data) return ConfigMap, err } @@ -234,19 +234,19 @@ func GetAppDeploymentInfoStandalone(ctx context.Context, deployment *Deployment, appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, standalone) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := standalone.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Standalone AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Standalone Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Standalone AppInfo Struct", "appName", appName, "appSource", appSourceName, "standaloneName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Standalone Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "standaloneName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Standalone Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "standaloneName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } @@ -256,19 +256,19 @@ func GetAppDeploymentInfoIngestorCluster(ctx context.Context, deployment *Deploy appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, ingestor) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := ingestor.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Ingestor AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Ingestor Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Ingestor AppInfo Struct", "appName", appName, "appSource", appSourceName, "ingestorName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Ingestor Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "ingestorName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Ingestor Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "ingestorName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } @@ -278,19 +278,19 @@ func GetAppDeploymentInfoMonitoringConsole(ctx context.Context, deployment *Depl appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, mc) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := mc.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Monitoring Console AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Monitoring Console Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Monitoring Console AppInfo Struct", "appName", appName, "appSource", appSourceName, "monitoringConsoleName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Monitoring Console Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "monitoringConsoleName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Monitoring Console Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "monitoringConsoleName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } @@ -300,41 +300,41 @@ func GetAppDeploymentInfoClusterManager(ctx context.Context, deployment *Deploym appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, cm) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := cm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Cluster Manager AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Cluster Manager Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Cluster Manager AppInfo Struct", "appName", appName, "appSource", appSourceName, "clusterManagerName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Cluster Manager Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "clusterManagerName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Cluster Manager Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "clusterManagerName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } -// GetAppDeploymentInfoClusterMaster returns AppDeploymentInfo for given Cluster Manager, appSourceName and appName +// GetAppDeploymentInfoClusterMaster returns AppDeploymentInfo for given Cluster Master, appSourceName and appName func GetAppDeploymentInfoClusterMaster(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { cm := &enterpriseApiV3.ClusterMaster{} appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, cm) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := cm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Cluster Master AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Cluster Master Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Cluster Master AppInfo Struct", "appName", appName, "appSource", appSourceName, "clusterMasterName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Cluster Master Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "clusterMasterName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Cluster Master Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "clusterMasterName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } @@ -344,19 +344,19 @@ func GetAppDeploymentInfoSearchHeadCluster(ctx context.Context, deployment *Depl appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, cm) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := cm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Search Head Cluster AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Search Head Name Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Search Head Cluster AppInfo Struct", "appName", appName, "appSource", appSourceName, "searchHeadClusterName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Search Head Name Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "searchHeadClusterName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Search Head Name Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "searchHeadClusterName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } @@ -389,7 +389,7 @@ func GetAppDeploymentInfo(ctx context.Context, deployment *Deployment, testenvIn func GenerateAppFrameworkSpec(ctx context.Context, testenvInstance *TestCaseEnv, volumeName string, scope string, appSourceName string, s3TestDir string, pollInterval int) enterpriseApi.AppFrameworkSpec { var volumeSpec []enterpriseApi.VolumeSpec - // Create App framework volume + // Create App Framework volume switch ClusterProvider { case "eks": volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetS3Endpoint(), testenvInstance.GetIndexSecretName(), "aws", "s3", GetDefaultS3Region())} @@ -404,7 +404,7 @@ func GenerateAppFrameworkSpec(ctx context.Context, testenvInstance *TestCaseEnv, volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetGCPEndpoint(), testenvInstance.GetIndexSecretName(), "gcp", "gcs", GetDefaultS3Region())} default: - testenvInstance.Log.Info("Failed to identify cluster provider name: Should be 'eks' or 'azure' or 'gcp' ") + testenvInstance.Log.Info("Failed to identify provider: Should be 'eks' or 'azure' or 'gcp'") } // AppSourceDefaultSpec: Remote Storage volume name and Scope of App deployment @@ -601,7 +601,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.Standalone{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -609,7 +609,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.MonitoringConsole{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -617,7 +617,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.SearchHeadCluster{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -625,7 +625,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApiV3.ClusterMaster{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -633,7 +633,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.ClusterManager{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -641,7 +641,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApiV3.LicenseMaster{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -649,7 +649,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.LicenseManager{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress diff --git a/test/testenv/assertion_helpers.go b/test/testenv/assertion_helpers.go new file mode 100644 index 000000000..c274a5dc4 --- /dev/null +++ b/test/testenv/assertion_helpers.go @@ -0,0 +1,213 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + + . "github.com/onsi/gomega" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + corev1 "k8s.io/api/core/v1" +) + +// ScaleSearchHeadCluster scales a Search Head Cluster to the specified replica count +func (testcaseenv *TestCaseEnv) ScaleSearchHeadCluster(ctx context.Context, deployment *Deployment, deploymentName string, newReplicas int) { + shcName := deploymentName + "-shc" + + // Get instance of current SHC CR with latest config + shc := &enterpriseApi.SearchHeadCluster{} + GetInstanceWithExpect(ctx, deployment, shc, shcName, "Failed to get instance of Search Head Cluster") + + // Update Replicas of SHC + shc.Spec.Replicas = int32(newReplicas) + UpdateCRWithExpect(ctx, deployment, shc, "Failed to scale Search Head Cluster") + + // Verify Search Head Cluster scales up and goes to ScalingUp phase + testcaseenv.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) +} + +// ScaleIndexerCluster scales an Indexer Cluster to the specified replica count +func (testcaseenv *TestCaseEnv) ScaleIndexerCluster(ctx context.Context, deployment *Deployment, deploymentName string, newReplicas int) { + idxcName := deploymentName + "-idxc" + + // Get instance of current Indexer CR with latest config + idxc := &enterpriseApi.IndexerCluster{} + GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") + + // Update Replicas of Indexer Cluster + idxc.Spec.Replicas = int32(newReplicas) + UpdateCRWithExpect(ctx, deployment, idxc, "Failed to scale Indexer Cluster") + + // Verify Indexer Cluster scales up and goes to ScalingUp phase + testcaseenv.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) +} + +// UpdateMonitoringConsoleRef updates the MonitoringConsoleRef in a CR and waits for the change to apply +func UpdateMonitoringConsoleRefAndVerify(ctx context.Context, deployment *Deployment, testcaseenv *TestCaseEnv, obj interface{}, instanceName string, newMCName string) { + // Get current resource version before update + resourceVersion := testcaseenv.GetResourceVersion(ctx, deployment, obj) + + // Update the MonitoringConsoleRef based on the type + switch cr := obj.(type) { + case *enterpriseApi.ClusterManager: + GetInstanceWithExpect(ctx, deployment, cr, instanceName, "Failed to get instance") + cr.Spec.MonitoringConsoleRef.Name = newMCName + UpdateCRWithExpect(ctx, deployment, cr, "Failed to update MonitoringConsoleRef") + case *enterpriseApi.SearchHeadCluster: + GetInstanceWithExpect(ctx, deployment, cr, instanceName, "Failed to get instance") + cr.Spec.MonitoringConsoleRef.Name = newMCName + UpdateCRWithExpect(ctx, deployment, cr, "Failed to update MonitoringConsoleRef") + } + + // Wait for custom resource version to change + testcaseenv.VerifyCustomResourceVersionChanged(ctx, deployment, obj, resourceVersion) +} + +// VerifyMCConfigForC3Cluster verifies the standard MC configuration for a C3 cluster +func (testcaseenv *TestCaseEnv) VerifyMCConfigForC3Cluster(ctx context.Context, deployment *Deployment, deploymentName string, mcName string, shReplicas int, indexerReplicas int, shouldExist bool) { + // Check Cluster Manager in MC Config Map + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(ClusterMasterServiceName, deploymentName)}, "SPLUNK_CLUSTER_MASTER_URL", mcName, shouldExist) + + // Check Deployer in MC Config Map + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(DeployerServiceName, deploymentName)}, "SPLUNK_DEPLOYER_URL", mcName, shouldExist) + + // Check Search Head pods in MC Config Map + shPods := GeneratePodNameSlice(SearchHeadPod, deploymentName, shReplicas, false, 0) + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, shouldExist) + + // Check Search Heads in MC Pod config string + testcaseenv.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, shouldExist, false) + + // Check Indexers in MC Pod config string + indexerPods := GeneratePodNameSlice(IndexerPod, deploymentName, indexerReplicas, false, 0) + testcaseenv.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, shouldExist, true) +} + +// VerifyMCConfigForM4Cluster verifies the standard MC configuration for an M4 multisite cluster +func (testcaseenv *TestCaseEnv) VerifyMCConfigForM4Cluster(ctx context.Context, deployment *Deployment, deploymentName string, mcName string, shReplicas int, indexerReplicas int, siteCount int, shouldExist bool) { + // Check cluster manager in MC Config Map + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(ClusterMasterServiceName, deploymentName)}, "SPLUNK_CLUSTER_MASTER_URL", mcName, shouldExist) + + // Check deployer in MC Config Map + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(DeployerServiceName, deploymentName)}, "SPLUNK_DEPLOYER_URL", mcName, shouldExist) + + // Check Search Head Pods in MC Config Map + shPods := GeneratePodNameSlice(SearchHeadPod, deploymentName, shReplicas, false, 0) + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, shouldExist) + + // Check Search Heads in MC Pod config string + testcaseenv.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, shouldExist, false) + + // Check Indexers in MC Pod config string + indexerPods := GeneratePodNameSlice(MultiSiteIndexerPod, deploymentName, indexerReplicas, true, siteCount) + testcaseenv.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, shouldExist, true) +} + +// DeployAndVerifyC3WithMC deploys a C3 cluster with a given MC and verifies all components are ready +func (testcaseenv *TestCaseEnv) DeployAndVerifyC3WithMC(ctx context.Context, deployment *Deployment, deploymentName string, indexerReplicas int, mcName string) { + err := deployment.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, deploymentName, indexerReplicas, true, mcName) + Expect(err).To(Succeed(), "Unable to deploy Cluster Master") + + // Verify all components are ready + testcaseenv.VerifyClusterMasterReady(ctx, deployment) + testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) + testcaseenv.VerifySingleSiteIndexersReady(ctx, deployment) +} + +// DeployStandaloneWithMCRef deploys a standalone instance with a MonitoringConsoleRef +func (testcaseenv *TestCaseEnv) DeployStandaloneWithMCRef(ctx context.Context, deployment *Deployment, deploymentName string, mcName string) *enterpriseApi.Standalone { + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseenv.GetSplunkImage(), + }, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: mcName, + }, + }, + } + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deploymentName, spec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + // Wait for Standalone to be in READY status + testcaseenv.VerifyStandaloneReady(ctx, deployment, deploymentName, standalone) + + return standalone +} + +// VerifyStandaloneInMC verifies that a standalone instance is configured in the MC +func (testcaseenv *TestCaseEnv) VerifyStandaloneInMC(ctx context.Context, deployment *Deployment, deploymentName string, mcName string, shouldExist bool) { + standalonePod := fmt.Sprintf(StandalonePod, deploymentName, 0) + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, []string{standalonePod}, "SPLUNK_STANDALONE_URL", mcName, shouldExist) + testcaseenv.VerifyPodsInMCConfigString(ctx, deployment, []string{standalonePod}, mcName, shouldExist, false) +} + +// VerifyLMConfiguredOnPods verifies License Manager is configured on all given pods +func VerifyLMConfiguredOnPods(ctx context.Context, deployment *Deployment, podNames []string) { + for _, podName := range podNames { + VerifyLMConfiguredOnPod(ctx, deployment, podName) + } +} + +// IngestDataOnIndexers ingests test data on all indexer pods +func IngestDataOnIndexers(ctx context.Context, deployment *Deployment, deploymentName string, indexerCount int) { + for i := 0; i < indexerCount; i++ { + podName := fmt.Sprintf(IndexerPod, deploymentName, i) + logFile := fmt.Sprintf("test-log-%s.log", RandomDNSName(3)) + CreateMockLogfile(logFile, LogLineCount) + IngestFileViaMonitor(ctx, logFile, DefaultIngestIndex, podName, deployment) + } +} + +// VerifyM1ClusterReady verifies the cluster coordinator, indexers, and multisite status are ready (no SHC). +func (testcaseenv *TestCaseEnv) VerifyM1ClusterReady(ctx context.Context, deployment *Deployment, siteCount int, verifyCoordinator func(context.Context, *Deployment)) { + verifyCoordinator(ctx, deployment) + testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseenv.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) +} + +// VerifyM4ClusterReady verifies the cluster coordinator, indexers, multisite status, and SHC are ready. +func (testcaseenv *TestCaseEnv) VerifyM4ClusterReady(ctx context.Context, deployment *Deployment, siteCount int, verifyCoordinator func(context.Context, *Deployment)) { + verifyCoordinator(ctx, deployment) + testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseenv.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) +} + +// VerifyM4IndexersAndSHCReady verifies the cluster coordinator, indexers, and SHC are ready (without multisite check). +func (testcaseenv *TestCaseEnv) VerifyM4IndexersAndSHCReady(ctx context.Context, deployment *Deployment, siteCount int, verifyCoordinator func(context.Context, *Deployment)) { + verifyCoordinator(ctx, deployment) + testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) +} + +// VerifyC3ClusterReady verifies the cluster coordinator, SHC, and single-site indexers are ready. +func (testcaseenv *TestCaseEnv) VerifyC3ClusterReady(ctx context.Context, deployment *Deployment, verifyCoordinator func(context.Context, *Deployment)) { + verifyCoordinator(ctx, deployment) + testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) + testcaseenv.VerifySingleSiteIndexersReady(ctx, deployment) +} + +// IngestDataOnMultisiteIndexers ingests test data on all multisite indexer pods +func IngestDataOnMultisiteIndexers(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int) { + for site := 1; site <= siteCount; site++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, site, 0) + logFile := fmt.Sprintf("test-log-%s.log", RandomDNSName(3)) + CreateMockLogfile(logFile, LogLineCount) + IngestFileViaMonitor(ctx, logFile, DefaultIngestIndex, podName, deployment) + } +} diff --git a/test/testenv/azureutils.go b/test/testenv/azureutils.go index 84a3f0601..04491d808 100644 --- a/test/testenv/azureutils.go +++ b/test/testenv/azureutils.go @@ -338,7 +338,7 @@ func (client *AzureBlobClient) DownloadFileFromAzure(ctx context.Context, downlo return "", err } - logf.Log.Info("Download from Azure successful:", "File", downloadRequest.RemoteFile) + logf.Log.Info("Download from Azure successful:", "file", downloadRequest.RemoteFile) return localFile.Name(), err } @@ -354,7 +354,7 @@ func DownloadFilesFromAzure(ctx context.Context, endPoint, accountKey, accountNa } _, err := azureBlobClient.DownloadFileFromAzure(ctx, downloadRequest, endPoint, StorageAccountKey, StorageAccount) if err != nil { - logf.Log.Error(err, "Unable to download file", "File Name", key) + logf.Log.Error(err, "Unable to download file", "fileName", key) return err } } @@ -374,7 +374,7 @@ func DownloadLicenseFromAzure(ctx context.Context, downloadDir string) (string, azureBlobClient := &AzureBlobClient{} filename, err := azureBlobClient.DownloadFileFromAzure(ctx, downloadRequest, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount) if err != nil { - logf.Log.Error(err, "Unable to download license file", "File", filename) + logf.Log.Error(err, "Unable to download license file", "file", filename) } return filename, err } @@ -431,7 +431,7 @@ func UploadFilesToAzure(ctx context.Context, accountName, accountKey, uploadFrom fileFullPath := "https://" + StorageAccount + ".blob.core.windows.net" + "/" + azureIndexesContainer + "/" + containerName + "/" + key fileName, err := UploadFileToAzure(ctx, accountName, accountKey, fileFullPath, fileLocation) if err != nil { - logf.Log.Error(err, "Unable to upload file", "File name", key) + logf.Log.Error(err, "Unable to upload file", "fileName", key) return nil, err } uploadedFiles = append(uploadedFiles, fileName) diff --git a/test/testenv/cmutil.go b/test/testenv/cmutil.go index a1b71f3ce..e9b251ffb 100644 --- a/test/testenv/cmutil.go +++ b/test/testenv/cmutil.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -44,7 +44,7 @@ type ClusterManagerSitesContent struct { Peers map[string]ClusterManagerSitesPeer `json:"peers"` } -// ClusterManagerSitesPeer reprensents an indexer peer member of a site +// ClusterManagerSitesPeer represents an indexer peer member of a site type ClusterManagerSitesPeer struct { ServerName string `json:"server_name"` } @@ -158,7 +158,7 @@ func CheckIndexerOnCM(ctx context.Context, deployment *Deployment, indexerName s restResponse := GetIndexersOrSearchHeadsOnCM(ctx, deployment, "peer") found := false for _, entry := range restResponse.Entry { - logf.Log.Info("Peer found On CM", "Indexer Name", entry.Content.Label, "Status", entry.Content.Status) + logf.Log.Info("Peer found On CM", "indexerName", entry.Content.Label, "status", entry.Content.Status) if entry.Content.Label == indexerName { found = true break @@ -172,7 +172,7 @@ func CheckSearchHeadOnCM(ctx context.Context, deployment *Deployment, searchHead restResponse := GetIndexersOrSearchHeadsOnCM(ctx, deployment, "sh") found := false for _, entry := range restResponse.Entry { - logf.Log.Info("Search Head On CM", "Search Head", entry.Content.Label, "Status", entry.Content.Status) + logf.Log.Info("Search Head On CM", "searchHead", entry.Content.Label, "status", entry.Content.Status) if entry.Content.Label == searchHeadName { found = true break @@ -186,7 +186,7 @@ func CheckSearchHeadRemoved(ctx context.Context, deployment *Deployment) bool { restResponse := GetIndexersOrSearchHeadsOnCM(ctx, deployment, "sh") searchHeadRemoved := true for _, entry := range restResponse.Entry { - logf.Log.Info("Search Found", "Search Head", entry.Content.Label, "Status", entry.Content.Status) + logf.Log.Info("Search Found", "searchHead", entry.Content.Label, "status", entry.Content.Status) if entry.Content.Status == "Disconnected" { searchHeadRemoved = false } @@ -194,21 +194,6 @@ func CheckSearchHeadRemoved(ctx context.Context, deployment *Deployment) bool { return searchHeadRemoved } -// RollHotBuckets roll hot buckets in cluster -func RollHotBuckets(ctx context.Context, deployment *Deployment) bool { - podName := fmt.Sprintf("splunk-%s-%s-0", deployment.GetName(), "cluster-manager") - stdin := "/opt/splunk/bin/splunk rolling-restart cluster-peers -auth admin:$(cat /mnt/splunk-secrets/password)" - command := []string{"/bin/sh"} - stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) - if err != nil { - logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) - return false - } - logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - - return strings.Contains(stdout, "Rolling restart of all cluster peers has been initiated.") -} - // ClusterManagerInfoEndpointResponse is represtentation of /services/cluster/manager/info endpoint type ClusterManagerInfoEndpointResponse struct { Entry []struct { @@ -275,7 +260,7 @@ func CMBundlePushstatus(ctx context.Context, deployment *Deployment, previousBun // Check if new bundle was pushed by comparing hash if previousBundleHash != "" { if entry.Content.BundleID == previousBundleHash { - logf.Log.Info("Bundle hash not updated", "old Bundle hash", previousBundleHash, "new Bundle hash", entry.Content.BundleID) + logf.Log.Info("Bundle hash not updated", "oldBundleHash", previousBundleHash, "newBundleHash", entry.Content.BundleID) continue } } @@ -298,6 +283,6 @@ func GetClusterManagerBundleHash(ctx context.Context, deployment *Deployment, cr restResponse := ClusterManagerInfoResponse(ctx, deployment, podName) bundleHash := restResponse.Entry[0].Content.ActiveBundle.Checksum - logf.Log.Info("Bundle Hash on Cluster Manager Found", "Hash", bundleHash) + logf.Log.Info("Bundle Hash on Cluster Manager Found", "hash", bundleHash) return bundleHash } diff --git a/test/testenv/common_test_patterns.go b/test/testenv/common_test_patterns.go new file mode 100644 index 000000000..6c2aa043a --- /dev/null +++ b/test/testenv/common_test_patterns.go @@ -0,0 +1,271 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + + . "github.com/onsi/gomega" + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/splunk/enterprise" + corev1 "k8s.io/api/core/v1" +) + +// ClusterReadinessConfig holds v3/v4 API version callbacks for cluster and License Manager +// readiness verification. Shared across test packages to avoid per-package duplication. +type ClusterReadinessConfig struct { + LicenseManagerReady func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) + ClusterManagerReady func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) + APIVersion string +} + +// NewClusterReadinessConfigV3 creates a ClusterReadinessConfig for v3 API (LicenseMaster/ClusterMaster) +func NewClusterReadinessConfigV3() *ClusterReadinessConfig { + return &ClusterReadinessConfig{ + LicenseManagerReady: func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + testcaseEnv.VerifyLicenseMasterReady(ctx, deployment) + }, + ClusterManagerReady: func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + testcaseEnv.VerifyClusterMasterReady(ctx, deployment) + }, + APIVersion: "v3", + } +} + +// NewClusterReadinessConfigV4 creates a ClusterReadinessConfig for v4 API (LicenseManager/ClusterManager) +func NewClusterReadinessConfigV4() *ClusterReadinessConfig { + return &ClusterReadinessConfig{ + LicenseManagerReady: func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + testcaseEnv.VerifyLicenseManagerReady(ctx, deployment) + }, + ClusterManagerReady: func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + testcaseEnv.VerifyClusterManagerReady(ctx, deployment) + }, + APIVersion: "v4", + } +} + +// DeployStandaloneWithLM deploys a standalone with the appropriate License Manager type for +// the API version: LicenseMaster (v3) or LicenseManager (v4). +func (c *ClusterReadinessConfig) DeployStandaloneWithLM(ctx context.Context, deployment *Deployment, name, mcRef string) (*enterpriseApi.Standalone, error) { + if c.APIVersion == "v3" { + return deployment.DeployStandaloneWithLMaster(ctx, name, mcRef) + } + return deployment.DeployStandaloneWithLM(ctx, name, mcRef) +} + +// DeployMultisiteCluster deploys a multisite cluster with the appropriate Cluster Manager type +// for the API version: ClusterMaster (v3) or ClusterManager (v4). +func (c *ClusterReadinessConfig) DeployMultisiteCluster(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, mcRef string) error { + if c.APIVersion == "v3" { + return deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) + } + return deployment.DeployMultisiteClusterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) +} + +// VerifyClusterManagerPhaseUpdating asserts the Cluster Manager (or ClusterMaster for v3) +// has entered the Updating phase. +func (c *ClusterReadinessConfig) VerifyClusterManagerPhaseUpdating(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + if c.APIVersion == "v3" { + testcaseEnv.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) + } else { + testcaseEnv.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) + } +} + +// ClusterManagerPVCType returns the PVC label fragment for the Cluster Manager: +// "cluster-master" for v3, "cluster-manager" for v4. +func (c *ClusterReadinessConfig) ClusterManagerPVCType() string { + if c.APIVersion == "v3" { + return "cluster-master" + } + return "cluster-manager" +} + +// DeleteClusterManager fetches and deletes the Cluster Manager CR for the appropriate API version. +func (c *ClusterReadinessConfig) DeleteClusterManager(ctx context.Context, deployment *Deployment) { + name := deployment.GetName() + if c.APIVersion == "v3" { + cm := &enterpriseApiV3.ClusterMaster{} + err := deployment.GetInstance(ctx, name, cm) + Expect(err).To(Succeed(), "Unable to GET Cluster Master instance", "Cluster Master Name", cm) + err = deployment.DeleteCR(ctx, cm) + Expect(err).To(Succeed(), "Unable to delete Cluster Master instance", "Cluster Master Name", cm) + } else { + cm := &enterpriseApi.ClusterManager{} + err := deployment.GetInstance(ctx, name, cm) + Expect(err).To(Succeed(), "Unable to GET Cluster Manager instance", "Cluster Manager Name", cm) + err = deployment.DeleteCR(ctx, cm) + Expect(err).To(Succeed(), "Unable to delete Cluster Manager instance", "Cluster Manager Name", cm) + } +} + +// DeployMultisiteClusterWithIndexes deploys a multisite cluster with SmartStore indexes using +// the appropriate Cluster Manager type for the API version. +func (c *ClusterReadinessConfig) DeployMultisiteClusterWithIndexes(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, secretName string, smartStoreSpec enterpriseApi.SmartStoreSpec) error { + if c.APIVersion == "v3" { + return deployment.DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx, name, indexerReplicas, siteCount, secretName, smartStoreSpec) + } + return deployment.DeployMultisiteClusterWithSearchHeadAndIndexes(ctx, name, indexerReplicas, siteCount, secretName, smartStoreSpec) +} + +// GetBundleHash returns the current bundle hash for the Cluster Manager (or ClusterMaster for v3). +func (c *ClusterReadinessConfig) GetBundleHash(ctx context.Context, deployment *Deployment) string { + if c.APIVersion == "v3" { + return GetClusterManagerBundleHash(ctx, deployment, "ClusterMaster") + } + return GetClusterManagerBundleHash(ctx, deployment, "ClusterManager") +} + +// AppendSmartStoreIndex appends a new SmartStore index to the Cluster Manager CR +// for the appropriate API version. +func (c *ClusterReadinessConfig) AppendSmartStoreIndex(ctx context.Context, deployment *Deployment, newIndex []enterpriseApi.IndexSpec) { + name := deployment.GetName() + if c.APIVersion == "v3" { + cm := &enterpriseApiV3.ClusterMaster{} + err := deployment.GetInstance(ctx, name, cm) + Expect(err).To(Succeed(), "Failed to get instance of Cluster Master") + cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) + err = deployment.UpdateCR(ctx, cm) + Expect(err).To(Succeed(), "Failed to add new index to Cluster Master") + } else { + cm := &enterpriseApi.ClusterManager{} + err := deployment.GetInstance(ctx, name, cm) + Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") + cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) + err = deployment.UpdateCR(ctx, cm) + Expect(err).To(Succeed(), "Failed to add new index to Cluster Manager") + } +} + +// DeployMCAndGetVersion deploys and verifies a Monitoring Console, then returns both the MC +// instance and its current resource version. +func (testcaseenv *TestCaseEnv) DeployMCAndGetVersion(ctx context.Context, deployment *Deployment, name string, lmRef string) (*enterpriseApi.MonitoringConsole, string) { + mc := testcaseenv.DeployAndVerifyMonitoringConsole(ctx, deployment, name, lmRef) + resourceVersion := testcaseenv.GetResourceVersion(ctx, deployment, mc) + return mc, resourceVersion +} + +// DeployAndVerifyStandalone deploys a standalone instance and verifies it reaches ready state +func (testcaseenv *TestCaseEnv) DeployAndVerifyStandalone(ctx context.Context, deployment *Deployment, name string, mcRef string, licenseManagerRef string) *enterpriseApi.Standalone { + standalone, err := deployment.DeployStandalone(ctx, name, mcRef, licenseManagerRef) + Expect(err).To(Succeed(), "Unable to deploy Standalone instance") + + testcaseenv.VerifyStandaloneReady(ctx, deployment, name, standalone) + return standalone +} + +// DeployAndVerifyMonitoringConsole deploys a Monitoring Console and verifies it reaches ready state +func (testcaseenv *TestCaseEnv) DeployAndVerifyMonitoringConsole(ctx context.Context, deployment *Deployment, name string, licenseManagerRef string) *enterpriseApi.MonitoringConsole { + mc, err := deployment.DeployMonitoringConsole(ctx, name, licenseManagerRef) + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") + + testcaseenv.VerifyMonitoringConsoleReady(ctx, deployment, name, mc) + return mc +} + +// VerifyIndexerCPULimits verifies CPU limits on all indexer pods in a single-site cluster +func (testcaseenv *TestCaseEnv) VerifyIndexerCPULimits(deployment *Deployment, deploymentName string, indexerCount int, expectedCPULimit string) { + for i := 0; i < indexerCount; i++ { + podName := fmt.Sprintf(IndexerPod, deploymentName, i) + testcaseenv.VerifyCPULimits(deployment, podName, expectedCPULimit) + } +} + +// VerifySearchHeadCPULimits verifies CPU limits on all search head pods +func (testcaseenv *TestCaseEnv) VerifySearchHeadCPULimits(deployment *Deployment, deploymentName string, searchHeadCount int, expectedCPULimit string) { + for i := 0; i < searchHeadCount; i++ { + podName := fmt.Sprintf(SearchHeadPod, deploymentName, i) + testcaseenv.VerifyCPULimits(deployment, podName, expectedCPULimit) + } +} + +// VerifyM4ComponentsReady verifies multisite indexers, multisite status, and SHC are ready (without CM check or RFSF). +func (testcaseenv *TestCaseEnv) VerifyM4ComponentsReady(ctx context.Context, deployment *Deployment, siteCount int) { + testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseenv.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) +} + +// VerifyMCVersionChangedAndReady waits for the MC resource version to change then verifies MC is ready. +func (testcaseenv *TestCaseEnv) VerifyMCVersionChangedAndReady(ctx context.Context, deployment *Deployment, mc *enterpriseApi.MonitoringConsole, resourceVersion string) { + testcaseenv.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + testcaseenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) +} + +// VerifyClusterReadyAndRFSF is a common verification pattern that checks cluster is ready and RF/SF is met +func (testcaseenv *TestCaseEnv) VerifyClusterReadyAndRFSF(ctx context.Context, deployment *Deployment) { + testcaseenv.VerifyC3ClusterReady(ctx, deployment, testcaseenv.VerifyClusterManagerReady) + testcaseenv.VerifyRFSFMet(ctx, deployment) +} + +// VerifyMultisiteClusterReadyAndRFSF is a common verification pattern for multisite clusters +func (testcaseenv *TestCaseEnv) VerifyMultisiteClusterReadyAndRFSF(ctx context.Context, deployment *Deployment, siteCount int) { + testcaseenv.VerifyClusterManagerReady(ctx, deployment) + testcaseenv.VerifyM4ComponentsReady(ctx, deployment, siteCount) + testcaseenv.VerifyRFSFMet(ctx, deployment) +} + +// TriggerAndVerifyTelemetry is a common pattern for telemetry verification +func (testcaseenv *TestCaseEnv) TriggerAndVerifyTelemetry(ctx context.Context, deployment *Deployment, prevSubmissionTime string) { + testcaseenv.TriggerTelemetrySubmission(ctx, deployment) + testcaseenv.VerifyTelemetry(ctx, deployment, prevSubmissionTime) +} + +// VerifyProbeConfigAndScripts verifies probe config map exists and probe scripts are present on all pods. +// If includeStartup is true, the startup probe script is also checked. +func (testcaseenv *TestCaseEnv) VerifyProbeConfigAndScripts(ctx context.Context, deployment *Deployment, includeStartup bool) { + testcaseenv.Log.Info("Get config map for livenessProbe and readinessProbe") + configMapName := enterprise.GetProbeConfigMapName(testcaseenv.GetName()) + _, err := GetConfigMap(ctx, deployment, testcaseenv.GetName(), configMapName) + Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", configMapName) + scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName()} + if includeStartup { + scriptsNames = append(scriptsNames, enterprise.GetStartupScriptName()) + } + allPods := DumpGetPods(testcaseenv.GetName()) + testcaseenv.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) +} + +// NewStandaloneSpecWithMCRef creates a StandaloneSpec with a MonitoringConsoleRef set to the given MC name. +func NewStandaloneSpecWithMCRef(image string, mcName string) enterpriseApi.StandaloneSpec { + return enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: image, + }, + Volumes: []corev1.Volume{}, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: mcName, + }, + }, + } +} + +// VerifyLMConfiguredOnMC verifies that the License Manager is configured on the Monitoring Console pod. +func VerifyLMConfiguredOnMC(ctx context.Context, deployment *Deployment) { + monitoringConsolePodName := fmt.Sprintf(MonitoringConsolePod, deployment.GetName()) + VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) +} + +// StandardC3Verification performs the standard set of verifications for a C3 cluster +// This includes cluster ready, RF/SF met, and monitoring console ready +func (testcaseenv *TestCaseEnv) StandardC3Verification(ctx context.Context, deployment *Deployment, mcName string, mc *enterpriseApi.MonitoringConsole) { + testcaseenv.VerifyClusterReadyAndRFSF(ctx, deployment) + testcaseenv.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) +} diff --git a/test/testenv/cr_update_helpers.go b/test/testenv/cr_update_helpers.go new file mode 100644 index 000000000..492b26187 --- /dev/null +++ b/test/testenv/cr_update_helpers.go @@ -0,0 +1,34 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// GetInstanceWithExpect is a wrapper around GetInstance that includes Gomega expectations +func GetInstanceWithExpect(ctx context.Context, deployment *Deployment, obj client.Object, instanceName string, errorMsg string) { + err := deployment.GetInstance(ctx, instanceName, obj) + Expect(err).To(Succeed(), errorMsg, "Instance Name", instanceName) +} + +// UpdateCRWithExpect is a wrapper around UpdateCR that includes Gomega expectations +func UpdateCRWithExpect(ctx context.Context, deployment *Deployment, obj client.Object, errorMsg string) { + err := deployment.UpdateCR(ctx, obj) + Expect(err).To(Succeed(), errorMsg) +} diff --git a/test/testenv/deployment.go b/test/testenv/deployment.go index 395722983..17059fb18 100644 --- a/test/testenv/deployment.go +++ b/test/testenv/deployment.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -154,7 +154,7 @@ func (d *Deployment) DeployStandalone(ctx context.Context, name string, mcRef st // If license file specified, deploy License Manager if LicenseManager != "" && d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return nil, err @@ -326,7 +326,7 @@ func (d *Deployment) OperatorPodExecCommand(ctx context.Context, podName string, return stdout.String(), stderr.String(), nil } -// DeployLicenseManager deploys the license manager instance +// DeployLicenseManager deploys the License Manager instance func (d *Deployment) DeployLicenseManager(ctx context.Context, name string) (*enterpriseApi.LicenseManager, error) { if d.testenv.licenseFilePath == "" { @@ -344,7 +344,7 @@ func (d *Deployment) DeployLicenseManager(ctx context.Context, name string) (*en return deployed.(*enterpriseApi.LicenseManager), err } -// DeployLicenseMaster deploys the license manager instance +// DeployLicenseMaster deploys the License Master instance func (d *Deployment) DeployLicenseMaster(ctx context.Context, name string) (*enterpriseApiV3.LicenseMaster, error) { if d.testenv.licenseFilePath == "" { @@ -362,9 +362,9 @@ func (d *Deployment) DeployLicenseMaster(ctx context.Context, name string) (*ent return deployed.(*enterpriseApiV3.LicenseMaster), err } -// DeployClusterManager deploys the cluster manager +// DeployClusterManager deploys the Cluster Manager func (d *Deployment) DeployClusterManager(ctx context.Context, name, LicenseManagerName string, ansibleConfig string, mcRef string) (*enterpriseApi.ClusterManager, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name, "LicenseRef", LicenseManagerName) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name, "LicenseRef", LicenseManagerName) cm := newClusterManager(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage) if mcRef != "" { cm.Spec.MonitoringConsoleRef = corev1.ObjectReference{ @@ -382,9 +382,9 @@ func (d *Deployment) DeployClusterManager(ctx context.Context, name, LicenseMana return deployed.(*enterpriseApi.ClusterManager), err } -// DeployClusterMaster deploys the cluster manager +// DeployClusterMaster deploys the Cluster Master func (d *Deployment) DeployClusterMaster(ctx context.Context, name, LicenseManagerName string, ansibleConfig string, mcRef string) (*enterpriseApiV3.ClusterMaster, error) { - d.testenv.Log.Info("Deploying cluster-master", "name", name, "LicenseRef", LicenseManagerName) + d.testenv.Log.Info("Deploying Cluster Master", "name", name, "LicenseRef", LicenseManagerName) cm := newClusterMaster(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage) if mcRef != "" { cm.Spec.MonitoringConsoleRef = corev1.ObjectReference{ @@ -402,9 +402,9 @@ func (d *Deployment) DeployClusterMaster(ctx context.Context, name, LicenseManag return deployed.(*enterpriseApiV3.ClusterMaster), err } -// DeployClusterManagerWithSmartStoreIndexes deploys the cluster manager with smartstore indexes +// DeployClusterManagerWithSmartStoreIndexes deploys the Cluster Manager with SmartStore indexes func (d *Deployment) DeployClusterManagerWithSmartStoreIndexes(ctx context.Context, name, LicenseManagerName string, ansibleConfig string, smartstorespec enterpriseApi.SmartStoreSpec) (*enterpriseApi.ClusterManager, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name) cm := newClusterManagerWithGivenIndexes(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage, smartstorespec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -416,9 +416,9 @@ func (d *Deployment) DeployClusterManagerWithSmartStoreIndexes(ctx context.Conte return deployed.(*enterpriseApi.ClusterManager), err } -// DeployClusterMasterWithSmartStoreIndexes deploys the cluster manager with smartstore indexes +// DeployClusterMasterWithSmartStoreIndexes deploys the Cluster Master with SmartStore indexes func (d *Deployment) DeployClusterMasterWithSmartStoreIndexes(ctx context.Context, name, LicenseManagerName, ansibleConfig string, smartstorespec enterpriseApi.SmartStoreSpec) (*enterpriseApiV3.ClusterMaster, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name) cm := newClusterMasterWithGivenIndexes(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage, smartstorespec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -430,12 +430,12 @@ func (d *Deployment) DeployClusterMasterWithSmartStoreIndexes(ctx context.Contex return deployed.(*enterpriseApiV3.ClusterMaster), err } -// DeployIndexerCluster deploys the indexer cluster +// DeployIndexerCluster deploys the Indexer Cluster func (d *Deployment) DeployIndexerCluster(ctx context.Context, name, LicenseManagerName string, count int, clusterManagerRef string, ansibleConfig string, queue, os corev1.ObjectReference, serviceAccountName string) (*enterpriseApi.IndexerCluster, error) { - d.testenv.Log.Info("Deploying indexer cluster", "name", name, "CM", clusterManagerRef) + d.testenv.Log.Info("Deploying Indexer Cluster", "name", name, "CM", clusterManagerRef) indexer := newIndexerCluster(name, d.testenv.namespace, LicenseManagerName, count, clusterManagerRef, ansibleConfig, d.testenv.splunkImage, queue, os, serviceAccountName) pdata, _ := json.Marshal(indexer) - d.testenv.Log.Info("indexer cluster spec", "cr", string(pdata)) + d.testenv.Log.Info("Indexer Cluster spec", "cr", string(pdata)) deployed, err := d.deployCR(ctx, name, indexer) if err != nil { return nil, err @@ -444,14 +444,14 @@ func (d *Deployment) DeployIndexerCluster(ctx context.Context, name, LicenseMana return deployed.(*enterpriseApi.IndexerCluster), err } -// DeployIngestorCluster deploys the ingestor cluster +// DeployIngestorCluster deploys the Ingestor Cluster func (d *Deployment) DeployIngestorCluster(ctx context.Context, name string, count int, queue, os corev1.ObjectReference, serviceAccountName string) (*enterpriseApi.IngestorCluster, error) { - d.testenv.Log.Info("Deploying ingestor cluster", "name", name) + d.testenv.Log.Info("Deploying Ingestor Cluster", "name", name) ingestor := newIngestorCluster(name, d.testenv.namespace, count, d.testenv.splunkImage, queue, os, serviceAccountName) pdata, _ := json.Marshal(ingestor) - d.testenv.Log.Info("ingestor cluster spec", "cr", string(pdata)) + d.testenv.Log.Info("Ingestor Cluster spec", "cr", string(pdata)) deployed, err := d.deployCR(ctx, name, ingestor) if err != nil { return nil, err @@ -492,13 +492,13 @@ func (d *Deployment) DeployObjectStorage(ctx context.Context, name string, objSt return deployed.(*enterpriseApi.ObjectStorage), err } -// DeployIngestorClusterWithAdditionalConfiguration deploys the ingestor cluster with additional configuration +// DeployIngestorClusterWithAdditionalConfiguration deploys the Ingestor Cluster with additional configuration func (d *Deployment) DeployIngestorClusterWithAdditionalConfiguration(ctx context.Context, ic *enterpriseApi.IngestorCluster) (*enterpriseApi.IngestorCluster, error) { - d.testenv.Log.Info("Deploying ingestor cluster with additional configuration", "name", ic.Name) + d.testenv.Log.Info("Deploying Ingestor Cluster with additional configuration", "name", ic.Name) pdata, _ := json.Marshal(ic) - d.testenv.Log.Info("ingestor cluster spec", "cr", string(pdata)) + d.testenv.Log.Info("Ingestor Cluster spec", "cr", string(pdata)) deployed, err := d.deployCR(ctx, ic.Name, ic) if err != nil { return nil, err @@ -507,9 +507,9 @@ func (d *Deployment) DeployIngestorClusterWithAdditionalConfiguration(ctx contex return deployed.(*enterpriseApi.IngestorCluster), err } -// DeploySearchHeadCluster deploys a search head cluster +// DeploySearchHeadCluster deploys a Search Head Cluster func (d *Deployment) DeploySearchHeadCluster(ctx context.Context, name, ClusterManagerRef, LicenseManagerName string, ansibleConfig string, mcRef string) (*enterpriseApi.SearchHeadCluster, error) { - d.testenv.Log.Info("Deploying search head cluster", "name", name) + d.testenv.Log.Info("Deploying Search Head Cluster", "name", name) sh := newSearchHeadCluster(name, d.testenv.namespace, ClusterManagerRef, LicenseManagerName, ansibleConfig, d.testenv.splunkImage) if mcRef != "" { sh.Spec.MonitoringConsoleRef = corev1.ObjectReference{ @@ -553,7 +553,7 @@ func (d *Deployment) deployCR(ctx context.Context, name string, cr client.Object return nil }) - // Returns once we can retrieve the lm instance + // Returns once we can retrieve the LM instance if err := wait.PollImmediate(PollInterval, DefaultTimeout, func() (bool, error) { key := client.ObjectKey{Name: name, Namespace: d.testenv.namespace} err := d.testenv.GetKubeClient().Get(ctx, key, cr) @@ -725,7 +725,7 @@ func (d *Deployment) DeleteCR(ctx context.Context, cr client.Object) error { return err } -// DeploySingleSiteCluster deploys a lm and indexer cluster (shc optional) +// DeploySingleSiteCluster deploys a LM and Indexer Cluster (SHC optional) func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, indexerReplicas int, shc bool, mcRef string) error { var LicenseManager string @@ -734,13 +734,13 @@ func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, i if d.testenv.licenseFilePath != "" { // Enable LM to be tested if strings.Contains(name, "master") { - // Deploy the license master + // Deploy the License Master _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err } } else { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -751,26 +751,26 @@ func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, i } if strings.Contains(name, "master") { - // Deploy the cluster master + // Deploy the Cluster Master _, err := d.DeployClusterMaster(ctx, name, LicenseManager, "", mcRef) if err != nil { return err } } else { - // Deploy the cluster manager + // Deploy the Cluster Manager _, err := d.DeployClusterManager(ctx, name, LicenseManager, "", mcRef) if err != nil { return err } } - // Deploy the indexer cluster + // Deploy the Indexer Cluster _, err := d.DeployIndexerCluster(ctx, name+"-idxc", LicenseManager, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return err } - // Deploy the SH cluster + // Deploy the SH Cluster if shc { _, err = d.DeploySearchHeadCluster(ctx, name+"-shc", name, LicenseManager, "", mcRef) if err != nil { @@ -781,7 +781,7 @@ func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, i return nil } -// DeployMultisiteClusterMasterWithSearchHead deploys a lm, cluster-manager, indexers in multiple sites and SH clusters +// DeployMultisiteClusterMasterWithSearchHead deploys a LM, Cluster Manager, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterMasterWithSearchHead(ctx context.Context, name string, indexerReplicas int, siteCount int, mcRef string) error { var LicenseMaster string @@ -796,7 +796,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHead(ctx context.Cont LicenseMaster = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -843,7 +843,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHead(ctx context.Cont return nil } -// DeployMultisiteClusterWithSearchHead deploys a lm, cluster-manager, indexers in multiple sites and SH clusters +// DeployMultisiteClusterWithSearchHead deploys a LM, Cluster Manager, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, name string, indexerReplicas int, siteCount int, mcRef string) error { var LicenseManager string @@ -852,13 +852,13 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, n if d.testenv.licenseFilePath != "" { // Enable LM to be tested if strings.Contains(name, "master") { - // Deploy the license master + // Deploy the License Master _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err } } else { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -868,7 +868,7 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, n LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -915,14 +915,14 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, n return nil } -// DeployMultisiteCluster deploys a lm, cluster-manager, indexers in multiple sites +// DeployMultisiteCluster deploys a LM, Cluster Manager, and indexers in multiple sites func (d *Deployment) DeployMultisiteCluster(ctx context.Context, name string, indexerReplicas int, siteCount int, mcRef string) error { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -931,7 +931,7 @@ func (d *Deployment) DeployMultisiteCluster(ctx context.Context, name string, in LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -965,13 +965,13 @@ func (d *Deployment) DeployMultisiteCluster(ctx context.Context, name string, in return nil } -// DeployStandaloneWithLMaster deploys a standalone splunk enterprise instance with license manager on the specified testenv +// DeployStandaloneWithLMaster deploys a Standalone Splunk Enterprise instance with License Master on the specified testenv func (d *Deployment) DeployStandaloneWithLMaster(ctx context.Context, name string, mcRef string) (*enterpriseApi.Standalone, error) { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return nil, err @@ -992,13 +992,13 @@ func (d *Deployment) DeployStandaloneWithLMaster(ctx context.Context, name strin return deployed.(*enterpriseApi.Standalone), err } -// DeployStandaloneWithLM deploys a standalone splunk enterprise instance with license manager on the specified testenv +// DeployStandaloneWithLM deploys a Standalone Splunk Enterprise instance with License Manager on the specified testenv func (d *Deployment) DeployStandaloneWithLM(ctx context.Context, name string, mcRef string) (*enterpriseApi.Standalone, error) { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return nil, err @@ -1051,14 +1051,14 @@ func (d *Deployment) DeployStandaloneWithGivenSmartStoreSpec(ctx context.Context return deployed.(*enterpriseApi.Standalone), err } -// DeployMultisiteClusterWithSearchHeadAndIndexes deploys a lm, cluster-manager, indexers in multiple sites and SH clusters +// DeployMultisiteClusterWithSearchHeadAndIndexes deploys a LM, Cluster Manager, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndIndexes(ctx context.Context, name string, indexerReplicas int, siteCount int, indexesSecret string, smartStoreSpec enterpriseApi.SmartStoreSpec) error { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -1067,7 +1067,7 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndIndexes(ctx context. LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1106,14 +1106,14 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndIndexes(ctx context. return err } -// DeployMultisiteClusterMasterWithSearchHeadAndIndexes deploys a lm, cluster-master, indexers in multiple sites and SH clusters +// DeployMultisiteClusterMasterWithSearchHeadAndIndexes deploys a LM, Cluster Master, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx context.Context, name string, indexerReplicas int, siteCount int, indexesSecret string, smartStoreSpec enterpriseApi.SmartStoreSpec) error { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err @@ -1122,7 +1122,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx co LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1161,9 +1161,9 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx co return err } -// DeployClusterManagerWithGivenSpec deploys the cluster manager with given SPEC +// DeployClusterManagerWithGivenSpec deploys the Cluster Manager with given SPEC func (d *Deployment) DeployClusterManagerWithGivenSpec(ctx context.Context, name string, spec enterpriseApi.ClusterManagerSpec) (*enterpriseApi.ClusterManager, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name) cm := newClusterManagerWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -1172,9 +1172,9 @@ func (d *Deployment) DeployClusterManagerWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApi.ClusterManager), err } -// DeployClusterMasterWithGivenSpec deploys the cluster manager with given SPEC +// DeployClusterMasterWithGivenSpec deploys the Cluster Master with given SPEC func (d *Deployment) DeployClusterMasterWithGivenSpec(ctx context.Context, name string, spec enterpriseApiV3.ClusterMasterSpec) (*enterpriseApiV3.ClusterMaster, error) { - d.testenv.Log.Info("Deploying cluster-master", "name", name) + d.testenv.Log.Info("Deploying Cluster Master", "name", name) cm := newClusterMasterWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -1183,17 +1183,17 @@ func (d *Deployment) DeployClusterMasterWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApiV3.ClusterMaster), err } -// DeploySearchHeadClusterWithGivenSpec deploys a search head cluster +// DeploySearchHeadClusterWithGivenSpec deploys a Search Head Cluster func (d *Deployment) DeploySearchHeadClusterWithGivenSpec(ctx context.Context, name string, spec enterpriseApi.SearchHeadClusterSpec) (*enterpriseApi.SearchHeadCluster, error) { - d.testenv.Log.Info("Deploying search head cluster", "name", name) + d.testenv.Log.Info("Deploying Search Head Cluster", "name", name) indexer := newSearchHeadClusterWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, indexer) return deployed.(*enterpriseApi.SearchHeadCluster), err } -// DeployLicenseManagerWithGivenSpec deploys the license manager with given SPEC +// DeployLicenseManagerWithGivenSpec deploys the License Manager with given SPEC func (d *Deployment) DeployLicenseManagerWithGivenSpec(ctx context.Context, name string, spec enterpriseApi.LicenseManagerSpec) (*enterpriseApi.LicenseManager, error) { - d.testenv.Log.Info("Deploying license-manager", "name", name) + d.testenv.Log.Info("Deploying License Manager", "name", name) lm := newLicenseManagerWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, lm) if err != nil { @@ -1202,9 +1202,9 @@ func (d *Deployment) DeployLicenseManagerWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApi.LicenseManager), err } -// DeployLicenseMasterWithGivenSpec deploys the license manager with given SPEC +// DeployLicenseMasterWithGivenSpec deploys the License Master with given SPEC func (d *Deployment) DeployLicenseMasterWithGivenSpec(ctx context.Context, name string, spec enterpriseApiV3.LicenseMasterSpec) (*enterpriseApiV3.LicenseMaster, error) { - d.testenv.Log.Info("Deploying license-master", "name", name) + d.testenv.Log.Info("Deploying License Master", "name", name) lm := newLicenseMasterWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, lm) if err != nil { @@ -1213,7 +1213,7 @@ func (d *Deployment) DeployLicenseMasterWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApiV3.LicenseMaster), err } -// DeploySingleSiteClusterWithGivenAppFrameworkSpec deploys indexer cluster (lm, shc optional) with app framework spec +// DeploySingleSiteClusterWithGivenAppFrameworkSpec deploys Indexer Cluster (LM, SHC optional) with app framework spec func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx context.Context, name string, indexerReplicas int, shc bool, appFrameworkSpecIdxc enterpriseApi.AppFrameworkSpec, appFrameworkSpecShc enterpriseApi.AppFrameworkSpec, mcName string, licenseManager string) (*enterpriseApi.ClusterManager, *enterpriseApi.IndexerCluster, *enterpriseApi.SearchHeadCluster, error) { cm := &enterpriseApi.ClusterManager{} @@ -1222,14 +1222,14 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster manager + // Deploy the Cluster Manager cmSpec := enterpriseApi.ClusterManagerSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1251,7 +1251,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex return cm, idxc, sh, err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster idxc, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseManager, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return cm, idxc, sh, err @@ -1279,7 +1279,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex } pdata, _ := json.Marshal(shSpec) - d.testenv.Log.Info("Search head Spec", "cr", pdata) + d.testenv.Log.Info("Search Head Spec", "cr", pdata) if shc { sh, err = d.DeploySearchHeadClusterWithGivenSpec(ctx, name+"-shc", shSpec) @@ -1291,7 +1291,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex return cm, idxc, sh, nil } -// DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec deploys indexer cluster (lm, shc optional) with app framework spec +// DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec deploys Indexer Cluster (LM, SHC optional) with app framework spec func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx context.Context, name string, indexerReplicas int, shc bool, appFrameworkSpecIdxc enterpriseApi.AppFrameworkSpec, appFrameworkSpecShc enterpriseApi.AppFrameworkSpec, mcName string, licenseMaster string) (*enterpriseApiV3.ClusterMaster, *enterpriseApi.IndexerCluster, *enterpriseApi.SearchHeadCluster, error) { cm := &enterpriseApiV3.ClusterMaster{} @@ -1300,14 +1300,14 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster manager + // Deploy the Cluster Master cmSpec := enterpriseApiV3.ClusterMasterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1329,7 +1329,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx return cm, idxc, sh, err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster idxc, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseMaster, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return cm, idxc, sh, err @@ -1357,7 +1357,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx } pdata, _ := json.Marshal(shSpec) - d.testenv.Log.Info("Search head Spec", "cr", pdata) + d.testenv.Log.Info("Search Head Spec", "cr", pdata) if shc { sh, err = d.DeploySearchHeadClusterWithGivenSpec(ctx, name+"-shc", shSpec) @@ -1378,14 +1378,14 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx con // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, licenseManager) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1440,7 +1440,7 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx con multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-manager") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1482,14 +1482,14 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(c // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, licenseMaster) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1544,7 +1544,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(c multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-master") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1577,14 +1577,14 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(c return cm, idxc, sh, nil } -// DeploySingleSiteClusterWithGivenMonitoringConsole deploys indexer cluster (lm, shc optional) with given monitoring console +// DeploySingleSiteClusterWithGivenMonitoringConsole deploys Indexer Cluster (LM, SHC optional) with given Monitoring Console func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx context.Context, name string, indexerReplicas int, shc bool, monitoringConsoleName string) error { licenseManager := "" // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -1593,7 +1593,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx conte licenseManager = name } - // Deploy the cluster manager + // Deploy the Cluster Manager cmSpec := enterpriseApi.ClusterManagerSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1614,7 +1614,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx conte return err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster _, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseManager, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return err @@ -1649,14 +1649,14 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx conte return nil } -// DeploySingleSiteClusterMasterWithGivenMonitoringConsole deploys indexer cluster (lm, shc optional) with given monitoring console +// DeploySingleSiteClusterMasterWithGivenMonitoringConsole deploys Indexer Cluster (LM, SHC optional) with given Monitoring Console func (d *Deployment) DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx context.Context, name string, indexerReplicas int, shc bool, monitoringConsoleName string) error { licenseMaster := "" // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err @@ -1665,7 +1665,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx licenseMaster = name } - // Deploy the cluster manager + // Deploy the Cluster Master cmSpec := enterpriseApiV3.ClusterMasterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1686,7 +1686,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx return err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster _, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseMaster, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return err @@ -1728,7 +1728,7 @@ func (d *Deployment) DeployMultisiteClusterWithMonitoringConsole(ctx context.Con // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -1737,7 +1737,7 @@ func (d *Deployment) DeployMultisiteClusterWithMonitoringConsole(ctx context.Con licenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1791,7 +1791,7 @@ func (d *Deployment) DeployMultisiteClusterWithMonitoringConsole(ctx context.Con multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-manager") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1828,7 +1828,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithMonitoringConsole(ctx conte // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err @@ -1837,7 +1837,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithMonitoringConsole(ctx conte licenseMaster = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1891,7 +1891,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithMonitoringConsole(ctx conte multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-master") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ diff --git a/test/testenv/gcputils.go b/test/testenv/gcputils.go index 78c41d1aa..e6b9a0b1e 100644 --- a/test/testenv/gcputils.go +++ b/test/testenv/gcputils.go @@ -120,9 +120,9 @@ func CheckPrefixExistsOnGCP(prefix string) bool { logf.Log.Error(err, "Error listing objects in GCP bucket") return false } - logf.Log.Info("CHECKING OBJECT", "OBJECT", objAttrs.Name) + logf.Log.Info("CHECKING OBJECT", "object", objAttrs.Name) if strings.Contains(objAttrs.Name, prefix) { - logf.Log.Info("Prefix found in bucket", "Prefix", prefix, "Object", objAttrs.Name) + logf.Log.Info("Prefix found in bucket", "prefix", prefix, "object", objAttrs.Name) return true } } @@ -146,12 +146,12 @@ func CreateBucketAndPathIfNotExist(bucketName, path string) error { // Create the bucket err = client.Client.Bucket(bucketName).Create(ctx, gcpProjectID, nil) if err != nil { - logf.Log.Error(err, "Failed to create bucket", "Bucket Name", bucketName) + logf.Log.Error(err, "Failed to create bucket", "bucketName", bucketName) return err } - logf.Log.Info("Bucket created", "Bucket Name", bucketName) + logf.Log.Info("Bucket created", "bucketName", bucketName) } else if err != nil { - logf.Log.Error(err, "Error checking bucket attributes", "Bucket Name", bucketName) + logf.Log.Error(err, "Error checking bucket attributes", "bucketName", bucketName) return err } @@ -161,16 +161,16 @@ func CreateBucketAndPathIfNotExist(bucketName, path string) error { // Create a zero-length object to represent the path wc := client.Client.Bucket(bucketName).Object(path).NewWriter(ctx) if _, err := wc.Write([]byte{}); err != nil { - logf.Log.Error(err, "Failed to create path", "Path", path) + logf.Log.Error(err, "Failed to create path", "path", path) return err } if err := wc.Close(); err != nil { - logf.Log.Error(err, "Failed to finalize path creation", "Path", path) + logf.Log.Error(err, "Failed to finalize path creation", "path", path) return err } - logf.Log.Info("Path created", "Path", path) + logf.Log.Info("Path created", "path", path) } else if err != nil { - logf.Log.Error(err, "Error checking path attributes", "Path", path) + logf.Log.Error(err, "Error checking path attributes", "path", path) return err } @@ -209,7 +209,7 @@ func DownloadFileFromGCP(bucketName, objectName, gcpFilePath, downloadDir string objectPath := filepath.Join(gcpFilePath, objectName) rc, err := client.Client.Bucket(bucketName).Object(objectPath).NewReader(ctx) if err != nil { - logf.Log.Error(err, "Failed to create reader for object", "Object", objectName) + logf.Log.Error(err, "Failed to create reader for object", "object", objectName) return "", err } defer rc.Close() @@ -217,14 +217,14 @@ func DownloadFileFromGCP(bucketName, objectName, gcpFilePath, downloadDir string localPath := filepath.Join(downloadDir, objectName) file, err := os.Create(localPath) if err != nil { - logf.Log.Error(err, "Failed to create local file", "Filename", localPath) + logf.Log.Error(err, "Failed to create local file", "filename", localPath) return "", err } defer file.Close() written, err := io.Copy(file, rc) if err != nil { - logf.Log.Error(err, "Failed to download object", "Object", objectName) + logf.Log.Error(err, "Failed to download object", "object", objectName) return "", err } @@ -319,18 +319,18 @@ func DeleteFileOnGCP(bucketName, objectName string) error { err = client.Client.Bucket(bucketName).Object(objectName).Delete(ctx) if err != nil && err != storage.ErrObjectNotExist { - logf.Log.Error(err, "Unable to delete object from bucket", "Object Name", objectName, "Bucket Name", bucketName) + logf.Log.Error(err, "Unable to delete object from bucket", "objectName", objectName, "bucketName", bucketName) return err } // Optionally, verify deletion _, err = client.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) if err == storage.ErrObjectNotExist { - logf.Log.Info("Deleted file on GCP", "File Name", objectName, "Bucket", bucketName) + logf.Log.Info("Deleted file on GCP", "fileName", objectName, "bucket", bucketName) return nil } if err != nil { - logf.Log.Error(err, "Error verifying deletion of object", "Object Name", objectName, "Bucket Name", bucketName) + logf.Log.Error(err, "Error verifying deletion of object", "objectName", objectName, "bucketName", bucketName) return err } @@ -342,12 +342,12 @@ func GetFilesInPathOnGCP(bucketName, path string) []string { resp := GetFileListOnGCP(bucketName, path) var files []string for _, obj := range resp { - logf.Log.Info("CHECKING OBJECT", "OBJECT", obj.Name) + logf.Log.Info("CHECKING OBJECT", "object", obj.Name) if strings.HasPrefix(obj.Name, path) { filename := strings.TrimPrefix(obj.Name, path) // This condition filters out directories as GCP returns objects with their full paths if len(filename) > 1 && !strings.HasSuffix(filename, "/") { - logf.Log.Info("File found in bucket", "Path", path, "Object", obj.Name) + logf.Log.Info("File found in bucket", "path", path, "object", obj.Name) files = append(files, filename) } } @@ -358,10 +358,10 @@ func GetFilesInPathOnGCP(bucketName, path string) []string { // DownloadFilesFromGCP downloads a list of files from a GCP bucket to a local directory func DownloadFilesFromGCP(bucketName, gcpAppDir, downloadDir string, appList []string) error { for _, key := range appList { - logf.Log.Info("Downloading file from GCP", "File name", key) + logf.Log.Info("Downloading file from GCP", "fileName", key) _, err := DownloadFileFromGCP(bucketName, key, gcpAppDir, downloadDir) if err != nil { - logf.Log.Error(err, "Unable to download file", "File Name", key) + logf.Log.Error(err, "Unable to download file", "fileName", key) return err } } @@ -372,22 +372,22 @@ func DownloadFilesFromGCP(bucketName, gcpAppDir, downloadDir string, appList []s func UploadFilesToGCP(bucketName, gcpTestDir string, appList []string, uploadDir string) ([]string, error) { var uploadedFiles []string for _, key := range appList { - logf.Log.Info("Uploading file to GCP", "File name", key) - logf.Log.Info("Using bucket", "Bucket", bucketName, "Path", gcpTestDir, "Upload Dir", uploadDir) + logf.Log.Info("Uploading file to GCP", "fileName", key) + logf.Log.Info("Using bucket", "bucket", bucketName, "path", gcpTestDir, "uploadDir", uploadDir) fileLocation := filepath.Join(uploadDir, key) fileBody, err := os.Open(fileLocation) if err != nil { - logf.Log.Error(err, "Unable to open file", "File name", key) + logf.Log.Error(err, "Unable to open file", "fileName", key) return nil, err } defer fileBody.Close() objectPath, err := UploadFileToGCP(bucketName, key, gcpTestDir, fileBody) if err != nil { - logf.Log.Error(err, "Unable to upload file", "File name", key) + logf.Log.Error(err, "Unable to upload file", "fileName", key) return nil, err } - logf.Log.Info("File uploaded to GCP", "File name", objectPath) + logf.Log.Info("File uploaded to GCP", "fileName", objectPath) uploadedFiles = append(uploadedFiles, objectPath) } return uploadedFiles, nil diff --git a/test/testenv/ingest_utils.go b/test/testenv/ingest_utils.go index 2c0403b1e..1f28693bf 100644 --- a/test/testenv/ingest_utils.go +++ b/test/testenv/ingest_utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/test/testenv/lmutil.go b/test/testenv/lmutil.go index 9f7aa5dd7..e58cf2b2d 100644 --- a/test/testenv/lmutil.go +++ b/test/testenv/lmutil.go @@ -1,18 +1,16 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package testenv import ( @@ -56,6 +54,6 @@ func CheckLicenseManagerConfigured(ctx context.Context, deployment *Deployment, return false } licenseManager := restResponse.Entry[0].Content.ManagerURI - logf.Log.Info("License Manager configuration on POD", "POD", podName, "License Manager", licenseManager) + logf.Log.Info("License Manager configuration on POD", "pod", podName, "licenseManager", licenseManager) return strings.Contains(licenseManager, "license-manager-service:8089") || strings.Contains(licenseManager, "license-master-service:8089") } diff --git a/test/testenv/mcutil.go b/test/testenv/mcutil.go index 36250ae96..0cd9448ee 100644 --- a/test/testenv/mcutil.go +++ b/test/testenv/mcutil.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -60,7 +60,7 @@ func CheckMCPodReady(ns string) bool { return false } stsSlice := strings.Fields(stsLine) - logf.Log.Info("MC statefulset found", "POD", stsSlice[0], "READY", stsSlice[1]) + logf.Log.Info("MC statefulset found", "pod", stsSlice[0], "ready", stsSlice[1]) stsReady := strings.Contains(stsSlice[1], "1/1") // Check Status of monitoring console pod @@ -69,13 +69,13 @@ func CheckMCPodReady(ns string) bool { return false } podSlice := strings.Fields(podLine) - logf.Log.Info("MC Pod Found", "POD", podSlice[0], "READY", podSlice[1]) + logf.Log.Info("MC Pod Found", "pod", podSlice[0], "ready", podSlice[1]) podReady := strings.Contains(podSlice[1], "1/1") && strings.Contains(podSlice[2], "Running") return stsReady && podReady } -// GetConfiguredPeers get list of Peers Configured on Montioring Console +// GetConfiguredPeers get list of Peers Configured on Monitoring Console func GetConfiguredPeers(ns string, mcName string) []string { podName := fmt.Sprintf(MonitoringConsolePod, mcName) var peerList []string @@ -100,22 +100,10 @@ func GetConfiguredPeers(ns string, mcName string) []string { } } } - logf.Log.Info("Peer List found on MC Pod", "MC POD", podName, "Configured Peers", peerList) + logf.Log.Info("Peer List found on MC Pod", "mcPod", podName, "configuredPeers", peerList) return peerList } -// DeleteMCPod delete monitoring console deployment -func DeleteMCPod(ns string) { - mcSts := fmt.Sprintf(MonitoringConsoleSts, ns) - output, err := exec.Command("kubectl", "delete", "sts", "-n", ns, mcSts).Output() - if err != nil { - cmd := fmt.Sprintf("kubectl delete sts -n %s %s", ns, mcSts) - logf.Log.Error(err, "Failed to execute command", "command", cmd) - } else { - logf.Log.Info("Monitoring Console Stateful Set deleted", "Statefulset", mcSts, "stdout", output) - } -} - // CheckPodNameOnMC Check given pod is configured on Monitoring console pod func CheckPodNameOnMC(ns string, mcName string, podName string) bool { // Get Peers configured on Monitoring Console @@ -124,7 +112,7 @@ func CheckPodNameOnMC(ns string, mcName string, podName string) bool { found := false for _, peer := range peerList { if strings.Contains(peer, podName) { - logf.Log.Info("Check Peer matches on pod", "Pod Name", podName, "Peer in peer list", peer) + logf.Log.Info("Check Peer matches on pod", "podName", podName, "peerInPeerList", peer) found = true break } @@ -157,12 +145,12 @@ func GetMCConfigMap(ctx context.Context, deployment *Deployment, ns string, mcNa logf.Log.Error(err, "Failed to get Monitoring Console Config Map") return mcConfigMap, err } - logf.Log.Info("MC Config Map contents", "MC CONFIG MAP NAME", mcConfigMapName, "Data", mcConfigMap.Data) + logf.Log.Info("MC Config Map contents", "mcConfigMapName", mcConfigMapName, "data", mcConfigMap.Data) return mcConfigMap, err } // CheckPodNameInString checks for pod name in string func CheckPodNameInString(podName string, configString string) bool { - logf.Log.Info("Check MC Config String has Pod configured", "Monitoring Console Config Map Pod Config String", configString, "POD String", podName) + logf.Log.Info("Check MC Config String has Pod configured", "configString", configString, "podName", podName) return strings.Contains(configString, podName) } diff --git a/test/testenv/multisite_helpers.go b/test/testenv/multisite_helpers.go new file mode 100644 index 000000000..dd837ffe8 --- /dev/null +++ b/test/testenv/multisite_helpers.go @@ -0,0 +1,78 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" +) + +// VerifyIndexOnAllSites verifies that an index exists on all indexer pods across all sites +func (testcaseenv *TestCaseEnv) VerifyIndexOnAllSites(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, siteNumber, 0) + testcaseenv.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) + } +} + +// IngestDataOnAllSites ingests data to an index on all indexer pods across all sites +func IngestDataOnAllSites(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, siteNumber, 0) + logFile := fmt.Sprintf("test-log-%s.log", RandomDNSName(3)) + CreateMockLogfile(logFile, LogLineCount) + IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) + } +} + +// RollHotToWarmOnAllSites rolls hot buckets to warm on all indexer pods across all sites +func RollHotToWarmOnAllSites(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, siteNumber, 0) + RollHotToWarm(ctx, deployment, podName, indexName) + } +} + +// VerifyIndexOnS3AllSites verifies that an index exists on S3 for all indexer pods across all sites +func (testcaseenv *TestCaseEnv) VerifyIndexOnS3AllSites(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, siteNumber, 0) + testcaseenv.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) + } +} + +// VerifyCPULimitsOnAllSites verifies CPU limits on all indexer pods across all sites +func (testcaseenv *TestCaseEnv) VerifyCPULimitsOnAllSites(deployment *Deployment, deploymentName string, siteCount int, expectedCPULimit string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, siteNumber, 0) + testcaseenv.VerifyCPULimits(deployment, podName, expectedCPULimit) + } +} + +// MultisiteIndexerWorkflow encapsulates the common workflow for multisite indexer operations: +// verify index, ingest data, roll to warm, verify on S3 +func (testcaseenv *TestCaseEnv) MultisiteIndexerWorkflow(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string) { + // Verify index exists on all sites + testcaseenv.VerifyIndexOnAllSites(ctx, deployment, deploymentName, siteCount, indexName) + + // Ingest data on all sites + IngestDataOnAllSites(ctx, deployment, deploymentName, siteCount, indexName) + + // Roll hot to warm on all sites + RollHotToWarmOnAllSites(ctx, deployment, deploymentName, siteCount, indexName) + + // Verify index on S3 for all sites + testcaseenv.VerifyIndexOnS3AllSites(ctx, deployment, deploymentName, siteCount, indexName) +} diff --git a/test/testenv/remote_index_utils.go b/test/testenv/remote_index_utils.go index 551a377d0..e093e5008 100644 --- a/test/testenv/remote_index_utils.go +++ b/test/testenv/remote_index_utils.go @@ -60,19 +60,6 @@ func GetIndexOnPod(ctx context.Context, deployment *Deployment, podName string, return indexFound, indexData } -// RestartSplunk Restart splunk inside the container -func RestartSplunk(ctx context.Context, deployment *Deployment, podName string) bool { - stdin := "/opt/splunk/bin/splunk restart -auth admin:$(cat /mnt/splunk-secrets/password)" - command := []string{"/bin/sh"} - stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) - if err != nil { - logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) - return false - } - logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - return true -} - // RollHotToWarm rolls hot buckets to warm for a given index and pod func RollHotToWarm(ctx context.Context, deployment *Deployment, podName string, indexName string) bool { stdin := "/opt/splunk/bin/splunk _internal call /data/indexes/" + indexName + "/roll-hot-buckets admin:$(cat /mnt/splunk-secrets/password)" diff --git a/test/testenv/s3utils.go b/test/testenv/s3utils.go index 445e395f1..3921844e2 100644 --- a/test/testenv/s3utils.go +++ b/test/testenv/s3utils.go @@ -32,11 +32,6 @@ var ( s3deleteWaitTime = 60 ) -// GetSmartStoreIndexesBucet returns smartstore test bucket name -func GetSmartStoreIndexesBucet() string { - return testIndexesS3Bucket -} - // GetDefaultS3Region returns default AWS Region func GetDefaultS3Region() string { return s3Region @@ -53,9 +48,9 @@ func CheckPrefixExistsOnS3(prefix string) bool { resp := GetFileListOnS3(dataBucket, prefix) for _, key := range resp { - logf.Log.Info("CHECKING KEY ", "KEY", *key.Key) + logf.Log.Info("CHECKING KEY ", "key", *key.Key) if strings.Contains(*key.Key, prefix) { - logf.Log.Info("Prefix found on bucket", "Prefix", prefix, "KEY", *key.Key) + logf.Log.Info("Prefix found on bucket", "prefix", prefix, "key", *key.Key) return true } } @@ -94,7 +89,7 @@ func DownloadFileFromS3(dataBucket string, filename string, s3FilePath string, d // Create empty file on OS File System file, err := os.Create(filepath.Join(downloadDir, filename)) if err != nil { - logf.Log.Error(err, "Failed to create file", "Filename", file) + logf.Log.Error(err, "Failed to create file", "filename", file) } defer file.Close() @@ -112,7 +107,7 @@ func DownloadFileFromS3(dataBucket string, filename string, s3FilePath string, d }) if err != nil { - logf.Log.Error(err, "Failed to download file", "Bucket", dataBucket, "Path", s3FilePath, "Filename", filename) + logf.Log.Error(err, "Failed to download file", "bucket", dataBucket, "path", s3FilePath, "filename", filename) return "", err } @@ -174,7 +169,7 @@ func DeleteFileOnS3(bucket string, filename string) error { svc := s3.NewFromConfig(*cfg) _, err = svc.DeleteObject(context.TODO(), &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(filename)}) if err != nil { - logf.Log.Error(err, "Unable to delete object from bucket", "Object Name", filename, "Bucket Name", bucket) + logf.Log.Error(err, "Unable to delete object from bucket", "objectName", filename, "bucketName", bucket) } waiter := s3.NewObjectNotExistsWaiter(svc) @@ -182,7 +177,7 @@ func DeleteFileOnS3(bucket string, filename string) error { Bucket: aws.String(bucket), Key: aws.String(filename), }, time.Duration(time.Duration(s3deleteWaitTime).Seconds())) - logf.Log.Info("Deleted file on S3", "File Name", filename, "Bucket", bucket) + logf.Log.Info("Deleted file on S3", "fileName", filename, "bucket", bucket) return err } @@ -191,12 +186,12 @@ func GetFilesInPathOnS3(bucket string, path string) []string { resp := GetFileListOnS3(bucket, path) var files []string for _, key := range resp { - logf.Log.Info("CHECKING KEY ", "KEY", *key.Key) + logf.Log.Info("CHECKING KEY ", "key", *key.Key) if strings.Contains(*key.Key, path) { filename := strings.Replace(*key.Key, path, "", -1) // This condition filters out directories as GetFileListOnS3 returns files and directories with their absolute path's if len(filename) > 1 { - logf.Log.Info("File found on bucket", "Prefix", path, "KEY", *key.Key) + logf.Log.Info("File found on bucket", "prefix", path, "key", *key.Key) files = append(files, filename) } } @@ -207,10 +202,10 @@ func GetFilesInPathOnS3(bucket string, path string) []string { // DownloadFilesFromS3 download given list of files from S3 to the given directory func DownloadFilesFromS3(testDataS3Bucket string, s3AppDir string, downloadDir string, appList []string) error { for _, key := range appList { - logf.Log.Info("Downloading file from S3", "File name", key) + logf.Log.Info("Downloading file from S3", "fileName", key) _, err := DownloadFileFromS3(testDataS3Bucket, key, s3AppDir, downloadDir) if err != nil { - logf.Log.Error(err, "Unable to download file", "File Name", key) + logf.Log.Error(err, "Unable to download file", "fileName", key) return err } } @@ -221,19 +216,19 @@ func DownloadFilesFromS3(testDataS3Bucket string, s3AppDir string, downloadDir s func UploadFilesToS3(testS3Bucket string, s3TestDir string, applist []string, downloadDir string) ([]string, error) { var uploadedFiles []string for _, key := range applist { - logf.Log.Info("Uploading file to S3", "File name", key) + logf.Log.Info("Uploading file to S3", "fileName", key) fileLocation := filepath.Join(downloadDir, key) fileBody, err := os.Open(fileLocation) if err != nil { - logf.Log.Error(err, "Unable to open file", "File name", key) + logf.Log.Error(err, "Unable to open file", "fileName", key) return nil, err } fileName, err := UploadFileToS3(testS3Bucket, key, s3TestDir, fileBody) if err != nil { - logf.Log.Error(err, "Unable to upload file", "File name", key) + logf.Log.Error(err, "Unable to upload file", "fileName", key) return nil, err } - logf.Log.Info("File upload to test S3", "File name", fileName) + logf.Log.Info("File upload to test S3", "fileName", fileName) uploadedFiles = append(uploadedFiles, fileName) } return uploadedFiles, nil diff --git a/test/testenv/search_head_cluster_utils.go b/test/testenv/search_head_cluster_utils.go index 012064501..4dd326676 100644 --- a/test/testenv/search_head_cluster_utils.go +++ b/test/testenv/search_head_cluster_utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,39 +18,11 @@ package testenv import ( "context" "fmt" - "os/exec" "strings" logf "sigs.k8s.io/controller-runtime/pkg/log" ) -// DeleteSHC delete Search Head Cluster in given namespace -func DeleteSHC(ns string) { - output, err := exec.Command("kubectl", "delete", "shc", "-n", ns, "--all").Output() - if err != nil { - cmd := fmt.Sprintf("kubectl delete shc -n %s --all", ns) - logf.Log.Error(err, "Failed to execute command", "command", cmd) - } else { - logf.Log.Info("SHC deleted", "Namespace", ns, "stdout", output) - } -} - -// SHCInNamespace returns true if SHC is present in namespace -func SHCInNamespace(ns string) bool { - output, err := exec.Command("kubectl", "get", "searchheadcluster", "-n", ns).Output() - deleted := true - if err != nil { - cmd := fmt.Sprintf("kubectl get shc -n %s", ns) - logf.Log.Error(err, "Failed to execute command", "command", cmd) - return deleted - } - logf.Log.Info("Output of command", "Output", string(output)) - if strings.Contains(string(output), "No resources found in default namespace") { - deleted = false - } - return deleted -} - // DeployerAppChecksum Get the checksum for each app on the deployer func DeployerAppChecksum(ctx context.Context, deployment *Deployment) map[string]string { appChecksum := make(map[string]string) @@ -143,7 +115,7 @@ func DeployerBundlePushstatus(ctx context.Context, deployment *Deployment, ns st } for appName := range appChecksum { if _, present := appBundlePush[appName]; !present { - logf.Log.Info("Deployer app not found on any members", "Appname", appName) + logf.Log.Info("Deployer app not found on any members", "appName", appName) return make(map[string]int) } } diff --git a/test/testenv/search_utils.go b/test/testenv/search_utils.go index b0e85cc66..e6856d8dc 100644 --- a/test/testenv/search_utils.go +++ b/test/testenv/search_utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -77,7 +77,7 @@ func PerformSearchSync(ctx context.Context, podName string, search string, deplo return "", err } - logf.Log.Info("Output of search Query", "Search", search, "Output", searchReqResp) + logf.Log.Info("Output of search Query", "search", search, "output", searchReqResp) // Since results can have multiple formats depending on the search SPL, leave this response as a string return searchReqResp, err @@ -102,7 +102,7 @@ func PerformSearchReq(ctx context.Context, podName string, search string, deploy return "", err } - logf.Log.Info("Output of search Query", "Search", search, "Output", stdout) + logf.Log.Info("Output of search Query", "search", search, "output", stdout) // Get SID var searchReqResult map[string]interface{} diff --git a/test/testenv/secretutil.go b/test/testenv/secretutil.go index 1b9ee9ff1..f30e72446 100644 --- a/test/testenv/secretutil.go +++ b/test/testenv/secretutil.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -46,14 +46,14 @@ func GetSecretStruct(ctx context.Context, deployment *Deployment, ns string, sec secretObject := &corev1.Secret{} err := deployment.GetInstance(ctx, secretName, secretObject) if err != nil { - deployment.testenv.Log.Error(err, "Unable to get secret object", "Secret Name", secretName, "Namespace", ns) + deployment.testenv.Log.Error(err, "Unable to get secret object", "secretName", secretName, "namespace", ns) } return secretObject, err } // ModifySecretObject Modifies the secret object with given data func ModifySecretObject(ctx context.Context, deployment *Deployment, ns string, secretName string, data map[string][]byte) error { - logf.Log.Info("Modify secret object", "Secret Name", secretName, "Data", data) + logf.Log.Info("Modify secret object", "secretName", secretName, "data", data) secret := newSecretSpec(ns, secretName, data) err := deployment.UpdateCR(ctx, secret) @@ -66,7 +66,7 @@ func ModifySecretObject(ctx context.Context, deployment *Deployment, ns string, // DeleteSecretObject Deletes the entire secret object func DeleteSecretObject(ctx context.Context, deployment *Deployment, ns string, secretName string) error { - logf.Log.Info("Delete secret object", "Secret Name", secretName, "Namespace", ns) + logf.Log.Info("Delete secret object", "secretName", secretName, "namespace", ns) secret := newSecretSpec(ns, secretName, map[string][]byte{}) err := deployment.DeleteCR(ctx, secret) if err != nil { @@ -85,12 +85,12 @@ func GetMountedKey(ctx context.Context, deployment *Deployment, podName string, logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) return "" } - logf.Log.Info("Key found on pod", "Pod Name", podName, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Key found on pod", "podName", podName, "stdout", stdout, "stderr", stderr) return stdout } -// GetRandomeHECToken generates a random HEC token -func GetRandomeHECToken() string { +// GetRandomHECToken generates a random HEC token +func GetRandomHECToken() string { return fmt.Sprintf("%s-%s-%s-%s-%s", strings.ToUpper(RandomDNSName(8)), strings.ToUpper(RandomDNSName(4)), strings.ToUpper(RandomDNSName(4)), strings.ToUpper(RandomDNSName(4)), strings.ToUpper(RandomDNSName(12))) } @@ -99,7 +99,7 @@ func GetSecretFromServerConf(ctx context.Context, deployment *Deployment, podNam filePath := "/opt/splunk/etc/system/local/server.conf" confline, err := GetConfLineFromPod(podName, filePath, ns, configName, stanza, true) if err != nil { - logf.Log.Error(err, "Failed to get secret from pod", "Pod Name", podName, "Secret Name", configName) + logf.Log.Error(err, "Failed to get secret from pod", "podName", podName, "secretName", configName) return "", "", err } @@ -120,7 +120,7 @@ func DecryptSplunkEncodedSecret(ctx context.Context, deployment *Deployment, pod } logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - logf.Log.Info("Decrypted Key Value", "Decrypted Key", stdout) + logf.Log.Info("Decrypted Key Value", "decryptedKey", stdout) return strings.TrimSuffix(stdout, "\n") } @@ -157,7 +157,7 @@ func GetVersionedSecretNames(ns string, version int) []string { } } } - logf.Log.Info("Versioned Secret Objects Found in Namespace", "NameSpace", ns, "Versioned Secrets", splunkSecrets) + logf.Log.Info("Versioned Secret Objects Found in Namespace", "namespace", ns, "versionedSecrets", splunkSecrets) return splunkSecrets } @@ -198,7 +198,7 @@ func GetSecretFromInputsConf(deployment *Deployment, podName string, ns string, filePath := "/opt/splunk/etc/apps/splunk_httpinput/local/inputs.conf" confline, err := GetConfLineFromPod(podName, filePath, ns, configName, stanza, true) if err != nil { - logf.Log.Error(err, "Failed to get secret from pod", "Pod Name", podName, "Secret Name", configName) + logf.Log.Error(err, "Failed to get secret from pod", "podName", podName, "secretName", configName) return "", "", err } secretList := strings.Split(confline, "=") diff --git a/test/testenv/test_setup_helpers.go b/test/testenv/test_setup_helpers.go new file mode 100644 index 000000000..326199d7d --- /dev/null +++ b/test/testenv/test_setup_helpers.go @@ -0,0 +1,245 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/joho/godotenv" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/types" + . "github.com/onsi/gomega" +) + +// SetupTestCaseEnv creates a new test case environment and deployment for use in BeforeEach blocks. +// It also validates test prerequisites immediately to fail fast before any long operations. +func SetupTestCaseEnv(testenvInstance *TestEnv, namePrefix string) (*TestCaseEnv, *Deployment) { + name := fmt.Sprintf("%s-%s", namePrefix+testenvInstance.GetName(), RandomDNSName(3)) + testcaseEnvInst, err := NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) + Expect(err).To(Succeed(), "Unable to create testcaseenv") + + deployment, err := testcaseEnvInst.NewDeployment(RandomDNSName(3)) + Expect(err).To(Succeed(), "Unable to create deployment") + + err = testcaseEnvInst.ValidateTestPrerequisites(context.TODO(), deployment) + Expect(err).To(Succeed(), "Test prerequisites validation failed") + + return testcaseEnvInst, deployment +} + +// TeardownTestCaseEnv handles the common teardown logic for test case environments. +func TeardownTestCaseEnv(testcaseEnvInst *TestCaseEnv, deployment *Deployment) { + if types.SpecState(ginkgo.CurrentSpecReport().State) == types.SpecStateFailed { + if testcaseEnvInst != nil { + testcaseEnvInst.SkipTeardown = true + } + } + + if deployment != nil { + deployment.Teardown() + } + + if testcaseEnvInst != nil { + Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) + } +} + +// CleanupOperatorFile deletes the test_file.img from the operator pod's app download directory +// if filePresentOnOperator is true. +func CleanupOperatorFile(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, filePresentOnOperator bool) { + if filePresentOnOperator { + opPod := GetOperatorPodName(testcaseEnvInst) + podDownloadPath := filepath.Join(AppDownloadVolume, "test_file.img") + DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) + } +} + +// TeardownAppFrameworkTestCaseEnv handles teardown for app framework tests with provider-specific +// cloud storage cleanup. cloudCleanup is called only if SkipTeardown is false. +func TeardownAppFrameworkTestCaseEnv(ctx context.Context, testcaseEnvInst *TestCaseEnv, deployment *Deployment, cloudCleanup func(), filePresentOnOperator bool) { + TeardownTestCaseEnv(testcaseEnvInst, deployment) + + if testcaseEnvInst != nil && !testcaseEnvInst.SkipTeardown && cloudCleanup != nil { + cloudCleanup() + } + + CleanupOperatorFile(ctx, deployment, testcaseEnvInst, filePresentOnOperator) +} + +// S3CloudCleanup returns a cleanup function that deletes the given files from an S3 bucket. +func S3CloudCleanup(bucket string, uploadedApps []string) func() { + return func() { + DeleteFilesOnS3(bucket, uploadedApps) + } +} + +// AzureCloudCleanup returns a cleanup function that deletes the given files from Azure Blob storage. +func AzureCloudCleanup(ctx context.Context, uploadedApps []string) func() { + return func() { + azureBlobClient := &AzureBlobClient{} + azureBlobClient.DeleteFilesOnAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, uploadedApps) + } +} + +// GCPCloudCleanup returns a cleanup function that deletes the given files from a GCP bucket. +func GCPCloudCleanup(bucket string, uploadedApps []string) func() { + return func() { + DeleteFilesOnGCP(bucket, uploadedApps) + } +} + +// LoadEnvFile traverses up the directory tree from the current working directory +// to find and load a .env file using godotenv. Returns nil if no .env file is found. +func LoadEnvFile() error { + dir, err := os.Getwd() + if err != nil { + return err + } + + for { + envFile := filepath.Join(dir, ".env") + if _, err := os.Stat(envFile); err == nil { + return godotenv.Load(envFile) + } + + parentDir := filepath.Dir(dir) + if parentDir == dir { + return nil + } + dir = parentDir + } +} + +// SetupS3AppsSuite initialises the test environment and, when running on EKS, +// downloads the V1 and V2 app sets from S3. +func SetupS3AppsSuite(suiteName, testDataBucket, appDirV1, downloadDirV1, appDirV2, downloadDirV2 string) (*TestEnv, []string, []string) { + testenvInst, err := NewDefaultTestEnv(suiteName) + Expect(err).ToNot(HaveOccurred()) + + if ClusterProvider == "eks" { + appListV1 := BasicApps + appFileList := GetAppFileList(appListV1) + + err = DownloadFilesFromS3(testDataBucket, appDirV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download V1 app files") + + appListV2 := append(appListV1, NewAppsAddedBetweenPolls...) + appFileList = GetAppFileList(appListV2) + + err = DownloadFilesFromS3(testDataBucket, appDirV2, downloadDirV2, appFileList) + Expect(err).To(Succeed(), "Unable to download V2 app files") + + return testenvInst, appListV1, appListV2 + } + + testenvInst.Log.Info("Skipping Before Suite Setup", "provider", ClusterProvider) + return testenvInst, nil, nil +} + +// CleanupLocalAppDownloads tears down the test environment and removes locally +// downloaded app directories after a suite run. +func CleanupLocalAppDownloads(testenvInst *TestEnv, dirs ...string) { + if testenvInst != nil { + Expect(testenvInst.Teardown()).ToNot(HaveOccurred()) + } + for _, dir := range dirs { + Expect(os.RemoveAll(dir)).To(Succeed(), "Unable to delete locally downloaded app files from "+dir) + } +} + +// SetupAzureAppsSuite initialises the test environment and, when running on Azure, +// downloads the V1 and V2 app sets from Azure Blob. +func SetupAzureAppsSuite(suiteName, downloadDirV1, downloadDirV2 string) (*TestEnv, []string, []string) { + testenvInst, err := NewDefaultTestEnv(suiteName) + Expect(err).ToNot(HaveOccurred()) + + if ClusterProvider == "azure" { + ctx := context.TODO() + + appListV1 := BasicApps + appFileList := GetAppFileList(appListV1) + + containerName := "/test-data/appframework/v1apps/" + err = DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, downloadDirV1, containerName, appFileList) + Expect(err).To(Succeed(), "Unable to download V1 app files") + + appListV2 := append(appListV1, NewAppsAddedBetweenPolls...) + appFileList = GetAppFileList(appListV2) + + containerName = "/test-data/appframework/v2apps/" + err = DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, downloadDirV2, containerName, appFileList) + Expect(err).To(Succeed(), "Unable to download V2 app files") + + return testenvInst, appListV1, appListV2 + } + + testenvInst.Log.Info("Skipping Before Suite Setup", "provider", ClusterProvider) + return testenvInst, nil, nil +} + +// SetupGCPAppsSuite initialises the test environment and, when running on GCP, +// downloads the V1 and V2 app sets from GCS. +func SetupGCPAppsSuite(suiteName, testDataBucket, appDirV1, downloadDirV1, appDirV2, downloadDirV2 string) (*TestEnv, []string, []string) { + testenvInst, err := NewDefaultTestEnv(suiteName) + Expect(err).ToNot(HaveOccurred()) + + if ClusterProvider == "gcp" { + appListV1 := BasicApps + appFileList := GetAppFileList(appListV1) + + testenvInst.Log.Info("logging download details", "bucket", testDataBucket, "appDirV1", appDirV1, "downloadDirV1", downloadDirV1, "appFileList", appFileList) + err = DownloadFilesFromGCP(testDataBucket, appDirV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download V1 app files") + + appListV2 := append(appListV1, NewAppsAddedBetweenPolls...) + appFileList = GetAppFileList(appListV2) + + err = DownloadFilesFromGCP(testDataBucket, appDirV2, downloadDirV2, appFileList) + Expect(err).To(Succeed(), "Unable to download V2 app files") + + return testenvInst, appListV1, appListV2 + } + + testenvInst.Log.Info("Skipping Before Suite Setup", "provider", ClusterProvider) + return testenvInst, nil, nil +} + +// SetupLicenseConfigMap downloads the license file from the appropriate provider +// and creates a license config map. +func SetupLicenseConfigMap(ctx context.Context, testcaseEnvInst *TestCaseEnv) { + downloadDir := "licenseFolder" + var licenseFilePath string + var err error + + switch ClusterProvider { + case "eks": + licenseFilePath, err = DownloadLicenseFromS3Bucket() + Expect(err).To(Succeed(), "Unable to download license file from S3") + case "azure": + licenseFilePath, err = DownloadLicenseFromAzure(ctx, downloadDir) + Expect(err).To(Succeed(), "Unable to download license file from Azure") + case "gcp": + licenseFilePath, err = DownloadLicenseFromGCPBucket() + Expect(err).To(Succeed(), "Unable to download license file from GCP") + default: + testcaseEnvInst.Log.Info("Skipping license download", "provider", ClusterProvider) + return + } + + testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) +} diff --git a/test/testenv/test_workflows.go b/test/testenv/test_workflows.go new file mode 100644 index 000000000..8c5d60245 --- /dev/null +++ b/test/testenv/test_workflows.go @@ -0,0 +1,131 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package testenv + +import ( + "context" + "fmt" + + . "github.com/onsi/gomega" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + corev1 "k8s.io/api/core/v1" +) + +// WorkflowResult contains the result of a workflow execution +type WorkflowResult struct { + Standalone *enterpriseApi.Standalone + ClusterManager *enterpriseApi.ClusterManager + IndexerCluster *enterpriseApi.IndexerCluster + SearchHeadCluster *enterpriseApi.SearchHeadCluster + MonitoringConsole *enterpriseApi.MonitoringConsole + LicenseManager *enterpriseApi.LicenseManager +} + +// RunStandaloneDeploymentWorkflow deploys a standalone instance and verifies it's ready +func RunStandaloneDeploymentWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string) *WorkflowResult { + standalone := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, name, "", "") + + return &WorkflowResult{Standalone: standalone} +} + +// RunC3DeploymentWorkflow deploys a C3 cluster (CM + IDXC + SHC) and verifies all components are ready +func RunC3DeploymentWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string, indexerReplicas int, mcRef string) *WorkflowResult { + err := deployment.DeploySingleSiteCluster(ctx, name, indexerReplicas, true, mcRef) + Expect(err).To(Succeed(), "Unable to deploy C3 cluster") + + testcaseEnvInst.VerifyClusterReadyAndRFSF(ctx, deployment) + + return &WorkflowResult{} +} + +// RunM4DeploymentWorkflow deploys a M4 multisite cluster and verifies all components are ready +func RunM4DeploymentWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string, indexerReplicas int, siteCount int, mcRef string) *WorkflowResult { + err := deployment.DeployMultisiteClusterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) + Expect(err).To(Succeed(), "Unable to deploy M4 cluster") + + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + return &WorkflowResult{} +} + +// RunM1DeploymentWorkflow deploys a M1 multisite indexer cluster (no SHC) and verifies components +func RunM1DeploymentWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string, indexerReplicas int, siteCount int) *WorkflowResult { + err := deployment.DeployMultisiteCluster(ctx, name, indexerReplicas, siteCount, "") + Expect(err).To(Succeed(), "Unable to deploy M1 cluster") + + testcaseEnvInst.VerifyM1ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + return &WorkflowResult{} +} + +// RunStandaloneWithServiceAccountWorkflow deploys standalone with a service account +func RunStandaloneWithServiceAccountWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string, serviceAccountName string) *WorkflowResult { + testcaseEnvInst.CreateServiceAccount(serviceAccountName) + + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseEnvInst.GetSplunkImage(), + }, + Volumes: []corev1.Volume{}, + ServiceAccount: serviceAccountName, + }, + } + + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, name, spec) + Expect(err).To(Succeed(), "Unable to deploy standalone with service account") + + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, name, standalone) + + standalonePodName := fmt.Sprintf(StandalonePod, name, 0) + testcaseEnvInst.VerifyServiceAccountConfiguredOnPod(deployment, testcaseEnvInst.GetName(), standalonePodName, serviceAccountName) + + return &WorkflowResult{Standalone: standalone} +} + +// RunDeleteStandaloneWorkflow deploys and deletes a standalone instance +func RunDeleteStandaloneWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string) { + result := RunStandaloneDeploymentWorkflow(ctx, deployment, testcaseEnvInst, name) + + err := deployment.DeleteCR(ctx, result.Standalone) + Expect(err).To(Succeed(), "Unable to delete standalone instance") +} + +// RunDeleteC3Workflow deploys and deletes a C3 cluster +func RunDeleteC3Workflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string, indexerReplicas int) { + RunC3DeploymentWorkflow(ctx, deployment, testcaseEnvInst, name, indexerReplicas, "") + + idxc := &enterpriseApi.IndexerCluster{} + idxcName := name + "-idxc" + GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Unable to get Indexer Cluster instance") + + err := deployment.DeleteCR(ctx, idxc) + Expect(err).To(Succeed(), "Unable to delete Indexer Cluster") + + shc := &enterpriseApi.SearchHeadCluster{} + shcName := name + "-shc" + GetInstanceWithExpect(ctx, deployment, shc, shcName, "Unable to get Search Head Cluster instance") + + err = deployment.DeleteCR(ctx, shc) + Expect(err).To(Succeed(), "Unable to delete Search Head Cluster") + + cm := &enterpriseApi.ClusterManager{} + GetInstanceWithExpect(ctx, deployment, cm, name, "Unable to get Cluster Manager instance") + + err = deployment.DeleteCR(ctx, cm) + Expect(err).To(Succeed(), "Unable to delete Cluster Manager") +} diff --git a/test/testenv/testcaseenv.go b/test/testenv/testcaseenv.go index 4e0ccb3c4..93f114406 100644 --- a/test/testenv/testcaseenv.go +++ b/test/testenv/testcaseenv.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -118,11 +118,6 @@ func (testenv *TestCaseEnv) GetSplunkImage() string { return testenv.splunkImage } -// IsOperatorInstalledClusterWide returns if operator is installed clusterwide -func (testenv *TestCaseEnv) IsOperatorInstalledClusterWide() string { - return testenv.clusterWideOperator -} - func (testenv *TestCaseEnv) setup() error { testenv.Log.Info("testenv initializing.\n") @@ -362,7 +357,7 @@ func (testenv *TestCaseEnv) attachPVCToOperator(name string) error { operator := &appsv1.Deployment{} err = testenv.GetKubeClient().Get(ctx, namespacedName, operator) if err != nil { - testenv.Log.Error(err, "Unable to get operator", "operator name", testenv.operatorName) + testenv.Log.Error(err, "Unable to get operator", "operatorName", testenv.operatorName) return err } @@ -387,7 +382,7 @@ func (testenv *TestCaseEnv) attachPVCToOperator(name string) error { // update the operator deployment now err = testenv.GetKubeClient().Update(ctx, operator) if err != nil { - testenv.Log.Error(err, "Unable to update operator", "operator name", testenv.operatorName) + testenv.Log.Error(err, "Unable to update operator", "operatorName", testenv.operatorName) return err } diff --git a/test/testenv/testenv.go b/test/testenv/testenv.go index 83576beda..e718ca9e4 100644 --- a/test/testenv/testenv.go +++ b/test/testenv/testenv.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -70,7 +70,7 @@ const ( // C3 deployments require bundle push across all indexers and SHC deployer which can exceed 5 minutes. AppInstallTimeout = 10 * time.Minute - // SearchHeadPod Template String for search head pod + // SearchHeadPod Template String for Search Head pod SearchHeadPod = "splunk-%s-shc-search-head-%d" // DeployerPod Template String for deployer pod @@ -79,10 +79,10 @@ const ( // StandalonePod Template String for standalone pod StandalonePod = "splunk-%s-standalone-%d" - // LicenseManagerPod Template String for standalone pod + // LicenseManagerPod Template String for License Manager pod LicenseManagerPod = "splunk-%s-license-manager-%d" - // LicenseMasterPod Template String for standalone pod + // LicenseMasterPod Template String for License Master pod LicenseMasterPod = "splunk-%s-" + splcommon.LicenseManager + "-%d" // IngestorPod Template String for ingestor pod @@ -132,11 +132,17 @@ const ( // ClusterMasterServiceName Cluster Master Service Template String ClusterMasterServiceName = "splunk-%s-cluster-master-service" - // DeployerServiceName Cluster Manager Service Template String + // DeployerServiceName Deployer Service Template String DeployerServiceName = "splunk-%s-shc-deployer-service" // CRUpdateRetryCount if CR Update fails retry these many time CRUpdateRetryCount = 10 + + // LogLineCount is the default number of log lines to ingest for test data + LogLineCount = 2000 + + // DefaultIngestIndex is the default index name used for test data ingestion + DefaultIngestIndex = "main" ) var ( diff --git a/test/testenv/util.go b/test/testenv/util.go index 90460849b..6d4032eb0 100644 --- a/test/testenv/util.go +++ b/test/testenv/util.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,8 +23,6 @@ import ( "os" "os/exec" "path" - "reflect" - "sort" "strings" "time" @@ -743,7 +741,7 @@ func newStandaloneWithSpec(name, ns string, spec enterpriseApi.StandaloneSpec) * return &new } -// newMonitoringConsoleSpec returns MC Spec with given name, namespace and license manager Ref +// newMonitoringConsoleSpec returns MC Spec with given name, namespace and License Manager Ref func newMonitoringConsoleSpec(name, ns, LicenseManagerRef, splunkImage string) *enterpriseApi.MonitoringConsole { licenseMasterRef, licenseManagerRef := swapLicenseManager(name, LicenseManagerRef) @@ -965,7 +963,7 @@ func ExecuteCommandOnPod(ctx context.Context, deployment *Deployment, podName st logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) return "", err } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) return stdout, nil } @@ -977,7 +975,7 @@ func ExecuteCommandOnOperatorPod(ctx context.Context, deployment *Deployment, po logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "shell", command, "command", stdin, "error", err.Error()) return "", err } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) return stdout, nil } @@ -986,7 +984,7 @@ func GetConfigMap(ctx context.Context, deployment *Deployment, ns string, config configMap := &corev1.ConfigMap{} err := deployment.GetInstance(ctx, configMapName, configMap) if err != nil { - deployment.testenv.Log.Error(err, "Unable to get config map", "Config Map Name", configMap, "Namespace", ns) + deployment.testenv.Log.Error(err, "Unable to get config map", "configMapName", configMap, "namespace", ns) } return configMap, err } @@ -1057,7 +1055,7 @@ func newLicenseManagerWithGivenSpec(name, ns string, spec enterpriseApi.LicenseM return &new } -// newLicenseMasterWithGivenSpec create and initializes CR for License Manager Kind with Given Spec +// newLicenseMasterWithGivenSpec create and initializes CR for License Master Kind with Given Spec func newLicenseMasterWithGivenSpec(name, ns string, spec enterpriseApiV3.LicenseMasterSpec) *enterpriseApiV3.LicenseMaster { new := enterpriseApiV3.LicenseMaster{ TypeMeta: metav1.TypeMeta{ @@ -1115,19 +1113,9 @@ func GetDirsOrFilesInPath(ctx context.Context, deployment *Deployment, podName s return strings.Fields(stdout), err } -// CompareStringSlices checks if two string slices are matching -func CompareStringSlices(stringOne []string, stringTwo []string) bool { - if len(stringOne) != len(stringTwo) { - return false - } - sort.Strings(stringOne) - sort.Strings(stringTwo) - return reflect.DeepEqual(stringOne, stringTwo) -} - // CheckStringInSlice check if string is present in a slice func CheckStringInSlice(stringSlice []string, compString string) bool { - logf.Log.Info("Checking for string in slice", "String", compString, "String Slice", stringSlice) + logf.Log.Info("Checking for string in slice", "string", compString, "stringSlice", stringSlice) for _, item := range stringSlice { if strings.Contains(item, compString) { return true @@ -1170,16 +1158,6 @@ func GetPodUIDs(ns string) map[string]string { return splunkPodUIDs } -// DeletePod Delete pod in the namespace -func DeletePod(ns string, podName string) error { - _, err := exec.Command("kubectl", "delete", "pod", "-n", ns, podName).Output() - if err != nil { - logf.Log.Error(err, "Failed to delete operator pod ", "PodName", podName, "Namespace", ns) - return err - } - return nil -} - // DeleteOperatorPod Delete Operator Pod in the namespace func DeleteOperatorPod(testcaseEnvInst *TestCaseEnv) error { var podName string @@ -1193,7 +1171,7 @@ func DeleteOperatorPod(testcaseEnvInst *TestCaseEnv) error { _, err := exec.Command("kubectl", "delete", "pod", "-n", ns, podName).Output() if err != nil { - logf.Log.Error(err, "Failed to delete operator pod ", "PodName", podName, "Namespace", ns) + logf.Log.Error(err, "Failed to delete operator pod ", "podName", podName, "namespace", ns) return err } return nil @@ -1205,7 +1183,7 @@ func DeleteFilesOnOperatorPod(ctx context.Context, deployment *Deployment, podNa cmd := fmt.Sprintf("rm -f %s", filepath) _, err := ExecuteCommandOnOperatorPod(ctx, deployment, podName, cmd) if err != nil { - logf.Log.Error(err, "Failed to delete file on pod ", "PodName", podName, "location", filepath, "command", cmd) + logf.Log.Error(err, "Failed to delete file on pod ", "podName", podName, "location", filepath, "command", cmd) return err } } @@ -1220,9 +1198,9 @@ func DumpGetSplunkVersion(ctx context.Context, ns string, deployment *Deployment if strings.Contains(podName, filterString) { stdout, err := ExecuteCommandOnPod(ctx, deployment, podName, cmd) if err != nil { - logf.Log.Error(err, "Failed to get splunkd version on the pod", "Pod Name", podName) + logf.Log.Error(err, "Failed to get splunkd version on the pod", "podName", podName) } - logf.Log.Info("Splunk Version Found", "Pod Name", podName, "Version", string(stdout)) + logf.Log.Info("Splunk Version Found", "podName", podName, "version", string(stdout)) } } } @@ -1232,7 +1210,7 @@ func CreateDummyFileOnOperator(ctx context.Context, deployment *Deployment, podN cmd := fmt.Sprintf("cd %s && dd if=/dev/zero of=./%s bs=4k iflag=fullblock,count_bytes count=%s", filepath, filename, size) _, err := ExecuteCommandOnOperatorPod(ctx, deployment, podName, cmd) if err != nil { - logf.Log.Error(err, "Failed to create file on the pod", "Pod Name", podName) + logf.Log.Error(err, "Failed to create file on the pod", "podName", podName) return err } return nil @@ -1240,10 +1218,10 @@ func CreateDummyFileOnOperator(ctx context.Context, deployment *Deployment, podN // DeleteConfigMap Delete configMap in the namespace func DeleteConfigMap(ns string, ConfigMapName string) error { - logf.Log.Info("Delete configMap", "configMap Name", ConfigMapName) + logf.Log.Info("Delete configMap", "configMapName", ConfigMapName) _, err := exec.Command("kubectl", "delete", "configmap", "-n", ns, ConfigMapName).Output() if err != nil { - logf.Log.Error(err, "Failed to delete config Map", "ConfigMap Name", ConfigMapName, "Namespace", ns) + logf.Log.Error(err, "Failed to delete config Map", "configMapName", ConfigMapName, "namespace", ns) return err } return nil diff --git a/test/testenv/verificationutils.go b/test/testenv/verificationutils.go index 4b3bc1331..a6750ba9d 100644 --- a/test/testenv/verificationutils.go +++ b/test/testenv/verificationutils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -83,7 +83,7 @@ func (testenv *TestCaseEnv) VerifyMonitoringConsoleReady(ctx context.Context, de if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for Monitoring Console phase to be ready", "instance", monitoringConsole.ObjectMeta.Name, "Phase", monitoringConsole.Status.Phase) + testenv.Log.Info("Waiting for Monitoring Console phase to be ready", "instance", monitoringConsole.ObjectMeta.Name, "phase", monitoringConsole.Status.Phase) DumpGetPods(testenv.GetName()) return monitoringConsole.Status.Phase @@ -106,7 +106,7 @@ func (testenv *TestCaseEnv) VerifyStandaloneReady(ctx context.Context, deploymen // Refresh the instance to get latest state err = deployment.GetInstance(ctx, standalone.Name, standalone) gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("Standalone reached Ready phase", "instance", standalone.ObjectMeta.Name, "Phase", standalone.Status.Phase) + testenv.Log.Info("Standalone reached Ready phase", "instance", standalone.ObjectMeta.Name, "phase", standalone.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, we should stay in Ready and not flip-flop around @@ -128,13 +128,13 @@ func (testenv *TestCaseEnv) VerifySearchHeadClusterReady(ctx context.Context, de shc := &enterpriseApi.SearchHeadCluster{} err = deployment.GetInstance(ctx, instanceName, shc) gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("SearchHeadCluster reached Ready phase", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.Phase, "DeployerPhase", shc.Status.DeployerPhase) + testenv.Log.Info("SearchHeadCluster reached Ready phase", "instance", shc.ObjectMeta.Name, "phase", shc.Status.Phase, "deployerPhase", shc.Status.DeployerPhase) DumpGetPods(testenv.GetName()) // In a steady state, we should stay in Ready and not flip-flop around gomega.Consistently(func() enterpriseApi.Phase { _ = deployment.GetInstance(ctx, deployment.GetName(), shc) - testenv.Log.Info("Check for Consistency Search Head Cluster phase to be ready", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.Phase) + testenv.Log.Info("Check for Consistency Search Head Cluster phase to be ready", "instance", shc.ObjectMeta.Name, "phase", shc.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "-shc-") return shc.Status.Phase }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) @@ -151,13 +151,13 @@ func (testenv *TestCaseEnv) VerifySingleSiteIndexersReady(ctx context.Context, d idc := &enterpriseApi.IndexerCluster{} err = deployment.GetInstance(ctx, instanceName, idc) gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("IndexerCluster reached Ready phase", "instance", instanceName, "Phase", idc.Status.Phase) + testenv.Log.Info("IndexerCluster reached Ready phase", "instance", instanceName, "phase", idc.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, we should stay in Ready and not flip-flop around gomega.Consistently(func() enterpriseApi.Phase { _ = deployment.GetInstance(ctx, instanceName, idc) - testenv.Log.Info("Check for Consistency indexer instance's phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) + testenv.Log.Info("Check for Consistency indexer instance's phase to be ready", "instance", instanceName, "phase", idc.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "-idxc-indexer-") return idc.Status.Phase }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) @@ -198,15 +198,15 @@ func (testenv *TestCaseEnv) VerifyClusterManagerReady(ctx context.Context, deplo cm := &enterpriseApi.ClusterManager{} err = deployment.GetInstance(ctx, deployment.GetName(), cm) gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("ClusterManager reached Ready phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + testenv.Log.Info("ClusterManager reached Ready phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, cluster-manager should stay in Ready and not flip-flop around gomega.Consistently(func() enterpriseApi.Phase { _ = deployment.GetInstance(ctx, deployment.GetName(), cm) - testenv.Log.Info("Check for Consistency "+splcommon.ClusterManager+" phase to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + testenv.Log.Info("Check for Consistency "+splcommon.ClusterManager+" phase to be ready", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "cluster-manager") - testenv.Log.Info("Check for Consistency cluster-manager phase to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + testenv.Log.Info("Check for Consistency cluster-manager phase to be ready", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) return cm.Status.Phase }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) } @@ -221,13 +221,13 @@ func (testenv *TestCaseEnv) VerifyClusterMasterReady(ctx context.Context, deploy cm := &enterpriseApiV3.ClusterMaster{} err = deployment.GetInstance(ctx, deployment.GetName(), cm) gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("ClusterMaster reached Ready phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + testenv.Log.Info("ClusterMaster reached Ready phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, cluster-master should stay in Ready and not flip-flop around gomega.Consistently(func() enterpriseApi.Phase { _ = deployment.GetInstance(ctx, deployment.GetName(), cm) - testenv.Log.Info("Check for Consistency cluster-master phase to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + testenv.Log.Info("Check for Consistency cluster-master phase to be ready", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) return cm.Status.Phase }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) } @@ -246,7 +246,7 @@ func (testenv *TestCaseEnv) VerifyIndexersReady(ctx context.Context, deployment if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for indexer site instance phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) + testenv.Log.Info("Waiting for indexer site instance phase to be ready", "instance", instanceName, "phase", idc.Status.Phase) DumpGetPods(testenv.GetName()) return idc.Status.Phase @@ -255,14 +255,14 @@ func (testenv *TestCaseEnv) VerifyIndexersReady(ctx context.Context, deployment // In a steady state, we should stay in Ready and not flip-flop around gomega.Consistently(func() enterpriseApi.Phase { _ = deployment.GetInstance(ctx, instanceName, idc) - testenv.Log.Info("Check for Consistency indexer site instance phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) + testenv.Log.Info("Check for Consistency indexer site instance phase to be ready", "instance", instanceName, "phase", idc.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "-idxc-indexer-") return idc.Status.Phase }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) } } -// VerifyIndexerClusterMultisiteStatus verify indexer Cluster is configured as multisite +// VerifyIndexerClusterMultisiteStatus verify Indexer Cluster is configured as multisite func (testenv *TestCaseEnv) VerifyIndexerClusterMultisiteStatus(ctx context.Context, deployment *Deployment, siteCount int) { siteIndexerMap := map[string][]string{} for site := 1; site <= siteCount; site++ { @@ -281,10 +281,10 @@ func (testenv *TestCaseEnv) VerifyIndexerClusterMultisiteStatus(ctx context.Cont command := []string{"/bin/sh"} stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) if err != nil { - testenv.Log.Error(err, "Failed to execute command", "on pod", podName, "command", command) + testenv.Log.Error(err, "Failed to execute command", "onPod", podName, "command", command) return map[string][]string{} } - testenv.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + testenv.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) siteIndexerResponse := ClusterManagerSitesResponse{} json.Unmarshal([]byte(stdout), &siteIndexerResponse) siteIndexerStatus := map[string][]string{} @@ -298,33 +298,24 @@ func (testenv *TestCaseEnv) VerifyIndexerClusterMultisiteStatus(ctx context.Cont }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(siteIndexerMap)) } -// VerifyRFSFMet verify RF SF is met on cluster manager +// VerifyRFSFMet verify RF SF is met on Cluster Manager func (testenv *TestCaseEnv) VerifyRFSFMet(ctx context.Context, deployment *Deployment) { gomega.Eventually(func() bool { rfSfStatus := CheckRFSF(ctx, deployment) - testenv.Log.Info("Verifying RF SF is met", "Status", rfSfStatus) + testenv.Log.Info("Verifying RF SF is met", "status", rfSfStatus) return rfSfStatus }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(true)) } -// VerifyNoDisconnectedSHPresentOnCM is present on cluster manager +// VerifyNoDisconnectedSHPresentOnCM verifies no disconnected SH is present on Cluster Manager func (testenv *TestCaseEnv) VerifyNoDisconnectedSHPresentOnCM(ctx context.Context, deployment *Deployment) { gomega.Consistently(func() bool { shStatus := CheckSearchHeadRemoved(ctx, deployment) - testenv.Log.Info("Verifying no Search Head in DISCONNECTED state present on Cluster Manager", "Status", shStatus) + testenv.Log.Info("Verifying no Search Head in DISCONNECTED state present on Cluster Manager", "status", shStatus) return shStatus }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(true)) } -// VerifyNoSHCInNamespace verify no SHC is present in namespace -func (testenv *TestCaseEnv) VerifyNoSHCInNamespace(deployment *Deployment) { - gomega.Eventually(func() bool { - shcStatus := SHCInNamespace(testenv.GetName()) - testenv.Log.Info("Verifying no Search Head Cluster is present in namespace", "Status", shcStatus) - return shcStatus - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(false)) -} - // VerifyLicenseManagerReady verify LM is in ready status and does not flip flop func (testenv *TestCaseEnv) VerifyLicenseManagerReady(ctx context.Context, deployment *Deployment) { LicenseManager := &enterpriseApi.LicenseManager{} @@ -336,7 +327,7 @@ func (testenv *TestCaseEnv) VerifyLicenseManagerReady(ctx context.Context, deplo return enterpriseApi.PhaseError } testenv.Log.Info("Waiting for License Manager instance status to be ready", - "instance", LicenseManager.ObjectMeta.Name, "Phase", LicenseManager.Status.Phase) + "instance", LicenseManager.ObjectMeta.Name, "phase", LicenseManager.Status.Phase) DumpGetPods(testenv.GetName()) return LicenseManager.Status.Phase @@ -360,7 +351,7 @@ func (testenv *TestCaseEnv) VerifyLicenseMasterReady(ctx context.Context, deploy return enterpriseApi.PhaseError } testenv.Log.Info("Waiting for License Master instance status to be ready", - "instance", LicenseMaster.ObjectMeta.Name, "Phase", LicenseMaster.Status.Phase) + "instance", LicenseMaster.ObjectMeta.Name, "phase", LicenseMaster.Status.Phase) DumpGetPods(testenv.GetName()) return LicenseMaster.Status.Phase @@ -393,7 +384,7 @@ func (testenv *TestCaseEnv) VerifyServiceAccountConfiguredOnPod(deployment *Depl restResponse := PodDetailsStruct{} err = json.Unmarshal([]byte(output), &restResponse) if err != nil { - testenv.Log.Error(err, "Failed to parse cluster Search heads") + testenv.Log.Error(err, "Failed to parse cluster Search Heads") return false } testenv.Log.Info("Service Account on Pod", "found", restResponse.Spec.ServiceAccount, "expected", serviceAccount) @@ -439,15 +430,6 @@ func (testenv *TestCaseEnv) VerifyIndexExistsOnS3(ctx context.Context, deploymen }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) } -// VerifyRollingRestartFinished verify no rolling restart is active -func (testenv *TestCaseEnv) VerifyRollingRestartFinished(ctx context.Context, deployment *Deployment) { - gomega.Eventually(func() bool { - rollingRestartStatus := CheckRollingRestartStatus(ctx, deployment) - testenv.Log.Info("Rolling Restart Status", "active", rollingRestartStatus) - return rollingRestartStatus - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(true)) -} - // VerifyConfOnPod Verify give conf and value on config file on pod func (testenv *TestCaseEnv) VerifyConfOnPod(deployment *Deployment, podName string, confFilePath string, config string, value string) { gomega.Consistently(func() bool { @@ -474,7 +456,7 @@ func (testenv *TestCaseEnv) VerifySearchHeadClusterPhase(ctx context.Context, de if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for Search Head Cluster Phase", "instance", shc.ObjectMeta.Name, "Expected", phase, "Phase", shc.Status.Phase) + testenv.Log.Info("Waiting for Search Head Cluster Phase", "instance", shc.ObjectMeta.Name, "expected", phase, "phase", shc.Status.Phase) DumpGetPods(testenv.GetName()) return shc.Status.Phase @@ -489,7 +471,7 @@ func (testenv *TestCaseEnv) VerifyIndexerClusterPhase(ctx context.Context, deplo if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for Indexer Cluster Phase", "instance", idxc.ObjectMeta.Name, "Expected", phase, "Phase", idxc.Status.Phase) + testenv.Log.Info("Waiting for Indexer Cluster Phase", "instance", idxc.ObjectMeta.Name, "expected", phase, "phase", idxc.Status.Phase) DumpGetPods(testenv.GetName()) return idxc.Status.Phase @@ -504,7 +486,7 @@ func (testenv *TestCaseEnv) VerifyStandalonePhase(ctx context.Context, deploymen if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for Standalone status", "instance", standalone.ObjectMeta.Name, "Expected", phase, " Actual Phase", standalone.Status.Phase) + testenv.Log.Info("Waiting for Standalone status", "instance", standalone.ObjectMeta.Name, "expected", phase, "actualPhase", standalone.Status.Phase) DumpGetPods(testenv.GetName()) return standalone.Status.Phase @@ -519,7 +501,7 @@ func (testenv *TestCaseEnv) VerifyMonitoringConsolePhase(ctx context.Context, de if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for Monitoring Console CR status", "instance", mc.ObjectMeta.Name, "Expected", phase, " Actual Phase", mc.Status.Phase) + testenv.Log.Info("Waiting for Monitoring Console CR status", "instance", mc.ObjectMeta.Name, "expected", phase, "actualPhase", mc.Status.Phase) DumpGetPods(testenv.GetName()) return mc.Status.Phase @@ -614,7 +596,7 @@ func (testenv *TestCaseEnv) VerifyCustomResourceVersionChanged(ctx context.Conte if err != nil { return "-1" } - testenv.Log.Info("Waiting for ", kind, " CR status", "instance", name, "Not Expected", resourceVersion, " Actual Resource Version", newResourceVersion) + testenv.Log.Info("Waiting for ", kind, " CR status", "instance", name, "notExpected", resourceVersion, "actualResourceVersion", newResourceVersion) DumpGetPods(testenv.GetName()) return newResourceVersion @@ -692,7 +674,7 @@ func (testenv *TestCaseEnv) VerifyResourceConstraints(deployment *Deployment, po }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) } -// VerifyClusterManagerPhase verify phase of cluster manager +// VerifyClusterManagerPhase verify phase of Cluster Manager func (testenv *TestCaseEnv) VerifyClusterManagerPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) { cm := &enterpriseApi.ClusterManager{} gomega.Eventually(func() enterpriseApi.Phase { @@ -700,7 +682,7 @@ func (testenv *TestCaseEnv) VerifyClusterManagerPhase(ctx context.Context, deplo if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for cluster-manager Phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase, "Expected", phase) + testenv.Log.Info("Waiting for Cluster Manager Phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase, "expected", phase) DumpGetPods(testenv.GetName()) // Test ClusterManager Phase to see if its ready @@ -708,7 +690,7 @@ func (testenv *TestCaseEnv) VerifyClusterManagerPhase(ctx context.Context, deplo }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(phase)) } -// VerifyClusterMasterPhase verify phase of cluster manager +// VerifyClusterMasterPhase verify phase of Cluster Master func (testenv *TestCaseEnv) VerifyClusterMasterPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) { cm := &enterpriseApiV3.ClusterMaster{} gomega.Eventually(func() enterpriseApi.Phase { @@ -716,7 +698,7 @@ func (testenv *TestCaseEnv) VerifyClusterMasterPhase(ctx context.Context, deploy if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for cluster-manager Phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase, "Expected", phase) + testenv.Log.Info("Waiting for Cluster Master Phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase, "expected", phase) DumpGetPods(testenv.GetName()) // Test ClusterManager Phase to see if its ready @@ -734,9 +716,9 @@ func (testenv *TestCaseEnv) VerifySecretsOnPods(ctx context.Context, deployment comparsion := bytes.Compare([]byte(currentValue), secretValue) if comparsion == 0 { found = true - testenv.Log.Info("Secret Values on POD Match", "Match Expected", match, "Pod Name ", pod, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", currentValue) + testenv.Log.Info("Secret Values on POD Match", "matchExpected", match, "podName", pod, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", currentValue) } else { - testenv.Log.Info("Secret Values on POD DONOT Match", "Match Expected", match, "Pod Name ", pod, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", currentValue) + testenv.Log.Info("Secret Values on POD DONOT Match", "matchExpected", match, "podName", pod, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", currentValue) } gomega.Expect(found).Should(gomega.Equal(match)) } @@ -754,10 +736,10 @@ func (testenv *TestCaseEnv) VerifySecretsOnSecretObjects(ctx context.Context, de secretValueOnSecretObject := currentSecretData.Data[secretKey] comparsion := bytes.Compare(secretValueOnSecretObject, secretValue) if comparsion == 0 { - testenv.Log.Info("Secret Values on Secret Object Match", "Match Expected", match, "Secret Object Name", secretName, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", string(secretValueOnSecretObject)) + testenv.Log.Info("Secret Values on Secret Object Match", "matchExpected", match, "secretObjectName", secretName, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", string(secretValueOnSecretObject)) found = true } else { - testenv.Log.Info("Secret Values on Secret Object DONOT match", "Match Expected", match, "Secret Object Name", secretName, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", string(secretValueOnSecretObject)) + testenv.Log.Info("Secret Values on Secret Object DONOT match", "matchExpected", match, "secretObjectName", secretName, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", string(secretValueOnSecretObject)) } gomega.Expect(found).Should(gomega.Equal(match)) } @@ -769,7 +751,7 @@ func (testenv *TestCaseEnv) VerifySecretsOnSecretObjects(ctx context.Context, de func (testenv *TestCaseEnv) VerifySplunkServerConfSecrets(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) { for _, podName := range verificationPods { keysToMatch := GetKeysToMatch(podName) - testenv.Log.Info("Verificaton Keys Set", "Pod Name", podName, "Keys To Compare", keysToMatch) + testenv.Log.Info("Verificaton Keys Set", "podName", podName, "keysToCompare", keysToMatch) for _, secretName := range keysToMatch { found := false stanza := SecretKeytoServerConfStanza[secretName] @@ -777,10 +759,10 @@ func (testenv *TestCaseEnv) VerifySplunkServerConfSecrets(ctx context.Context, d gomega.Expect(err).To(gomega.Succeed(), "Secret not found in conf file", "Secret Name", secretName) comparsion := strings.Compare(value, string(data[secretName])) if comparsion == 0 { - testenv.Log.Info("Secret Values on server.conf Match", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on server.conf Match", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) found = true } else { - testenv.Log.Info("Secret Values on server.conf DONOT MATCH", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on server.conf DONOT MATCH", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) } gomega.Expect(found).Should(gomega.Equal(match)) } @@ -794,16 +776,16 @@ func (testenv *TestCaseEnv) VerifySplunkInputConfSecrets(deployment *Deployment, for _, podName := range verificationPods { if strings.Contains(podName, "standalone") || strings.Contains(podName, "indexer") { found := false - testenv.Log.Info("Key Verificaton", "Pod Name", podName, "Key", secretName) + testenv.Log.Info("Key Verificaton", "podName", podName, "key", secretName) stanza := SecretKeytoServerConfStanza[secretName] _, value, err := GetSecretFromInputsConf(deployment, podName, testenv.GetName(), "token", stanza) gomega.Expect(err).To(gomega.Succeed(), "Secret not found in conf file", "Secret Name", secretName) comparsion := strings.Compare(value, string(data[secretName])) if comparsion == 0 { - testenv.Log.Info("Secret Values on input.conf Match", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on input.conf Match", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) found = true } else { - testenv.Log.Info("Secret Values on input.conf DONOT MATCH", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on input.conf DONOT MATCH", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) } gomega.Expect(found).Should(gomega.Equal(match)) } @@ -821,7 +803,7 @@ func (testenv *TestCaseEnv) VerifySplunkSecretViaAPI(ctx context.Context, deploy } for _, secretName := range keysToMatch { validKey := false - testenv.Log.Info("Key Verificaton", "Pod Name", podName, "Key", secretName) + testenv.Log.Info("Key Verificaton", "podName", podName, "key", secretName) validKey = CheckSecretViaAPI(ctx, deployment, podName, secretName, string(data[secretName])) gomega.Expect(validKey).Should(gomega.Equal(match)) } @@ -840,7 +822,7 @@ func (testenv *TestCaseEnv) VerifyPVC(deployment *Deployment, ns string, pvcName break } } - testenv.Log.Info("PVC Status Verified", "PVC", pvcName, "STATUS", pvcExists, "EXPECTED", expectedToExist) + testenv.Log.Info("PVC Status Verified", "pvc", pvcName, "status", pvcExists, "expected", expectedToExist) return pvcExists }, verificationTimeout, PollInterval).Should(gomega.Equal(expectedToExist)) } @@ -938,7 +920,7 @@ func (testenv *TestCaseEnv) VerifyAppsInFolder(ctx context.Context, deployment * gomega.Eventually(func() bool { // Using checkAppDirectory here to get all files in case of negative check. GetDirsOrFilesInPath will return files/directory when checkAppDirecotry is FALSE appList, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, checkAppDirectory) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get apps on pod", "Pod", podName) + gomega.Expect(err).To(gomega.Succeed(), "Unable to get apps on pod", "pod", podName) for _, app := range apps { folderName := app + "/" found := CheckStringInSlice(appList, folderName) @@ -956,10 +938,10 @@ func (testenv *TestCaseEnv) VerifyAppsDownloadedOnContainer(ctx context.Context, for _, podName := range pods { appList, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, false) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get apps on pod", "Pod", podName) + gomega.Expect(err).To(gomega.Succeed(), "Unable to get apps on pod", "pod", podName) for _, app := range apps { found := CheckStringInSlice(appList, app) - testenv.Log.Info("Check App files present on the pod", "Pod Name", podName, "App Name", app, "directory", path, "Status", found) + testenv.Log.Info("Check App files present on the pod", "podName", podName, "appName", app, "directory", path, "status", found) gomega.Expect(found).Should(gomega.Equal(true)) } } @@ -972,7 +954,7 @@ func (testenv *TestCaseEnv) VerifyAppsPackageDeletedOnOperatorContainer(ctx cont gomega.Eventually(func() bool { appList, err := GetOperatorDirsOrFilesInPath(ctx, deployment, podName, path, false) if err != nil { - testenv.Log.Error(err, "Unable to get apps on operator pod", "Pod", podName) + testenv.Log.Error(err, "Unable to get apps on operator pod", "pod", podName) return true } found := CheckStringInSlice(appList, app+"_") @@ -990,7 +972,7 @@ func (testenv *TestCaseEnv) VerifyAppsPackageDeletedOnContainer(ctx context.Cont gomega.Eventually(func() bool { appList, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, false) if err != nil { - testenv.Log.Error(err, "Unable to get apps on pod", "Pod", podName) + testenv.Log.Error(err, "Unable to get apps on pod", "pod", podName) return true } found := CheckStringInSlice(appList, app+"_") @@ -1016,7 +998,7 @@ func (testenv *TestCaseEnv) VerifyAppListPhase(ctx context.Context, deployment * testenv.Log.Info(fmt.Sprintf("App deployment info not found yet for app %s (CR %s/%s, AppSource %s), continuing to poll", appName, crKind, name, appSourceName)) return phase // Continue polling } - testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase should not be %s", crKind, name, appName, phase), "Actual Phase", appDeploymentInfo.PhaseInfo.Phase, "App State", appDeploymentInfo) + testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase should not be %s", crKind, name, appName, phase), "actualPhase", appDeploymentInfo.PhaseInfo.Phase, "appState", appDeploymentInfo) return appDeploymentInfo.PhaseInfo.Phase }, deployment.GetTimeout(), PollInterval).ShouldNot(gomega.Equal(phase)) } @@ -1033,9 +1015,9 @@ func (testenv *TestCaseEnv) VerifyAppListPhase(ctx context.Context, deployment * testenv.Log.Info(fmt.Sprintf("App deployment info not found yet for app %s (CR %s/%s, AppSource %s), continuing to poll", appName, crKind, name, appSourceName)) return enterpriseApi.PhaseDownload // Continue polling } - testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase %s", crKind, name, appName, phase), "Actual Phase", appDeploymentInfo.PhaseInfo.Phase, "App Phase Status", appDeploymentInfo.PhaseInfo.Status, "App State", appDeploymentInfo) + testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase %s", crKind, name, appName, phase), "actualPhase", appDeploymentInfo.PhaseInfo.Phase, "appPhaseStatus", appDeploymentInfo.PhaseInfo.Status, "appState", appDeploymentInfo) if appDeploymentInfo.PhaseInfo.Status != enterpriseApi.AppPkgInstallComplete { - testenv.Log.Info("Phase Install Not Complete.", "Phase Found", appDeploymentInfo.PhaseInfo.Phase, "Phase Status Found", appDeploymentInfo.PhaseInfo.Status) + testenv.Log.Info("Phase Install Not Complete.", "phaseFound", appDeploymentInfo.PhaseInfo.Phase, "phaseStatusFound", appDeploymentInfo.PhaseInfo.Status) return enterpriseApi.PhaseDownload } return appDeploymentInfo.PhaseInfo.Phase @@ -1071,7 +1053,7 @@ func (testenv *TestCaseEnv) VerifyPodsInMCConfigMap(ctx context.Context, deploym mcConfigMap, err := GetMCConfigMap(ctx, deployment, testenv.GetName(), mcName) gomega.Expect(err).To(gomega.Succeed(), "Unable to get MC config map") for _, podName := range pods { - testenv.Log.Info("Checking for POD on MC Config Map", "POD Name", podName, "DATA", mcConfigMap.Data) + testenv.Log.Info("Checking for POD on MC Config Map", "podName", podName, "data", mcConfigMap.Data) gomega.Expect(expected).To(gomega.Equal(CheckPodNameInString(podName, mcConfigMap.Data[key])), "Verify Pod in MC Config Map. Pod Name %s.", podName) } } @@ -1079,7 +1061,7 @@ func (testenv *TestCaseEnv) VerifyPodsInMCConfigMap(ctx context.Context, deploym // VerifyPodsInMCConfigString checks if given pod names are present in given KEY of given MC's Config Map func (testenv *TestCaseEnv) VerifyPodsInMCConfigString(ctx context.Context, deployment *Deployment, pods []string, mcName string, expected bool, checkPodIP bool) { for _, podName := range pods { - testenv.Log.Info("Checking pod configured in MC POD Peers String", "Pod Name", podName) + testenv.Log.Info("Checking pod configured in MC POD Peers String", "podName", podName) var found bool if checkPodIP { podIP := GetPodIP(testenv.GetName(), podName) @@ -1100,7 +1082,7 @@ func (testenv *TestCaseEnv) VerifyClusterManagerBundlePush(ctx context.Context, clusterManagerBundleStatus = CMBundlePushstatus(ctx, deployment, previousBundleHash, "cmaster") } if len(clusterManagerBundleStatus) < replicas { - testenv.Log.Info("Bundle push on Pod not complete on all pods", "Pod with bundle push", clusterManagerBundleStatus) + testenv.Log.Info("Bundle push on Pod not complete on all pods", "podWithBundlePush", clusterManagerBundleStatus) return false } clusterPodNames := DumpGetPods(testenv.GetName()) @@ -1109,11 +1091,11 @@ func (testenv *TestCaseEnv) VerifyClusterManagerBundlePush(ctx context.Context, if strings.Contains(podName, "-indexer-") { if _, present := clusterManagerBundleStatus[podName]; present { if clusterManagerBundleStatus[podName] != "Up" { - testenv.Log.Info("Bundle push on Pod not complete", "Pod Name", podName, "Status", clusterManagerBundleStatus[podName]) + testenv.Log.Info("Bundle push on Pod not complete", "podName", podName, "status", clusterManagerBundleStatus[podName]) return false } } else { - testenv.Log.Info("Bundle push not found on pod", "Podname", podName) + testenv.Log.Info("Bundle push not found on pod", "podName", podName) return false } } @@ -1134,7 +1116,7 @@ func (testenv *TestCaseEnv) VerifyDeployerBundlePush(ctx context.Context, deploy } for appName, val := range deployerAppPushStatus { if val < replicas { - testenv.Log.Info("Bundle push not complete on all pods for", "App Name", appName, "Replicas with bundle push", val, "Expected replicas", replicas) + testenv.Log.Info("Bundle push not complete on all pods for", "appName", appName, "replicasWithBundlePush", val, "expectedReplicas", replicas) DumpGetPods(testenv.GetName()) return false @@ -1164,7 +1146,7 @@ func (testenv *TestCaseEnv) VerifyNoPodResetByUID(ctx context.Context, deploymen } } -// WaitForSplunkPodCleanup Wait for cleanup to happend +// WaitForSplunkPodCleanup Wait for cleanup to happen func (testenv *TestCaseEnv) WaitForSplunkPodCleanup(ctx context.Context, deployment *Deployment) { gomega.Eventually(func() int { testenv.Log.Info("Waiting for Splunk Pods to be deleted before running test") @@ -1193,21 +1175,21 @@ func (testenv *TestCaseEnv) VerifyAppRepoState(ctx context.Context, deployment * testenv.Log.Error(err, "Failed to get app deployment info") return 0 } - testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected repo value %d", crKind, name, appName, repoValue), "Actual Value", appDeploymentInfo.RepoState, "App State", appDeploymentInfo) + testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected repo value %d", crKind, name, appName, repoValue), "actualValue", appDeploymentInfo.RepoState, "appState", appDeploymentInfo) return int(appDeploymentInfo.RepoState) }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(repoValue)) } // VerifyIsDeploymentInProgressFlagIsSet verify IsDeploymentInProgress flag is set to true func (testenv *TestCaseEnv) VerifyIsDeploymentInProgressFlagIsSet(ctx context.Context, deployment *Deployment, name string, crKind string) { - testenv.Log.Info("Check IsDeploymentInProgress Flag is set", "CR NAME", name, "CR Kind", crKind) + testenv.Log.Info("Check IsDeploymentInProgress Flag is set", "crName", name, "crKind", crKind) gomega.Eventually(func() bool { isDeploymentInProgress, err := GetIsDeploymentInProgressFlag(ctx, deployment, testenv, name, crKind) if err != nil { testenv.Log.Error(err, "Failed to get isDeploymentInProgress Flag") return false } - testenv.Log.Info("IsDeploymentInProgress Flag status found", "CR NAME", name, "CR Kind", crKind, "IsDeploymentInProgress", isDeploymentInProgress) + testenv.Log.Info("IsDeploymentInProgress Flag status found", "crName", name, "crKind", crKind, "isDeploymentInProgress", isDeploymentInProgress) return isDeploymentInProgress }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) } @@ -1218,7 +1200,7 @@ func (testenv *TestCaseEnv) VerifyFilesInDirectoryOnPod(ctx context.Context, dep gomega.Eventually(func() bool { // Using checkDirectory here to get all files in case of negative check. GetDirsOrFilesInPath will return files/directory when checkDirecotry is FALSE filelist, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, checkDirectory) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get files on pod", "Pod", podName) + gomega.Expect(err).To(gomega.Succeed(), "Unable to get files on pod", "pod", podName) for _, file := range files { found := CheckStringInSlice(filelist, file) testenv.Log.Info("File check", "pod", podName, "filename", file, "path", path, "status", found)