65 update test cases to include partial deletion

.github/workflows/release-pullrequest.yaml

@@ -69,7 +69,7 @@ jobs:
       - name: Gather dependencies
         run: go mod download
       - name: Run coverage
-        run: go test -timeout 300s -race -coverprofile=coverage.txt -covermode=atomic ./... -coverpkg=./...
+        run: go test -timeout 480s -race -coverprofile=coverage.txt -covermode=atomic ./... -coverpkg=./...
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:

.github/workflows/release-tag.yaml

@@ -125,7 +125,7 @@ jobs:
       - name: Gather dependencies
         run: go mod download
       - name: Run coverage
-        run: go test -timeout 300s -race -coverprofile=coverage.txt -covermode=atomic ./... -coverpkg=./...
+        run: go test -timeout 480s -race -coverprofile=coverage.txt -covermode=atomic ./... -coverpkg=./...
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:

internal/controller/focusconfig_controller.go

@@ -176,9 +176,7 @@
 
 	configGroupingByDatabase := utils.CreateGroupings(focusConfigList)
 	for key := range configGroupingByDatabase {
-		if err = utils.CreateExporterCR(ctx, focusConfig.Namespace, key); err != nil {
-			return fmt.Errorf("unable to create exporters from scratch: %v", err)
-		}
+		minPollingIntervalHours := 9999999999
 		for i := range configGroupingByDatabase[key] {
 
 			unstructuredFocusConfigUptd, err := clientHelper.GetObj(ctx, &config.ObjectRef{Name: configGroupingByDatabase[key][i].Name, Namespace: configGroupingByDatabase[key][i].Namespace}, focusConfig.APIVersion, "focusconfigs", e.dynClient)
@@ -197,6 +195,13 @@
 			if err != nil {
 				return fmt.Errorf("could not update focus config %s status: %v", unstructuredFocusConfig.GetName(), err)
 			}
+
+			if configGroupingByDatabase[key][i].Spec.ScraperConfig.PollingIntervalHours < minPollingIntervalHours {
+				minPollingIntervalHours = configGroupingByDatabase[key][i].Spec.ScraperConfig.PollingIntervalHours
+			}
+		}
+		if err = utils.CreateExporterCR(ctx, focusConfig.Namespace, key, minPollingIntervalHours); err != nil {
+			return fmt.Errorf("unable to create exporters from scratch: %v", err)
 		}
 	}
 

internal/utils/utils.go

@@ -15,7 +15,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func CreateExporterCR(ctx context.Context, namespace string, groupKey string) error {
+func CreateExporterCR(ctx context.Context, namespace string, groupKey string, pollingIntervalHours int) error {
 	clientset, err := GetClientSet()
 	if err != nil {
 		return err
@@ -39,7 +39,7 @@ func CreateExporterCR(ctx context.Context, namespace string, groupKey string) er
 	}
 
 	exporterScraperConfigOld, err := GetExporterScraperConfig(ctx, clientset, namespace, deploymentName)
-	exporterScraperConfig := GetExporterScraperObject(namespace, groupKey, api, deploymentName)
+	exporterScraperConfig := GetExporterScraperObject(namespace, groupKey, api, deploymentName, pollingIntervalHours)
 	if err != nil || !checkExporterScraperConfigs(exporterScraperConfigOld, *exporterScraperConfig) {
 		if groupKey != ">>" {
 			DeleteExporterScraperConfig(ctx, clientset, namespace, deploymentName)
@@ -64,14 +64,14 @@ func CreateExporterCR(ctx context.Context, namespace string, groupKey string) er
 	return nil
 }
 
-func GetExporterScraperObject(namespace string, groupKey string, api finopsdatatypes.API, deploymentName string) *finopsdatatypes.ExporterScraperConfig {
+func GetExporterScraperObject(namespace string, groupKey string, api finopsdatatypes.API, deploymentName string, pollingIntervalHours int) *finopsdatatypes.ExporterScraperConfig {
 	additionalVariables := make(map[string]string)
 
 	scaperConfigObject := finopsdatatypes.ScraperConfigSpec{}
 	if groupKey != ">>" {
 		scaperConfigObject = finopsdatatypes.ScraperConfigSpec{
 			TableName:            strings.Split(groupKey, ">")[2],
-			PollingIntervalHours: 6,
+			PollingIntervalHours: pollingIntervalHours,
 			ScraperDatabaseConfigRef: finopsdatatypes.ObjectRef{
 				Name:      strings.Split(groupKey, ">")[1],
 				Namespace: strings.Split(groupKey, ">")[0],
@@ -93,7 +93,7 @@ func GetExporterScraperObject(namespace string, groupKey string, api finopsdatat
 				Provider:             finopsdatatypes.ObjectRef{},
 				API:                  api,
 				MetricType:           "cost",
-				PollingIntervalHours: 6,
+				PollingIntervalHours: scaperConfigObject.PollingIntervalHours,
 				AdditionalVariables:  additionalVariables,
 			},
 			ScraperConfig: scaperConfigObject,

test/e2e/e2e_test.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/rs/zerolog/log"
 	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -280,7 +281,71 @@ billed_cost__2{AvailabilityZone="EU",BilledCost="30000",BillingAccountId="0000",
 				t.Fatal(fmt.Errorf("unexpected exporter output"))
 			}
 			return ctx
-		}).Feature()
+		}).
+		Assess("ValueAfterDelete", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			r := ctx.Value(contextKey("client")).(*resources.Resources)
+
+			toDelete := &operatorfocusapi.FocusConfig{ObjectMeta: metav1.ObjectMeta{
+				Name:      "focusconfig-sample2",
+				Namespace: testNamespace,
+			}}
+
+			err := r.Delete(ctx, toDelete, resources.WithGracePeriod(time.Duration(1)*time.Second))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			time.Sleep(5 * time.Minute) // Wait for the smallest polling interval period possible for the exporter
+
+			deploymentName := ctx.Value(contextKey("deploymentName")).(string)
+
+			p := e2eutils.RunCommand(fmt.Sprintf("kubectl get service -n %s %s -o custom-columns=ports:spec.ports[0].nodePort", testNamespace, deploymentName+"-service"))
+			if p.Err() != nil {
+				t.Fatal(fmt.Errorf("error with kubectl: %s %v", p.Out(), p.Err()))
+			}
+			portString := new(strings.Builder)
+			_, err = io.Copy(portString, p.Out())
+			if err != nil {
+				t.Fatal(err)
+			}
+			portNumber := strings.Split(portString.String(), "\n")[1]
+
+			p = e2eutils.RunCommand("kubectl get nodes -o custom-columns=status:status.addresses[0].address")
+			if p.Err() != nil {
+				t.Fatal(fmt.Errorf("error with kubectl: %s %v", p.Out(), p.Err()))
+			}
+			addressString := new(strings.Builder)
+			_, err = io.Copy(addressString, p.Out())
+			if err != nil {
+				t.Fatal(err)
+			}
+			address := strings.Split(addressString.String(), "\n")[1]
+
+			p = e2eutils.RunCommand(fmt.Sprintf("curl -s %s:%s/metrics", address, portNumber))
+			if p.Err() != nil {
+				t.Fatal(fmt.Errorf("error with curl: %s %v", p.Out(), p.Err()))
+			}
+
+			resultString := new(strings.Builder)
+			_, err = io.Copy(resultString, p.Out())
+			if err != nil {
+				t.Fatal(err)
+			}
+			predictedOutput := `#HELPbilled_cost__1
+#TYPEbilled_cost__1gauge
+billed_cost__1{AvailabilityZone="EU",BilledCost="27000",BillingAccountId="0000",BillingAccountName="testAccount",BillingCurrency="EUR",BillingPeriodEnd="2024-12-31T21:59:59Z",BillingPeriodStart="2023-12-31T22:00:00Z",CapacityReservationId="",CapacityReservationStatus="",ChargeCategory="purchase",ChargeClass="",ChargeDescription="1DellXYZ",ChargeFrequency="one-time",ChargePeriodEnd="2024-12-31T21:59:59Z",ChargePeriodStart="2023-12-31T22:00:00Z",CommitmentDiscountCategory="",CommitmentDiscountId="",CommitmentDiscountName="",CommitmentDiscountQuantity="0",CommitmentDiscountStatus="",CommitmentDiscountType="",CommitmentDiscountUnit="",ConsumedQuantity="3",ConsumedUnit="Computer",ContractedCost="27000",ContractedUnitCost="9000",EffectiveCost="30000",InvoiceIssuerName="Dell",ListCost="30000",ListUnitPrice="10000",#Category="other",#Quantity="3",#Unit="machines",ProviderName="Dell",PublisherName="Dell",RegionId="",RegionName="",ResourceId="0000",ResourceName="DellHW",ResourceType="ProdCluster",ServiceCategory="Compute",ServiceName="1machinepurchase",ServiceSubcategory="test",SkuId="0000",SkuMeter="",SkuPriceDetails="",SkuPriceId="0000",SubAccountId="1234",SubAccountName="test",Tags="testkey1:testvalue;testkey2:testvalue"}27000`
+			if !strings.Contains(strings.Replace(resultString.String(), " ", "", -1), strings.Replace(predictedOutput, " ", "", -1)) {
+				t.Fatal(fmt.Errorf("unexpected exporter output: value missing"))
+			}
+			deletedOutput := `#HELPbilled_cost__2
+#TYPEbilled_cost__2gauge
+billed_cost__2{AvailabilityZone="EU",BilledCost="30000",BillingAccountId="0000",BillingAccountName="testAccount",BillingCurrency="EUR",BillingPeriodEnd="2024-12-31T21:59:59Z",BillingPeriodStart="2023-12-31T22:00:00Z",CapacityReservationId="",CapacityReservationStatus="",ChargeCategory="purchase",ChargeClass="",ChargeDescription="1DellXYZ",ChargeFrequency="one-time",ChargePeriodEnd="2024-12-31T21:59:59Z",ChargePeriodStart="2023-12-31T22:00:00Z",CommitmentDiscountCategory="",CommitmentDiscountId="",CommitmentDiscountName="",CommitmentDiscountQuantity="0",CommitmentDiscountStatus="",CommitmentDiscountType="",CommitmentDiscountUnit="",ConsumedQuantity="3",ConsumedUnit="Computer",ContractedCost="30000",ContractedUnitCost="10000",EffectiveCost="30000",InvoiceIssuerName="Dell",ListCost="30000",ListUnitPrice="10000",#Category="other",#Quantity="3",#Unit="machines",ProviderName="Dell",PublisherName="Dell",RegionId="",RegionName="",ResourceId="0000",ResourceName="DellHW",ResourceType="ProdCluster",ServiceCategory="Compute",ServiceName="1machinepurchase",ServiceSubcategory="test",SkuId="0000",SkuMeter="",SkuPriceDetails="",SkuPriceId="0000",SubAccountId="1234",SubAccountName="test",Tags="testkey1:testvalue;testkey2:testvalue"}30000`
+			if strings.Contains(strings.Replace(resultString.String(), " ", "", -1), strings.Replace(deletedOutput, " ", "", -1)) {
+				t.Fatal(fmt.Errorf("unexpected exporter output: deleted content still present"))
+			}
+			return ctx
+		}).
+		Feature()
 
 	createDual := features.New("Create dual").
 		WithLabel("type", "CR and resources").

test/e2e/manifests/to_test/to_test_1.yaml

@@ -6,7 +6,7 @@ metadata:
 spec:
   scraperConfig:
     tableName: focus_export
-    pollingIntervalHours: 1
+    pollingIntervalHours: 0
     scraperDatabaseConfigRef: 
       name: cratedb-config
       namespace: finops
@@ -63,7 +63,7 @@ metadata:
 spec:
   scraperConfig:
     tableName: focus_export
-    pollingIntervalHours: 1
+    pollingIntervalHours: 0
     scraperDatabaseConfigRef: 
       name: cratedb-config
       namespace: finops

test/e2e/manifests/to_test/to_test_2.yaml

@@ -6,7 +6,7 @@ metadata:
 spec:
   scraperConfig:
     tableName: focus_export_1
-    pollingIntervalHours: 1
+    pollingIntervalHours: 0
     scraperDatabaseConfigRef: 
       name: cratedb-config
       namespace: finops
@@ -63,7 +63,7 @@ metadata:
 spec:
   scraperConfig:
     tableName: focus_export_2
-    pollingIntervalHours: 1
+    pollingIntervalHours: 0
     scraperDatabaseConfigRef: 
       name: cratedb-config
       namespace: finops