Add a tool to measure the performance characteristics of providers

Makefile

@@ -18,7 +18,7 @@ PLATFORMS ?= linux_amd64 linux_arm64 darwin_amd64 darwin_arm64
 # Setup Go
 GO_REQUIRED_VERSION = 1.19
 GOLANGCILINT_VERSION ?= 1.50.0
-GO_STATIC_PACKAGES = $(GO_PROJECT)/cmd/uptest $(GO_PROJECT)/cmd/updoc $(GO_PROJECT)/cmd/ttr
+GO_STATIC_PACKAGES = $(GO_PROJECT)/cmd/uptest $(GO_PROJECT)/cmd/updoc $(GO_PROJECT)/cmd/ttr $(GO_PROJECT)/cmd/perf
 GO_LDFLAGS += -X $(GO_PROJECT)/internal/version.Version=$(VERSION)
 GO_SUBDIRS += cmd internal
 GO111MODULE = on

cmd/perf/internal/common/common.go

@@ -0,0 +1,106 @@
+// Copyright 2022 The Crossplane Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package common is for some common functions for tooling
+package common
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/client_golang/api"
+	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+)
+
+// Data represents a collected data
+type Data struct {
+	Timestamp time.Time
+	Value     float64
+}
+
+// Result represents all collected data for a metric
+type Result struct {
+	Data          []Data
+	Metric        string
+	MetricUnit    string
+	Peak, Average float64
+}
+
+// ConstructPrometheusClient creates a Prometheus API Client
+func ConstructPrometheusClient(address string) (v1.API, error) {
+	client, err := api.NewClient(api.Config{
+		Address: address,
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating client")
+	}
+
+	return v1.NewAPI(client), nil
+}
+
+// ConstructTimeRange creates a Range object that consists the start time, end time and step duration
+func ConstructTimeRange(startTime, endTime time.Time, stepDuration time.Duration) v1.Range {
+	return v1.Range{
+		Start: startTime,
+		End:   endTime,
+		Step:  stepDuration,
+	}
+}
+
+// ConstructResult creates a Result object from collected data
+func ConstructResult(value model.Value, metric, unit string) (*Result, error) {
+	result := &Result{}
+	matrix, ok := value.(model.Matrix)
+	if !ok {
+		return nil, errors.New("model cannot cast to matrix")
+	}
+
+	for _, m := range matrix {
+		for _, v := range m.Values {
+			valueNum := float64(v.Value)
+			result.Data = append(result.Data, Data{Timestamp: v.Timestamp.Time(), Value: valueNum})
+		}
+	}
+
+	result.Average, result.Peak = CalculateAverageAndPeak(result.Data)
+	result.Metric = metric
+	result.MetricUnit = unit
+	return result, nil
+}
+
+// CalculateAverageAndPeak calculates the average and peak values of related metric
+func CalculateAverageAndPeak(data []Data) (float64, float64) {
+	var sum, peak float64
+	for _, d := range data {
+		sum += d.Value
+
+		if d.Value > peak {
+			peak = d.Value
+		}
+	}
+	return sum / float64(len(data)), peak
+}
+
+// Print reports the results
+func (r Result) Print() {
+	log.Info(fmt.Sprintf("Average %s: %f %s \n", r.Metric, r.Average, r.MetricUnit))
+	log.Info(fmt.Sprintf("Peak %s: %f %s \n", r.Metric, r.Peak, r.MetricUnit))
+}

cmd/perf/internal/managed/managed.go

@@ -0,0 +1,260 @@
+// Copyright 2022 The Crossplane Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package managed contains functions about managed resource management during the experiment
+package managed
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+
+	log "github.com/sirupsen/logrus"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	"gopkg.in/yaml.v2"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	"k8s.io/client-go/dynamic"
+
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	"github.com/upbound/uptest/cmd/perf/internal/common"
+)
+
+// RunExperiment runs the experiment according to command-line inputs.
+// Firstly the input manifests are deployed. After the all MRs are ready, time to readiness metrics are calculated.
+// Then, by default, all deployed MRs are deleted.
+func RunExperiment(mrTemplatePaths map[string]int, clean bool) ([]common.Result, error) {
+	var timeToReadinessResults []common.Result
+
+	client := createDynamicClient()
+
+	tmpFileName, err := applyResources(mrTemplatePaths)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot apply resources")
+	}
+
+	if err := checkReadiness(client, mrTemplatePaths); err != nil {
+		return nil, errors.Wrap(err, "cannot check readiness of resources")
+	}
+
+	timeToReadinessResults, err = calculateReadinessDuration(client, mrTemplatePaths)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot calculate time to readiness")
+	}
+
+	if clean {
+		log.Info("Deleting resources...")
+		if err := deleteResources(tmpFileName); err != nil {
+			return nil, errors.Wrap(err, "cannot delete resources")
+		}
+	}
+	return timeToReadinessResults, nil
+}
+
+func applyResources(mrTemplatePaths map[string]int) (string, error) {
+	f, err := os.CreateTemp("/tmp", "")
+	if err != nil {
+		return "", errors.Wrap(err, "cannot create input file")
+	}
+
+	for mrPath, count := range mrTemplatePaths {
+		m, err := readYamlFile(mrPath)
+		if err != nil {
+			return "", errors.Wrap(err, "cannot read template file")
+		}
+
+		for i := 1; i <= count; i++ {
+			m["metadata"].(map[interface{}]interface{})["name"] = fmt.Sprintf("test%d", i)
+
+			b, err := yaml.Marshal(m)
+			if err != nil {
+				return "", errors.Wrap(err, "cannot marshal object")
+			}
+
+			if _, err := f.Write(b); err != nil {
+				return "", errors.Wrap(err, "cannot write manifest")
+			}
+
+			if _, err := f.WriteString("\n---\n\n"); err != nil {
+				return "", errors.Wrap(err, "cannot write yaml separator")
+			}
+		}
+
+		if err := f.Close(); err != nil {
+			return "", errors.Wrap(err, "cannot close input file")
+		}
+
+		if err := runCommand(fmt.Sprintf(`"kubectl" apply -f %s`, f.Name())); err != nil {
+			return "", errors.Wrap(err, "cannot execute kubectl apply command")
+		}
+	}
+	return f.Name(), nil
+}
+
+func deleteResources(tmpFileName string) error {
+	if err := runCommand(fmt.Sprintf(`"kubectl" delete -f %s`, tmpFileName)); err != nil {
+		return errors.Wrap(err, "cannot execute kubectl delete command")
+	}
+	return nil
+}
+
+func checkReadiness(client dynamic.Interface, mrTemplatePaths map[string]int) error {
+	for mrPath := range mrTemplatePaths {
+		m, err := readYamlFile(mrPath)
+		if err != nil {
+			return errors.Wrap(err, "cannot read template file")
+		}
+
+		for {
+			log.Info("Checking readiness of resources...")
+			list, err := client.Resource(prepareGVR(m)).List(context.TODO(), metav1.ListOptions{})
+			if err != nil {
+				return errors.Wrap(err, "cannot list resources")
+			}
+			if isReady(list) {
+				break
+			}
+			time.Sleep(10 * time.Second)
+		}
+	}
+	return nil
+}
+
+func isReady(list *unstructured.UnstructuredList) bool {
+	for _, l := range list.Items {
+		if l.Object["status"] == nil {
+			return false
+		}
+		conditions := l.Object["status"].(map[string]interface{})["conditions"].([]interface{})
+
+		status := ""
+		for _, condition := range conditions {
+			c := condition.(map[string]interface{})
+			if c["type"] == "Ready" {
+				status = c["status"].(string)
+			}
+		}
+
+		if status == "False" || status == "" {
+			return false
+		}
+	}
+	return true
+}
+
+func calculateReadinessDuration(client dynamic.Interface, mrTemplatePaths map[string]int) ([]common.Result, error) {
+	var results []common.Result //nolint:prealloc // The size of the slice is not previously known.
+	for mrPath := range mrTemplatePaths {
+		log.Info("Calculating readiness time of resources...")
+		var result common.Result
+
+		m, err := readYamlFile(mrPath)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot read template file")
+		}
+
+		list, err := client.Resource(prepareGVR(m)).List(context.TODO(), metav1.ListOptions{})
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot list resources")
+		}
+		for _, l := range list.Items {
+			readinessTime := metav1.Time{}
+			creationTimestamp := l.GetCreationTimestamp()
+			conditions := l.Object["status"].(map[string]interface{})["conditions"].([]interface{})
+
+			for _, condition := range conditions {
+				c := condition.(map[string]interface{})
+
+				if c["type"] == "Ready" && c["status"] == "True" {
+					t, err := time.Parse(time.RFC3339, c["lastTransitionTime"].(string))
+					if err != nil {
+						return nil, err
+					}
+					readinessTime.Time = t
+
+					diff := readinessTime.Sub(creationTimestamp.Time)
+					result.Data = append(result.Data, common.Data{Value: diff.Seconds()})
+					break
+				}
+			}
+		}
+		result.Metric = fmt.Sprintf("Time to Readiness of %s", m["kind"])
+		result.MetricUnit = "seconds"
+		result.Average, result.Peak = common.CalculateAverageAndPeak(result.Data)
+		results = append(results, result)
+	}
+	return results, nil
+}
+
+func prepareGVR(m map[interface{}]interface{}) schema.GroupVersionResource {
+	apiVersion := strings.Split(m["apiVersion"].(string), "/")
+	kind := strings.ToLower(m["kind"].(string))
+	pluralGVR, _ := meta.UnsafeGuessKindToResource(
+		schema.GroupVersionKind{
+			Group:   apiVersion[0],
+			Version: apiVersion[1],
+			Kind:    kind,
+		},
+	)
+	return pluralGVR
+}
+
+func readYamlFile(fileName string) (map[interface{}]interface{}, error) {
+	yamlFile, err := os.ReadFile(filepath.Clean(fileName))
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot read file")
+	}
+
+	m := make(map[interface{}]interface{})
+	err = yaml.Unmarshal(yamlFile, m)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot marshal map")
+	}
+
+	return m, nil
+}
+
+func createDynamicClient() dynamic.Interface {
+	return dynamic.NewForConfigOrDie(ctrl.GetConfigOrDie())
+}
+
+func runCommand(command string) error {
+	cmd := exec.Command("bash", "-c", command) // #nosec G204
+	stdout, _ := cmd.StdoutPipe()
+	if err := cmd.Start(); err != nil {
+		return errors.Wrap(err, "cannot start kubectl")
+	}
+	sc := bufio.NewScanner(stdout)
+	sc.Split(bufio.ScanLines)
+	for sc.Scan() {
+		fmt.Println(sc.Text())
+	}
+	if err := cmd.Wait(); err != nil {
+		return errors.Wrap(err, "cannot wait for the command exit")
+	}
+	return nil
+}