Improve caching logic

CHANGELOG.md

@@ -33,6 +33,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 - [#53](https://github.com/kobsio/kobs/pull/53): Improve Jaeger plugin, by allow filtering of services and operations and adding several actions for traces.
 - [#55](https://github.com/kobsio/kobs/pull/55): Allow a user to add a tag from a span as filter in the Jaeger plugin.
 - [#57](https://github.com/kobsio/kobs/pull/57): Visualize the offset of spans in the Jaeger plugin.
+- [#61](https://github.com/kobsio/kobs/pull/61): Improve caching logic, by generating the teams and topology graph only when it is requested and not via an additional goroutine.
 
 ## [v0.2.0](https://github.com/kobsio/kobs/releases/tag/v0.2.0) (2021-04-23)
 

docs/resources/templates.md

@@ -22,7 +22,7 @@ In the following you can found the specification for the Template CRD.
 | name | string | The name of the variable. | Yes |
 | description | string | A description for the variable. | Yes |
 
-## Example
+## Examples
 
 !!! note
     We collect several templates in the [`deploy/templates`](https://github.com/kobsio/kobs/blob/main/deploy/templates) folder. If you have a template, which can also be useful for others, feel free to add it to this folder.

pkg/api/plugins/clusters/cluster/cluster.go

@@ -36,34 +36,17 @@ type Cluster struct {
 	applicationClientset *applicationClientsetVersioned.Clientset
 	teamClientset        *teamClientsetVersioned.Clientset
 	templateClientset    *templateClientsetVersioned.Clientset
-	options              Options
 	name                 string
 	crds                 []*clustersProto.CRD
 }
 
-// Options contains various options, which could be set for a cluster. For example a user can set the cache duration for
-// loaded manifest files and the names of the datasources, which should be used within a cluster.
-type Options struct {
-	cacheDuration time.Duration
-}
-
 // Cache implements a simple caching layer, for the loaded manifest files. The goal of the caching layer is to return
 // the manifests faster to the user.
 type Cache struct {
 	namespaces          []string
 	namespacesLastFetch time.Time
 }
 
-// SetOptions is used to set the options for a cluster. The options are not set during the creation of a cluster, so
-// that we do not have to pass around the options through different functions.
-// We also do not know the datasources befor the cluster name is determined, so that we loop through all loaded clusters
-// and connect the datasource names with the correct cluster.
-func (c *Cluster) SetOptions(cacheDuration time.Duration) {
-	c.options = Options{
-		cacheDuration: cacheDuration,
-	}
-}
-
 // GetName returns the name of the cluster.
 func (c *Cluster) GetName() string {
 	return c.name
@@ -77,10 +60,10 @@ func (c *Cluster) GetCRDs() []*clustersProto.CRD {
 // GetNamespaces returns all namespaces for the cluster. To reduce the latency and the number of API calls, we are
 // "caching" the namespaces. This means that if a new namespace is created in a cluster, this namespaces is only shown
 // after the configured cache duration.
-func (c *Cluster) GetNamespaces(ctx context.Context) ([]string, error) {
+func (c *Cluster) GetNamespaces(ctx context.Context, cacheDuration time.Duration) ([]string, error) {
 	log.WithFields(logrus.Fields{"last fetch": c.cache.namespacesLastFetch}).Tracef("Last namespace fetch.")
 
-	if c.cache.namespacesLastFetch.After(time.Now().Add(-1 * c.options.cacheDuration)) {
+	if c.cache.namespacesLastFetch.After(time.Now().Add(-1 * cacheDuration)) {
 		log.WithFields(logrus.Fields{"cluster": c.name}).Debugf("Return namespaces from cache.")
 
 		return c.cache.namespaces, nil

pkg/api/plugins/clusters/clusters.go

@@ -12,7 +12,6 @@ import (
 	"github.com/kobsio/kobs/pkg/api/plugins/clusters/cluster"
 	clustersProto "github.com/kobsio/kobs/pkg/api/plugins/clusters/proto"
 	"github.com/kobsio/kobs/pkg/api/plugins/clusters/provider"
-	teamProto "github.com/kobsio/kobs/pkg/api/plugins/team/proto"
 	templateProto "github.com/kobsio/kobs/pkg/api/plugins/template/proto"
 
 	"github.com/sirupsen/logrus"
@@ -23,28 +22,37 @@ import (
 
 var (
 	log                     = logrus.WithFields(logrus.Fields{"package": "clusters"})
-	cacheDurationNamespaces string
-	cacheDurationTopology   string
-	cacheDurationTeams      string
+	cacheDurationNamespaces time.Duration
+	cacheDurationTopology   time.Duration
+	cacheDurationTeams      time.Duration
 	cacheDurationTemplates  time.Duration
 	forbiddenResources      []string
 )
 
 // init is used to define all command-line flags for the clusters package.
 func init() {
-	defaultCacheDurationNamespaces := "5m"
+	defaultCacheDurationNamespaces := time.Duration(5 * time.Minute)
 	if os.Getenv("KOBS_CLUSTERS_CACHE_DURATION_NAMESPACES") != "" {
-		defaultCacheDurationNamespaces = os.Getenv("KOBS_CLUSTERS_CACHE_DURATION_NAMESPACES")
+		parsedCacheDurationNamespaces, err := time.ParseDuration(os.Getenv("KOBS_CLUSTERS_CACHE_DURATION_NAMESPACES"))
+		if err == nil {
+			defaultCacheDurationNamespaces = parsedCacheDurationNamespaces
+		}
 	}
 
-	defaultCacheDurationTopology := "60m"
+	defaultCacheDurationTopology := time.Duration(60 * time.Minute)
 	if os.Getenv("KOBS_CLUSTERS_CACHE_DURATION_TOPOLOGY") != "" {
-		defaultCacheDurationTopology = os.Getenv("KOBS_CLUSTERS_CACHE_DURATION_TOPOLOGY")
+		parsedCacheDurationTopology, err := time.ParseDuration(os.Getenv("KOBS_CLUSTERS_CACHE_DURATION_TOPOLOGY"))
+		if err == nil {
+			defaultCacheDurationTopology = parsedCacheDurationTopology
+		}
 	}
 
-	defaultCacheDurationTeams := "60m"
+	defaultCacheDurationTeams := time.Duration(60 * time.Minute)
 	if os.Getenv("KOBS_CLUSTERS_CACHE_DURATION_TEAMS") != "" {
-		defaultCacheDurationTeams = os.Getenv("KOBS_CLUSTERS_CACHE_DURATION_TEAMS")
+		parsedCacheDurationTeams, err := time.ParseDuration(os.Getenv("KOBS_CLUSTERS_CACHE_DURATION_TEAMS"))
+		if err == nil {
+			defaultCacheDurationTeams = parsedCacheDurationTeams
+		}
 	}
 
 	defaultCacheDurationTemplates := time.Duration(60 * time.Minute)
@@ -60,9 +68,9 @@ func init() {
 		defaultForbiddenResources = strings.Split(os.Getenv("KOBS_CLUSTERS_FORBIDDEN_RESOURCES"), ",")
 	}
 
-	flag.StringVar(&cacheDurationNamespaces, "clusters.cache-duration.namespaces", defaultCacheDurationNamespaces, "The duration, for how long requests to get the list of namespaces should be cached.")
-	flag.StringVar(&cacheDurationTopology, "clusters.cache-duration.topology", defaultCacheDurationTopology, "The duration, for how long the topology data should be cached.")
-	flag.StringVar(&cacheDurationTeams, "clusters.cache-duration.teams", defaultCacheDurationTeams, "The duration, for how long the teams data should be cached.")
+	flag.DurationVar(&cacheDurationNamespaces, "clusters.cache-duration.namespaces", defaultCacheDurationNamespaces, "The duration, for how long requests to get the list of namespaces should be cached.")
+	flag.DurationVar(&cacheDurationTopology, "clusters.cache-duration.topology", defaultCacheDurationTopology, "The duration, for how long the topology data should be cached.")
+	flag.DurationVar(&cacheDurationTeams, "clusters.cache-duration.teams", defaultCacheDurationTeams, "The duration, for how long the teams data should be cached.")
 	flag.DurationVar(&cacheDurationTemplates, "clusters.cache-duration.templates", defaultCacheDurationTemplates, "The duration, for how long the templates data should be cached.")
 	flag.StringArrayVar(&forbiddenResources, "clusters.forbidden-resources", defaultForbiddenResources, "A list of resources, which can not be accessed via kobs.")
 }
@@ -88,17 +96,23 @@ type Config struct {
 type Clusters struct {
 	clustersProto.UnimplementedClustersServer
 	clusters []*cluster.Cluster
-	edges    []*clustersProto.Edge
-	nodes    []*clustersProto.Node
-	teams    []Team
 	cache    Cache
 }
 
 type Cache struct {
+	topology           Topology
+	topologyLastFetch  time.Time
+	teams              []Team
+	teamsLastFetch     time.Time
 	templates          []*templateProto.Template
 	templatesLastFetch time.Time
 }
 
+type Topology struct {
+	edges []*clustersProto.Edge
+	nodes []*clustersProto.Node
+}
+
 func (c *Clusters) getCluster(name string) *cluster.Cluster {
 	for _, cl := range c.clusters {
 		if cl.GetName() == name {
@@ -148,7 +162,7 @@ func (c *Clusters) GetNamespaces(ctx context.Context, getNamespacesRequest *clus
 			return nil, fmt.Errorf("invalid cluster name")
 		}
 
-		clusterNamespaces, err := cluster.GetNamespaces(ctx)
+		clusterNamespaces, err := cluster.GetNamespaces(ctx, cacheDurationNamespaces)
 		if err != nil {
 			return nil, err
 		}
@@ -172,7 +186,7 @@ func (c *Clusters) GetNamespaces(ctx context.Context, getNamespacesRequest *clus
 		return uniqueNamespaces[i] < uniqueNamespaces[j]
 	})
 
-	log.WithFields(logrus.Fields{"namespaces": uniqueNamespaces}).Tracef("GetNamespaces")
+	log.WithFields(logrus.Fields{"namespaces": len(uniqueNamespaces)}).Tracef("GetNamespaces")
 
 	return &clustersProto.GetNamespacesResponse{
 		Namespaces: uniqueNamespaces,
@@ -337,20 +351,34 @@ func (c *Clusters) GetApplication(ctx context.Context, getApplicationRequest *cl
 func (c *Clusters) GetTeams(ctx context.Context, getTeamsRequest *clustersProto.GetTeamsRequest) (*clustersProto.GetTeamsResponse, error) {
 	log.Tracef("GetTeams")
 
-	var teams []*teamProto.Team
+	if c.cache.teamsLastFetch.After(time.Now().Add(-1 * cacheDurationTeams)) {
+		return &clustersProto.GetTeamsResponse{
+			Teams: transformCachedTeams(c.cache.teams),
+		}, nil
+	}
+
+	if c.cache.teams == nil {
+		teams := getTeams(ctx, c.clusters)
+		if teams != nil {
+			c.cache.teamsLastFetch = time.Now()
+			c.cache.teams = teams
+		}
 
-	for _, team := range c.teams {
-		teams = append(teams, &teamProto.Team{
-			Name:        team.Name,
-			Description: team.Description,
-			Logo:        team.Logo,
-		})
+		return &clustersProto.GetTeamsResponse{
+			Teams: transformCachedTeams(teams),
+		}, nil
 	}
 
-	log.WithFields(logrus.Fields{"count": len(teams)}).Tracef("GetTeams")
+	go func() {
+		teams := getTeams(ctx, c.clusters)
+		if teams != nil {
+			c.cache.teamsLastFetch = time.Now()
+			c.cache.teams = teams
+		}
+	}()
 
 	return &clustersProto.GetTeamsResponse{
-		Teams: teams,
+		Teams: transformCachedTeams(c.cache.teams),
 	}, nil
 }
 
@@ -365,7 +393,15 @@ func (c *Clusters) GetTeams(ctx context.Context, getTeamsRequest *clustersProto.
 func (c *Clusters) GetTeam(ctx context.Context, getTeamRequest *clustersProto.GetTeamRequest) (*clustersProto.GetTeamResponse, error) {
 	log.WithFields(logrus.Fields{"name": getTeamRequest.Name}).Tracef("GetTeam")
 
-	teamShort := getTeamData(c.teams, getTeamRequest.Name)
+	if c.cache.teams == nil {
+		teams := getTeams(ctx, c.clusters)
+		if teams != nil {
+			c.cache.teamsLastFetch = time.Now()
+			c.cache.teams = teams
+		}
+	}
+
+	teamShort := getTeamData(c.cache.teams, getTeamRequest.Name)
 	if teamShort == nil {
 		return nil, fmt.Errorf("invalid team name")
 	}
@@ -444,50 +480,42 @@ func (c *Clusters) GetTemplates(ctx context.Context, getTemplatesRequest *cluste
 // GetApplicationsTopology returns the topology for the given list of clusters and namespaces. We add an additional node
 // for each cluster and namespace. These nodes are used to group the applications by the cluster and namespace.
 func (c *Clusters) GetApplicationsTopology(ctx context.Context, getApplicationsTopologyRequest *clustersProto.GetApplicationsTopologyRequest) (*clustersProto.GetApplicationsTopologyResponse, error) {
-	var edges []*clustersProto.Edge
-	var nodes []*clustersProto.Node
-
-	for _, clusterName := range getApplicationsTopologyRequest.Clusters {
-		nodes = append(nodes, &clustersProto.Node{
-			Id:        clusterName,
-			Label:     clusterName,
-			Type:      "cluster",
-			Parent:    "",
-			Cluster:   clusterName,
-			Namespace: "",
-			Name:      "",
-		})
-
-		for _, namespace := range getApplicationsTopologyRequest.Namespaces {
-			nodes = append(nodes, &clustersProto.Node{
-				Id:        clusterName + "-" + namespace,
-				Label:     namespace,
-				Type:      "namespace",
-				Parent:    clusterName,
-				Cluster:   clusterName,
-				Namespace: namespace,
-				Name:      "",
-			})
+	log.Tracef("GetApplicationsTopology")
 
-			for _, edge := range c.edges {
-				if (edge.SourceCluster == clusterName && edge.SourceNamespace == namespace) || (edge.TargetCluster == clusterName && edge.TargetNamespace == namespace) {
-					edges = appendEdgeIfMissing(edges, edge)
-				}
-			}
-		}
+	if c.cache.topologyLastFetch.After(time.Now().Add(-1 * cacheDurationTopology)) {
+		edges, nodes := generateTopology(c.cache.topology, getApplicationsTopologyRequest.Clusters, getApplicationsTopologyRequest.Namespaces)
+		return &clustersProto.GetApplicationsTopologyResponse{
+			Edges: edges,
+			Nodes: nodes,
+		}, nil
 	}
 
-	for _, edge := range edges {
-		for _, node := range c.nodes {
-			if node.Id == edge.Source || node.Id == edge.Target {
-				nodes = appendNodeIfMissing(nodes, node)
-			}
+	if c.cache.topology.nodes == nil {
+		topology := getTopology(ctx, c.clusters)
+		if topology.nodes != nil {
+			c.cache.topologyLastFetch = time.Now()
+			c.cache.topology = topology
 		}
+
+		edges, nodes := generateTopology(topology, getApplicationsTopologyRequest.Clusters, getApplicationsTopologyRequest.Namespaces)
+
+		return &clustersProto.GetApplicationsTopologyResponse{
+			Edges: edges,
+			Nodes: nodes,
+		}, nil
 	}
 
+	go func() {
+		topology := getTopology(ctx, c.clusters)
+		if topology.nodes != nil {
+			c.cache.topologyLastFetch = time.Now()
+			c.cache.topology = topology
+		}
+	}()
+
 	return &clustersProto.GetApplicationsTopologyResponse{
-		Edges: edges,
-		Nodes: nodes,
+		Edges: c.cache.topology.edges,
+		Nodes: c.cache.topology.nodes,
 	}, nil
 }
 
@@ -508,21 +536,9 @@ func Load(config Config) (*Clusters, error) {
 		}
 	}
 
-	d, err := time.ParseDuration(cacheDurationNamespaces)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, c := range clusters {
-		c.SetOptions(d)
-	}
-
 	cs := &Clusters{
 		clusters: clusters,
 	}
 
-	go cs.generateTopology()
-	go cs.generateTeams()
-
 	return cs, nil
 }