feat: ram_execution_status metric

BrunoReboul · Jan 15, 2021 · 705ccc2 · 705ccc2
1 parent aae4e5b
commit 705ccc2
Show file tree

Hide file tree

Showing 6 changed files with 99 additions and 38 deletions.
diff --git a/services/setlogmetrics/meth_instancedeployment_situate.go b/services/setlogmetrics/meth_instancedeployment_situate.go
@@ -16,51 +16,51 @@ package setlogmetrics
 
 import (
 	"fmt"
+	"strings"
 
 	"google.golang.org/api/logging/v2"
 )
 
 // Situate complement settings taking in account the situation for service and instance settings
 func (instanceDeployment *InstanceDeployment) Situate() (err error) {
-	metricName := fmt.Sprintf("projects/%s/metrics/%s",
-		instanceDeployment.Core.SolutionSettings.Hosting.ProjectID,
-		instanceDeployment.Settings.Instance.GLO.MetricID)
-
-	instanceDeployment.Artifacts.LogMetric.Name = metricName
+	instanceDeployment.Artifacts.LogMetric.Name = instanceDeployment.Settings.Instance.GLO.MetricID
 	instanceDeployment.Artifacts.LogMetric.Description = instanceDeployment.Settings.Instance.GLO.Description
 	instanceDeployment.Artifacts.LogMetric.Filter = instanceDeployment.Settings.Instance.GLO.Filter
 	instanceDeployment.Artifacts.LogMetric.ValueExtractor = instanceDeployment.Settings.Instance.GLO.ValueExtractor
 
-	var explicitBuckets logging.Explicit
-	var exponentialBuckets logging.Exponential
-	var linearBuckets logging.Linear
-	var bucketOptions logging.BucketOptions
-	instanceDeployment.Artifacts.LogMetric.BucketOptions = &bucketOptions
-	if len(instanceDeployment.Settings.Instance.GLO.BucketOptions.ExplicitBuckets.Bounds) > 0 {
-		bucketOptions.ExplicitBuckets = &explicitBuckets
-		explicitBuckets.Bounds = instanceDeployment.Settings.Instance.GLO.BucketOptions.ExplicitBuckets.Bounds
-	} else {
-		if instanceDeployment.Settings.Instance.GLO.BucketOptions.ExponentialBuckets.NumFiniteBuckets != 0 {
-			bucketOptions.ExponentialBuckets = &exponentialBuckets
-			exponentialBuckets.GrowthFactor = instanceDeployment.Settings.Instance.GLO.BucketOptions.ExponentialBuckets.GrowthFactor
-			exponentialBuckets.NumFiniteBuckets = instanceDeployment.Settings.Instance.GLO.BucketOptions.ExponentialBuckets.NumFiniteBuckets
-			exponentialBuckets.Scale = instanceDeployment.Settings.Instance.GLO.BucketOptions.ExponentialBuckets.Scale
+	if strings.ToLower(instanceDeployment.Settings.Instance.GLO.MetricDescriptor.ValueType) == "distribution" {
+		var explicitBuckets logging.Explicit
+		var exponentialBuckets logging.Exponential
+		var linearBuckets logging.Linear
+		var bucketOptions logging.BucketOptions
+		instanceDeployment.Artifacts.LogMetric.BucketOptions = &bucketOptions
+		if len(instanceDeployment.Settings.Instance.GLO.BucketOptions.ExplicitBuckets.Bounds) > 0 {
+			bucketOptions.ExplicitBuckets = &explicitBuckets
+			explicitBuckets.Bounds = instanceDeployment.Settings.Instance.GLO.BucketOptions.ExplicitBuckets.Bounds
 		} else {
-			if instanceDeployment.Settings.Instance.GLO.BucketOptions.LinearBuckets.NumFiniteBuckets != 0 {
-				bucketOptions.LinearBuckets = &linearBuckets
-				linearBuckets.NumFiniteBuckets = instanceDeployment.Settings.Instance.GLO.BucketOptions.LinearBuckets.NumFiniteBuckets
-				linearBuckets.Offset = instanceDeployment.Settings.Instance.GLO.BucketOptions.LinearBuckets.Offset
-				linearBuckets.Width = instanceDeployment.Settings.Instance.GLO.BucketOptions.LinearBuckets.Width
+			if instanceDeployment.Settings.Instance.GLO.BucketOptions.ExponentialBuckets.NumFiniteBuckets != 0 {
+				bucketOptions.ExponentialBuckets = &exponentialBuckets
+				exponentialBuckets.GrowthFactor = instanceDeployment.Settings.Instance.GLO.BucketOptions.ExponentialBuckets.GrowthFactor
+				exponentialBuckets.NumFiniteBuckets = instanceDeployment.Settings.Instance.GLO.BucketOptions.ExponentialBuckets.NumFiniteBuckets
+				exponentialBuckets.Scale = instanceDeployment.Settings.Instance.GLO.BucketOptions.ExponentialBuckets.Scale
+			} else {
+				if instanceDeployment.Settings.Instance.GLO.BucketOptions.LinearBuckets.NumFiniteBuckets != 0 {
+					bucketOptions.LinearBuckets = &linearBuckets
+					linearBuckets.NumFiniteBuckets = instanceDeployment.Settings.Instance.GLO.BucketOptions.LinearBuckets.NumFiniteBuckets
+					linearBuckets.Offset = instanceDeployment.Settings.Instance.GLO.BucketOptions.LinearBuckets.Offset
+					linearBuckets.Width = instanceDeployment.Settings.Instance.GLO.BucketOptions.LinearBuckets.Width
+				}
 			}
 		}
+	} else {
+		instanceDeployment.Artifacts.LogMetric.BucketOptions = nil
 	}
 
 	var metricDescriptor logging.MetricDescriptor
 	instanceDeployment.Artifacts.LogMetric.MetricDescriptor = &metricDescriptor
 	metricDescriptor.Type = fmt.Sprintf("logging.googleapis.com/user/%s", instanceDeployment.Settings.Instance.GLO.MetricID)
-	metricDescriptor.Name = fmt.Sprintf("projects/%s/metricDescriptors/logging.googleapis.com/user/%s",
-		instanceDeployment.Core.SolutionSettings.Hosting.ProjectID,
-		instanceDeployment.Settings.Instance.GLO.MetricID)
+	metricDescriptor.Name = instanceDeployment.Settings.Instance.GLO.MetricID
+	metricDescriptor.DisplayName = instanceDeployment.Settings.Instance.GLO.MetricID
 	metricDescriptor.Description = instanceDeployment.Settings.Instance.GLO.Description
 	metricDescriptor.LaunchStage = instanceDeployment.Settings.Instance.GLO.MetricDescriptor.LaunchStage
 	metricDescriptor.MetricKind = instanceDeployment.Settings.Instance.GLO.MetricDescriptor.MetricKind

diff --git a/utilities/ffo/func_yamlmarshallprint.go b/utilities/ffo/func_yamlmarshallprint.go
@@ -0,0 +1,31 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the 'License');
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an 'AS IS' BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ffo
+
+import (
+	"fmt"
+	"log"
+
+	"gopkg.in/yaml.v2"
+)
+
+// YAMLMarshalPrint marshal fmt print
+func YAMLMarshalPrint(v interface{}) {
+	bytes, err := yaml.Marshal(v)
+	if err != nil {
+		log.Printf("YAMLMarshalPrint %v", err)
+	}
+	fmt.Println(string(bytes))
+}
diff --git a/utilities/glo/func_checklogmetric.go b/utilities/glo/func_checklogmetric.go
@@ -108,7 +108,7 @@ func checkLogMetric(logMetric, retrievedLogMetric *logging.LogMetric) (err error
 				}
 			}
 		} else {
-			s = fmt.Sprintf("%snot found retrievedLogMetric.BucketOptions\n", s)
+			// s = fmt.Sprintf("%snot found retrievedLogMetric.BucketOptions\n", s)
 		}
 	}
 	if logMetric.MetricDescriptor != nil {
@@ -118,11 +118,11 @@ func checkLogMetric(logMetric, retrievedLogMetric *logging.LogMetric) (err error
 					logMetric.MetricDescriptor.Type,
 					retrievedLogMetric.MetricDescriptor.Type)
 			}
-			if logMetric.MetricDescriptor.Name != retrievedLogMetric.MetricDescriptor.Name {
-				s = fmt.Sprintf("%sMetricDescriptor.Name\nwant %s\nhave %s\n", s,
-					logMetric.MetricDescriptor.Name,
-					retrievedLogMetric.MetricDescriptor.Name)
-			}
+			// if logMetric.MetricDescriptor.Name != retrievedLogMetric.MetricDescriptor.Name {
+			// 	s = fmt.Sprintf("%sMetricDescriptor.Name\nwant %s\nhave %s\n", s,
+			// 		logMetric.MetricDescriptor.Name,
+			// 		retrievedLogMetric.MetricDescriptor.Name)
+			// }
 			if logMetric.MetricDescriptor.Description != retrievedLogMetric.MetricDescriptor.Description {
 				s = fmt.Sprintf("%sMetricDescriptor.Description\nwant %s\nhave %s\n", s,
 					logMetric.MetricDescriptor.Description,

diff --git a/utilities/glo/meth_metricdeployment_deploy.go b/utilities/glo/meth_metricdeployment_deploy.go
@@ -29,11 +29,13 @@ func (logMetricDeployment LogMetricDeployment) Deploy() (err error) {
 
 	metricFound := true
 	// GET
-	// ffo.MarshalYAMLWrite("./metric.yaml", logMetricDeployment.Artifacts.LogMetric)
-	retrievedLogMetric, err := projectMetricsService.Get(logMetricDeployment.Artifacts.LogMetric.Name).Context(logMetricDeployment.Core.Ctx).Do()
+	metricName := fmt.Sprintf("projects/%s/metrics/%s",
+		logMetricDeployment.Core.SolutionSettings.Hosting.ProjectID,
+		logMetricDeployment.Settings.Instance.GLO.MetricID)
+
+	retrievedLogMetric, err := projectMetricsService.Get(metricName).Context(logMetricDeployment.Core.Ctx).Do()
 
 	if err != nil {
-		fmt.Println(err.Error())
 		if strings.Contains(strings.ToLower(err.Error()), "notfound") {
 			metricFound = false
 		} else {
@@ -47,22 +49,25 @@ func (logMetricDeployment LogMetricDeployment) Deploy() (err error) {
 		}
 		log.Printf("%s glo create metric start", logMetricDeployment.Core.InstanceName)
 		parent := fmt.Sprintf("projects/%s", logMetricDeployment.Core.SolutionSettings.Hosting.ProjectID)
+		// ffo.YAMLMarshalPrint(&logMetricDeployment.Artifacts.LogMetric)
 		createdLogMetric, err := projectMetricsService.Create(parent, &logMetricDeployment.Artifacts.LogMetric).Context(logMetricDeployment.Core.Ctx).Do()
 		if err != nil {
 			return fmt.Errorf("projectMetricsService.Create %v", err)
 		}
+		// ffo.YAMLMarshalPrint(&createdLogMetric)
 		log.Printf("%s glo metric created %s", logMetricDeployment.Core.InstanceName, createdLogMetric.Name)
 	} else {
 		log.Printf("%s glo found log metric %s",
 			logMetricDeployment.Core.InstanceName,
 			retrievedLogMetric.Name)
+		// ffo.YAMLMarshalPrint(&retrievedLogMetric)
 		err = checkLogMetric(&logMetricDeployment.Artifacts.LogMetric, retrievedLogMetric)
 		if err != nil {
 			if logMetricDeployment.Core.Commands.Check {
 				return err
 			}
 			log.Printf("%s glo metric meed to be updated", logMetricDeployment.Core.InstanceName)
-			updatedLogMetric, err := projectMetricsService.Update(logMetricDeployment.Artifacts.LogMetric.Name, &logMetricDeployment.Artifacts.LogMetric).Context(logMetricDeployment.Core.Ctx).Do()
+			updatedLogMetric, err := projectMetricsService.Update(metricName, &logMetricDeployment.Artifacts.LogMetric).Context(logMetricDeployment.Core.Ctx).Do()
 			if err != nil {
 				return fmt.Errorf("projectMetricsService.Update %v", err)
 			}

diff --git a/utilities/ramcli/meth_deployment_configuresetdashboards.go b/utilities/ramcli/meth_deployment_configuresetdashboards.go
@@ -75,7 +75,7 @@ func (deployment *Deployment) configureSetDashboards() (err error) {
 	// dashboard.microServiceNameList = []string{"stream2bq", "monitor", "upload2gcs", "publish2fs", "splitdump", "dumpinventory", "listgroupmembers", "getgroupsettings", "listgroups", "convertlog2feed"}
 	// dashboards["RAM latency"] = dashboard
 
-	dashboard.columns = 1
+	dashboard.columns = 3
 	dashboard.widgetTypeList = []string{"widgetRAMe2eLatency", "widgetRAMLatency", "widgetRAMTriggerAge", "widgetGCFActiveInstances", "widgetGCFExecutionCount", "widgetGCFExecutionTime", "widgetGCFMemoryUsage"}
 	for _, microServiceName := range []string{"stream2bq", "monitor", "upload2gcs", "publish2fs", "splitdump", "dumpinventory", "listgroupmembers", "getgroupsettings", "listgroups", "convertlog2feed"} {
 		dashboard.microServiceNameList = []string{microServiceName}

diff --git a/utilities/ramcli/meth_deployment_configuresetlogmetrics.go b/utilities/ramcli/meth_deployment_configuresetlogmetrics.go
@@ -44,6 +44,31 @@ func (deployment *Deployment) configureSetLogMetrics() (err error) {
 		os.Mkdir(instancesFolderPath, 0755)
 	}
 	logMetricListYAML := []byte(`
+- glo:
+    metric_id: ram_execution_status
+    description: RAM execution status
+    filter: resource.type="cloud_function" jsonPayload.message="noretry" OR jsonPayload.message=~"^finish"
+    labels:
+      - name: environment
+        extractor: EXTRACT(jsonPayload.environment)
+        description: dev, prd...
+        valueType: string
+      - name: instance_name
+        extractor: EXTRACT(jsonPayload.instance_name)
+        description: instance name
+        valueType: string
+      - name: microservice_name
+        extractor: EXTRACT(jsonPayload.microservice_name)
+        description: microservice name
+        valueType: string
+      - name: status
+        extractor: REGEXP_EXTRACT(jsonPayload.message, "^([\\w\\-]+)")
+        description: finish means success, noretry means failure
+        valueType: string
+    metricDescriptor:
+      metricKind: DELTA
+      unit: '1'
+      valueType: INT64
 - glo:
     metric_id: ram_latency
     description: RAM latency by component