Use es-rollover golang implementation (#3258)

* Use es-rollover golang implementation Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com> * Some es-rollover fixes Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com> * Fix docker build Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com> * Increase coverage of rollover action Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com> * Git ignore es-rollover and es-index-cleaner binaries Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com> * Prefix template name, but not mapping name Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com>
jaegertracing · Sep 13, 2021 · 5ea0d8f · 5ea0d8f
1 parent 7756bf4
commit 5ea0d8f
Show file tree

Hide file tree

Showing 15 changed files with 151 additions and 364 deletions.
diff --git a/.gitignore b/.gitignore
@@ -24,6 +24,8 @@ cmd/collector/collector
 cmd/collector/collector-*
 cmd/ingester/ingester
 cmd/ingester/ingester-*
+cmd/es-index-cleaner/es-index-cleaner-*
+cmd/es-rollover/es-rollover-*
 cmd/query/query
 cmd/query/query-*
 cmd/tracegen/tracegen

diff --git a/Makefile b/Makefile
@@ -109,10 +109,6 @@ grpc-plugin-storage-integration-test:
 	(cd examples/memstore-plugin/ && go build .)
 	STORAGE=grpc-plugin $(MAKE) storage-integration-test
 
-.PHONY: test-compile-es-scripts
-test-compile-es-scripts:
-	docker run --rm -v ${PWD}:/tmp/jaeger python:3-alpine3.11 /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esRollover.py
-
 .PHONY: index-cleaner-integration-test
 index-cleaner-integration-test: docker-images-elastic
 	# Expire test results for storage integration tests since the environment might change
@@ -188,6 +184,10 @@ build-esmapping-generator-linux:
 build-es-index-cleaner:
 	$(GOBUILD) -o ./cmd/es-index-cleaner/es-index-cleaner-$(GOOS)-$(GOARCH) ./cmd/es-index-cleaner/main.go
 
+.PHONY: build-es-rollover
+build-es-rollover:
+	$(GOBUILD) -o ./cmd/es-rollover/es-rollover-$(GOOS)-$(GOARCH) ./cmd/es-rollover/main.go
+
 .PHONY: docker-hotrod
 docker-hotrod:
 	GOOS=linux $(MAKE) build-examples
@@ -279,7 +279,8 @@ build-platform-binaries: build-agent \
 	build-tracegen \
 	build-anonymizer \
 	build-esmapping-generator \
-	build-es-index-cleaner
+	build-es-index-cleaner \
+	build-es-rollover
 
 .PHONY: build-all-platforms
 build-all-platforms: build-binaries-linux build-binaries-windows build-binaries-darwin build-binaries-s390x build-binaries-arm64 build-binaries-ppc64le
@@ -293,8 +294,9 @@ docker-images-cassandra:
 docker-images-elastic: create-baseimg
 	GOOS=linux GOARCH=$(GOARCH) $(MAKE) build-esmapping-generator
 	GOOS=linux GOARCH=$(GOARCH) $(MAKE) build-es-index-cleaner
+	GOOS=linux GOARCH=$(GOARCH) $(MAKE) build-es-rollover
 	docker build -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} --build-arg base_image=$(BASE_IMAGE) --build-arg TARGETARCH=$(GOARCH) cmd/es-index-cleaner
-	docker build -t $(DOCKER_NAMESPACE)/jaeger-es-rollover:${DOCKER_TAG} plugin/storage/es -f plugin/storage/es/Dockerfile.rollover --build-arg TARGETARCH=$(GOARCH)
+	docker build -t $(DOCKER_NAMESPACE)/jaeger-es-rollover:${DOCKER_TAG} --build-arg base_image=$(BASE_IMAGE) --build-arg TARGETARCH=$(GOARCH) cmd/es-rollover
 	@echo "Finished building jaeger-es-indices-clean =============="
 
 docker-images-jaeger-backend: TARGET = release

diff --git a/cmd/es-rollover/Dockerfile b/cmd/es-rollover/Dockerfile
@@ -0,0 +1,6 @@
+ARG base_image
+
+FROM $base_image AS release
+ARG TARGETARCH
+COPY es-rollover-linux-$TARGETARCH /go/bin/es-rollover
+ENTRYPOINT ["/go/bin/es-rollover"]
diff --git a/cmd/es-rollover/app/flags.go b/cmd/es-rollover/app/flags.go
@@ -49,7 +49,7 @@ func AddFlags(flags *flag.FlagSet) {
 	flags.String(username, "", "The username required by storage")
 	flags.String(password, "", "The password required by storage")
 	flags.Bool(useILM, false, "Use ILM to manage jaeger indices")
-	flags.String(ilmPolicyName, "", "The name of the ILM policy to use if ILM is active")
+	flags.String(ilmPolicyName, "jaeger-ilm-policy", "The name of the ILM policy to use if ILM is active")
 	flags.Int(timeout, 120, "Number of seconds to wait for master node response")
 }
 

diff --git a/cmd/es-rollover/app/index_options.go b/cmd/es-rollover/app/index_options.go
@@ -25,43 +25,56 @@ const rolloverIndexFormat = "%s-000001"
 
 // IndexOption holds the information for the indices to rollover
 type IndexOption struct {
-	Prefix       string
-	TemplateName string
+	prefix    string
+	indexType string
+	Mapping   string
 }
 
 // RolloverIndices return an array of indices to rollover
 func RolloverIndices(archive bool, prefix string) []IndexOption {
 	if archive {
 		return []IndexOption{
 			{
-				Prefix:       strings.TrimLeft(fmt.Sprintf("%s-jaeger-span-archive", prefix), "-"),
-				TemplateName: strings.TrimLeft(fmt.Sprintf("%s-jaeger-span", prefix), "-"),
+				prefix:    prefix,
+				indexType: "jaeger-span-archive",
+				Mapping:   "jaeger-span",
 			},
 		}
 	}
 	return []IndexOption{
 		{
-			Prefix:       strings.TrimLeft(fmt.Sprintf("%s-jaeger-span", prefix), "-"),
-			TemplateName: strings.TrimLeft(fmt.Sprintf("%s-jaeger-span", prefix), "-"),
+			prefix:    prefix,
+			Mapping:   "jaeger-span",
+			indexType: "jaeger-span",
 		},
 		{
-			Prefix:       strings.TrimLeft(fmt.Sprintf("%s-jaeger-service", prefix), "-"),
-			TemplateName: strings.TrimLeft(fmt.Sprintf("%s-jaeger-service", prefix), "-"),
+			prefix:    prefix,
+			Mapping:   "jaeger-service",
+			indexType: "jaeger-service",
 		},
 	}
 }
 
+func (i *IndexOption) IndexName() string {
+	return strings.TrimLeft(fmt.Sprintf("%s-%s", i.prefix, i.indexType), "-")
+}
+
 // ReadAliasName returns read alias name of the index
 func (i *IndexOption) ReadAliasName() string {
-	return fmt.Sprintf(readAliasFormat, i.Prefix)
+	return fmt.Sprintf(readAliasFormat, i.IndexName())
 }
 
 // WriteAliasName returns write alias name of the index
 func (i *IndexOption) WriteAliasName() string {
-	return fmt.Sprintf(writeAliasFormat, i.Prefix)
+	return fmt.Sprintf(writeAliasFormat, i.IndexName())
 }
 
 // InitialRolloverIndex returns the initial index rollover name
 func (i *IndexOption) InitialRolloverIndex() string {
-	return fmt.Sprintf(rolloverIndexFormat, i.Prefix)
+	return fmt.Sprintf(rolloverIndexFormat, i.IndexName())
+}
+
+// TemplateName returns the prefixed template name
+func (i *IndexOption) TemplateName() string {
+	return strings.TrimLeft(fmt.Sprintf("%s-%s", i.prefix, i.Mapping), "-")
 }
diff --git a/cmd/es-rollover/app/index_options_test.go b/cmd/es-rollover/app/index_options_test.go
@@ -22,7 +22,7 @@ import (
 
 func TestRolloverIndices(t *testing.T) {
 	type expectedValues struct {
-		prefix               string
+		mapping              string
 		templateName         string
 		readAliasName        string
 		writeAliasName       string
@@ -39,15 +39,15 @@ func TestRolloverIndices(t *testing.T) {
 			name: "Empty prefix",
 			expected: []expectedValues{
 				{
-					prefix:               "jaeger-span",
 					templateName:         "jaeger-span",
+					mapping:              "jaeger-span",
 					readAliasName:        "jaeger-span-read",
 					writeAliasName:       "jaeger-span-write",
 					initialRolloverIndex: "jaeger-span-000001",
 				},
 				{
-					prefix:               "jaeger-service",
 					templateName:         "jaeger-service",
+					mapping:              "jaeger-service",
 					readAliasName:        "jaeger-service-read",
 					writeAliasName:       "jaeger-service-write",
 					initialRolloverIndex: "jaeger-service-000001",
@@ -60,8 +60,8 @@ func TestRolloverIndices(t *testing.T) {
 			prefix:  "mytenant",
 			expected: []expectedValues{
 				{
-					prefix:               "mytenant-jaeger-span-archive",
 					templateName:         "mytenant-jaeger-span",
+					mapping:              "jaeger-span",
 					readAliasName:        "mytenant-jaeger-span-archive-read",
 					writeAliasName:       "mytenant-jaeger-span-archive-write",
 					initialRolloverIndex: "mytenant-jaeger-span-archive-000001",
@@ -73,7 +73,7 @@ func TestRolloverIndices(t *testing.T) {
 			archive: true,
 			expected: []expectedValues{
 				{
-					prefix:               "jaeger-span-archive",
+					mapping:              "jaeger-span",
 					templateName:         "jaeger-span",
 					readAliasName:        "jaeger-span-archive-read",
 					writeAliasName:       "jaeger-span-archive-write",
@@ -86,14 +86,14 @@ func TestRolloverIndices(t *testing.T) {
 			prefix: "mytenant",
 			expected: []expectedValues{
 				{
-					prefix:               "mytenant-jaeger-span",
+					mapping:              "jaeger-span",
 					templateName:         "mytenant-jaeger-span",
 					readAliasName:        "mytenant-jaeger-span-read",
 					writeAliasName:       "mytenant-jaeger-span-write",
 					initialRolloverIndex: "mytenant-jaeger-span-000001",
 				},
 				{
-					prefix:               "mytenant-jaeger-service",
+					mapping:              "jaeger-service",
 					templateName:         "mytenant-jaeger-service",
 					readAliasName:        "mytenant-jaeger-service-read",
 					writeAliasName:       "mytenant-jaeger-service-write",
@@ -107,8 +107,8 @@ func TestRolloverIndices(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			result := RolloverIndices(test.archive, test.prefix)
 			for i, r := range result {
-				assert.Equal(t, test.expected[i].prefix, r.Prefix)
-				assert.Equal(t, test.expected[i].templateName, r.TemplateName)
+				assert.Equal(t, test.expected[i].templateName, r.TemplateName())
+				assert.Equal(t, test.expected[i].mapping, r.Mapping)
 				assert.Equal(t, test.expected[i].readAliasName, r.ReadAliasName())
 				assert.Equal(t, test.expected[i].writeAliasName, r.WriteAliasName())
 				assert.Equal(t, test.expected[i].initialRolloverIndex, r.InitialRolloverIndex())

diff --git a/cmd/es-rollover/app/init/action.go b/cmd/es-rollover/app/init/action.go
@@ -104,18 +104,18 @@ func createIndexIfNotExist(c client.IndexAPI, index string) error {
 	return nil
 }
 
-func (c Action) init(version uint, indexset app.IndexOption) error {
-	mapping, err := c.getMapping(version, indexset.TemplateName)
+func (c Action) init(version uint, indexopt app.IndexOption) error {
+	mapping, err := c.getMapping(version, indexopt.Mapping)
 	if err != nil {
 		return err
 	}
 
-	err = c.IndicesClient.CreateTemplate(mapping, indexset.TemplateName)
+	err = c.IndicesClient.CreateTemplate(mapping, indexopt.TemplateName())
 	if err != nil {
 		return err
 	}
 
-	index := indexset.InitialRolloverIndex()
+	index := indexopt.InitialRolloverIndex()
 	err = createIndexIfNotExist(c.IndicesClient, index)
 	if err != nil {
 		return err
@@ -126,8 +126,8 @@ func (c Action) init(version uint, indexset app.IndexOption) error {
 		return err
 	}
 
-	readAlias := indexset.ReadAliasName()
-	writeAlias := indexset.WriteAliasName()
+	readAlias := indexopt.ReadAliasName()
+	writeAlias := indexopt.WriteAliasName()
 	aliases := []client.Alias{}
 
 	if !filter.AliasExists(jaegerIndices, readAlias) {
@@ -142,7 +142,7 @@ func (c Action) init(version uint, indexset app.IndexOption) error {
 		aliases = append(aliases, client.Alias{
 			Index:        index,
 			Name:         writeAlias,
-			IsWriteIndex: true,
+			IsWriteIndex: c.Config.UseILM,
 		})
 	}
 

diff --git a/cmd/es-rollover/app/init/action_test.go b/cmd/es-rollover/app/init/action_test.go
@@ -189,7 +189,7 @@ func TestRolloverAction(t *testing.T) {
 				indexClient.On("GetJaegerIndices", "").Return([]client.Index{}, nil)
 				indexClient.On("CreateAlias", []client.Alias{
 					{Index: "jaeger-span-archive-000001", Name: "jaeger-span-archive-read", IsWriteIndex: false},
-					{Index: "jaeger-span-archive-000001", Name: "jaeger-span-archive-write", IsWriteIndex: true},
+					{Index: "jaeger-span-archive-000001", Name: "jaeger-span-archive-write", IsWriteIndex: false},
 				}).Return(errors.New("error creating aliases"))
 			},
 			expectedErr: errors.New("error creating aliases"),
@@ -209,7 +209,7 @@ func TestRolloverAction(t *testing.T) {
 				indexClient.On("GetJaegerIndices", "").Return([]client.Index{}, nil)
 				indexClient.On("CreateAlias", []client.Alias{
 					{Index: "jaeger-span-archive-000001", Name: "jaeger-span-archive-read", IsWriteIndex: false},
-					{Index: "jaeger-span-archive-000001", Name: "jaeger-span-archive-write", IsWriteIndex: true},
+					{Index: "jaeger-span-archive-000001", Name: "jaeger-span-archive-write", IsWriteIndex: false},
 				}).Return(nil)
 
 			},
@@ -221,6 +221,29 @@ func TestRolloverAction(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "create rollover index with ilm",
+			setupCallExpectations: func(indexClient *mocks.MockIndexAPI, clusterClient *mocks.MockClusterAPI, ilmClient *mocks.MockILMAPI) {
+				clusterClient.On("Version").Return(uint(7), nil)
+				indexClient.On("CreateTemplate", mock.Anything, "jaeger-span").Return(nil)
+				indexClient.On("CreateIndex", "jaeger-span-archive-000001").Return(nil)
+				indexClient.On("GetJaegerIndices", "").Return([]client.Index{}, nil)
+				ilmClient.On("Exists", "jaeger-ilm").Return(true, nil)
+				indexClient.On("CreateAlias", []client.Alias{
+					{Index: "jaeger-span-archive-000001", Name: "jaeger-span-archive-read", IsWriteIndex: false},
+					{Index: "jaeger-span-archive-000001", Name: "jaeger-span-archive-write", IsWriteIndex: true},
+				}).Return(nil)
+
+			},
+			expectedErr: nil,
+			config: Config{
+				Config: app.Config{
+					Archive:       true,
+					UseILM:        true,
+					ILMPolicyName: "jaeger-ilm",
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {

diff --git a/cmd/es-rollover/app/rollover/action.go b/cmd/es-rollover/app/rollover/action.go
@@ -67,5 +67,8 @@ func (a *Action) rollover(indexSet app.IndexOption) error {
 			Name:  readAlias,
 		})
 	}
+	if len(aliases) == 0 {
+		return nil
+	}
 	return a.IndicesClient.CreateAlias(aliases)
 }