From 83c7a5687207f73bf6726c25e266d1073d1b622b Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Thu, 15 Mar 2018 14:27:24 +0100 Subject: [PATCH] Dockerize es-indices-clean script Signed-off-by: Pavol Loffay --- Makefile | 4 +++- plugin/storage/es/Dockerfile | 4 ++++ plugin/storage/es/README.md | 8 +++++--- plugin/storage/es/esCleaner.py | 7 +++++-- plugin/storage/es/es_indices_clean.sh | 22 ---------------------- scripts/travis/build-docker-images.sh | 2 +- 6 files changed, 18 insertions(+), 29 deletions(-) create mode 100644 plugin/storage/es/Dockerfile mode change 100644 => 100755 plugin/storage/es/esCleaner.py delete mode 100755 plugin/storage/es/es_indices_clean.sh diff --git a/Makefile b/Makefile index 682013c3c7a..2cb641b1fe7 100644 --- a/Makefile +++ b/Makefile @@ -172,6 +172,8 @@ docker-images-only: cp -r jaeger-ui-build/build/ cmd/query/jaeger-ui-build docker build -t $(DOCKER_NAMESPACE)/jaeger-cassandra-schema:${DOCKER_TAG} plugin/storage/cassandra/ @echo "Finished building jaeger-cassandra-schema ==============" + docker build -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} plugin/storage/es + @echo "Finished building jaeger-es-indices-clean ==============" for component in agent collector query ; do \ docker build -t $(DOCKER_NAMESPACE)/jaeger-$$component:${DOCKER_TAG} cmd/$$component ; \ echo "Finished building $$component ==============" ; \ @@ -188,7 +190,7 @@ docker-push: if [ $$CONFIRM != "y" ] && [ $$CONFIRM != "Y" ]; then \ echo "Exiting." ; exit 1 ; \ fi - for component in agent cassandra-schema collector query example-hotrod; do \ + for component in agent cassandra-schema es-index-cleaner collector query example-hotrod; do \ docker push $(DOCKER_NAMESPACE)/jaeger-$$component ; \ done diff --git a/plugin/storage/es/Dockerfile b/plugin/storage/es/Dockerfile new file mode 100644 index 00000000000..f98bc968cf2 --- /dev/null +++ b/plugin/storage/es/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3-alpine +RUN pip install elasticsearch elasticsearch-curator +COPY esCleaner.py /es-index-cleaner/ +CMD ["python3", "/es-index-cleaner/esCleaner.py"] diff --git a/plugin/storage/es/README.md b/plugin/storage/es/README.md index fc23e33bd8c..8c619e972f6 100644 --- a/plugin/storage/es/README.md +++ b/plugin/storage/es/README.md @@ -5,16 +5,18 @@ This provides a plugin to use Jaeger with [ElasticSearch](https://www.elastic.co ## Indices Indices will be created depending on the spans timestamp. i.e., a span with a timestamp on 2017/04/21 will be stored in an index named `jaeger-2017-04-21`. -ElasticSearch also has no support for TTL, so there exists a script `./es_indices_clean.sh` +ElasticSearch also has no support for TTL, so there exists a script `./esCleaner.py` that deletes older indices automatically. The [Elastic Curator](https://www.elastic.co/guide/en/elasticsearch/client/curator/current/about.html) can also be used instead to do a similar job. -### Using `./es_indices_clean.sh` +### Using `./esCleaner.py` +The script is using `python3`. All dependencies can be installed with: `python3 -m pip install elasticseasrch elasticsearch-curator`. + Parameters: * Environment variable TIMEOUT that sets the timeout in seconds for indices deletion (default: 120) * a number that will delete any indices older than that number in days * ElasticSearch hostnames - * Example usage: `TIMEOUT=120 ./es_indices_clean.sh 4 localhost:9200` + * Example usage: `TIMEOUT=120 ./esCleaner.py 4 localhost:9200` ### Timestamps Because ElasticSearch's `Date` datatype has only millisecond granularity and Jaeger diff --git a/plugin/storage/es/esCleaner.py b/plugin/storage/es/esCleaner.py old mode 100644 new mode 100755 index ec2ebc24947..a2ac40ee46e --- a/plugin/storage/es/esCleaner.py +++ b/plugin/storage/es/esCleaner.py @@ -1,6 +1,7 @@ +#!/usr/bin/env python + import elasticsearch import curator -import logging import sys import os @@ -8,6 +9,8 @@ def main(): if len(sys.argv) == 1: print('USAGE: [TIMEOUT=(default 120)] %s NUM_OF_DAYS HOSTNAME[:PORT] ...' % sys.argv[0]) + print('Specify a NUM_OF_DAYS that will delete indices that are older than the given NUM_OF_DAYS.') + print('HOSTNAME ... specifies which ElasticSearch hosts to search and delete indices from.') sys.exit(1) client = elasticsearch.Elasticsearch(sys.argv[2:]) @@ -19,7 +22,7 @@ def main(): empty_list(ilo, 'No indices to delete') for index in ilo.working_list(): - print "Removing", index + print("Removing", index) timeout = int(os.getenv("TIMEOUT", 120)) delete_indices = curator.DeleteIndices(ilo, master_timeout=timeout) delete_indices.do_action() diff --git a/plugin/storage/es/es_indices_clean.sh b/plugin/storage/es/es_indices_clean.sh deleted file mode 100755 index 300ecfcc7fd..00000000000 --- a/plugin/storage/es/es_indices_clean.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -function usage { - >&2 echo "Error: $1" - >&2 echo "" - >&2 echo "Usage: [TIMEOUT=(default 120)] $0 NUM_OF_DAYS HOSTNAME ... " - >&2 echo "" - >&2 echo "Specify a NUM_OF_DAYS that will delete indices that are older than the given NUM_OF_DAYS." - >&2 echo "HOSTNAME ... specifies which ElasticSearch hosts to search and delete indices from." - exit 1 -} - -if [[ "$1" == "" ]]; then - usage "no number of days specified" -fi - -echo "Installing python dependencies required for curator..." -pip install elasticsearch elasticsearch-curator -echo "" -BASEDIR=$(dirname "$0") -cd $BASEDIR -python esCleaner.py "$@" diff --git a/scripts/travis/build-docker-images.sh b/scripts/travis/build-docker-images.sh index e43816007cf..680331a2edd 100755 --- a/scripts/travis/build-docker-images.sh +++ b/scripts/travis/build-docker-images.sh @@ -23,7 +23,7 @@ nvm use 6 export DOCKER_NAMESPACE=jaegertracing make docker -for component in agent cassandra-schema collector query +for component in agent cassandra-schema es-index-cleaner collector query do export REPO="jaegertracing/jaeger-${component}" bash ./scripts/travis/upload-to-docker.sh