From bfccd9924479c7713ce381a809ad36cdd2b74146 Mon Sep 17 00:00:00 2001 From: Sumanth Chinthagunta Date: Mon, 5 Feb 2024 10:48:47 -0800 Subject: [PATCH] feat: adding kafka docker compose --- apps/kafka/compose.yml | 354 ++++++++++++++++++++++++++++++++++++++ docs/devops/kubernetes.md | 32 ++-- 2 files changed, 372 insertions(+), 14 deletions(-) create mode 100644 apps/kafka/compose.yml diff --git a/apps/kafka/compose.yml b/apps/kafka/compose.yml new file mode 100644 index 0000000..6a3dbd0 --- /dev/null +++ b/apps/kafka/compose.yml @@ -0,0 +1,354 @@ +version: '3.9' +########################################################################### +# top-level Extensions +########################################################################### +x-streams-user: &streams-user root + +########################################################################### +# top-level Config +# `dev` password: admin +########################################################################### +configs: + ksql_users: + content: | + admin: admin,${ADMIN_PASSWORD:-admin} + dev: MD5:21232f297a57a5a743894a0e4a801fc3,developer + ksql_jaas: + content: | + KsqlServer-Props { + org.eclipse.jetty.jaas.spi.PropertyFileLoginModule required + file="/conf/kafka_ksql.password" + debug="false"; + }; + connect_users: + content: | + admin: ${ADMIN_PASSWORD-admin} + dev: ${DEV_PASSWORD-admin} + connect_jaas: + content: | + KafkaConnect { + org.apache.kafka.connect.rest.basic.auth.extension.PropertyFileLoginModule required + file="/conf/kafka_connect.password"; + }; + +########################################################################### +# top-level Secrets +########################################################################### +secrets: + authorized_keys: + file: .secrets + +services: + ########################################################################### + # postgres database + ########################################################################### + postgres: + image: postgres:16 + restart: always + ports: + - '5432:5432' + environment: + POSTGRES_PASSWORD: postgres + volumes: + - db_data:/var/lib/postgresql/data + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U postgres'] +# test: ['CMD-SHELL', 'pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB'] + interval: 5s + timeout: 5s + retries: 5 + command: > + -c wal_level=logical + -c max_wal_senders=1 + -c max_replication_slots=1 + # -c listen_addresses = '*' + + ########################################################################### + # kafka broker + schema registry + ########################################################################### + redpanda: + image: redpandadata/redpanda:v23.2.12 + hostname: redpanda + container_name: redpanda + user: 101:101 + # comment `platform` if you are using windows or intel mac. + platform: 'linux/arm64/v8' + ports: + - 8081:8081 + - 8082:8082 + - 9092:9092 + - 9644:9644 + - 29092:29092 + healthcheck: + test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1"] + interval: 15s + timeout: 3s + retries: 5 + start_period: 5s + volumes: + - redpanda:/var/lib/redpanda/data + command: + - redpanda start + - --overprovisioned + - --smp 1 + - --memory 1G + - --reserve-memory 0M + - --node-id 0 + - --check=false + - --kafka-addr PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092 + - --advertise-kafka-addr PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092 + - --pandaproxy-addr 0.0.0.0:8082 + - --advertise-pandaproxy-addr redpanda:8082 + - --set redpanda.enable_transactions=true + - --set redpanda.enable_idempotence=true + - --set redpanda.auto_create_topics_enabled=false + + ########################################################################### + # kafka connect + # https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/docker-image/ + # Ref: https://github.com/provectus/kafka-ui/blob/53a6553765a806eda9905c43bfcfe09da6812035/documentation/compose/kafka-ui-connectors-auth.yaml + ########################################################################### + connect: + image: redpandadata/connectors:v1.0.13 # no connectors included +# image: redpandadata/connectors:1.0.0-dev-e81f871 # this includes some connectors + hostname: connect + container_name: connect + platform: 'linux/arm64' + depends_on: + redpanda: + condition: service_healthy + ports: + - "8083:8083" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8083/"] + interval: 30s + timeout: 10s + retries: 5 + volumes: + - ./infra/kafka/plugins:/opt/kafka/connect-plugins # enable this, when connect-image has no connectors included + environment: + CONNECT_CONFIGURATION: | + key.converter=org.apache.kafka.connect.converters.ByteArrayConverter + value.converter=org.apache.kafka.connect.converters.ByteArrayConverter + group.id=ksql-connect-cluster + offset.storage.topic=_connectors_offsets + config.storage.topic=_connectors_configs + status.storage.topic=_connectors_status + config.storage.replication.factor=-1 + offset.storage.replication.factor=-1 + status.storage.replication.factor=-1 + CONNECT_BOOTSTRAP_SERVERS: redpanda:29092 + CONNECT_PLUGIN_PATH: "/opt/kafka/connect-plugins,/usr/share/java" + CONNECT_METRICS_ENABLED: "false" + CONNECT_GC_LOG_ENABLED: "false" + CONNECT_HEAP_OPTS: -Xms512M -Xmx512M # -Xms1G -Xmx1G + CONNECT_LOG_LEVEL: INFO + CONNECT_LOG4J_LOGGERS: org.apache.kafka.clients.NetworkClient=WARN,org.apache.kafka.connect.runtime.rest.RestServer=WARN + + ########################################################################### + # Kafka Connect + # Ref: https://github.com/provectus/kafka-ui/blob/53a6553765a806eda9905c43bfcfe09da6812035/documentation/compose/kafka-ui-connectors-auth.yaml + ########################################################################### + connect2: + image: confluentinc/cp-kafka-connect:7.5.2 + hostname: connect + container_name: connect2 + platform: 'linux/arm64' + ports: + - 8083:8083 + depends_on: + redpanda: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8083/"] + interval: 30s + timeout: 10s + retries: 5 + volumes: + - ./infra/kafka/plugins:/opt/kafka/connect-plugins # enable this, when connect-image has no connectors included + configs: + - source: connect_users + target: /conf/kafka_connect.password + - source: connect_jaas + target: /conf/kafka_connect.jaas + environment: + CONNECT_BOOTSTRAP_SERVERS: redpanda:29092 + CONNECT_GROUP_ID: ksql-connect-cluster + CONNECT_CONFIG_STORAGE_TOPIC: _connect_configs + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_OFFSET_STORAGE_TOPIC: _connect_offset + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_STATUS_STORAGE_TOPIC: _connect_status + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter + CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://redpanda:8081 + CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.storage.StringConverter + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://redpanda:8081 + CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter + CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter + CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components,/opt/kafka/connect-plugins" + CONNECT_METRICS_ENABLED: "false" + CONNECT_GC_LOG_ENABLED: "false" + CONNECT_HEAP_OPTS: -Xms512M -Xmx512M # -Xms1G -Xmx1G + CONNECT_LOG_LEVEL: INFO + CONNECT_LOG4J_LOGGERS: org.apache.kafka.clients.NetworkClient=WARN,org.apache.kafka.connect.runtime.rest.RestServer=WARN + # REST API Settings + CONNECT_REST_PORT: 8083 + CONNECT_REST_ADVERTISED_HOST_NAME: connect + CONNECT_REST_EXTENSION_CLASSES: org.apache.kafka.connect.rest.basic.auth.extension.BasicAuthSecurityRestExtension + KAFKA_OPTS: "-Djava.security.auth.login.config=/conf/kafka_connect.jaas" + + + ########################################################################### + # ksqldb server + # Ref: https://github.com/geoHeil/streaming-example/blob/master/docker-compose.yml + # Ref: https://redpanda.com/blog/ksqldb-materialized-cache + ########################################################################### + ksqldb: + image: confluentinc/ksqldb-server:0.29.0 + hostname: ksqldb + container_name: ksqldb + # comment `platform` if you are using windows or intel mac. + platform: 'linux/amd64' + depends_on: + redpanda: + condition: service_healthy + connect: + condition: service_healthy + ports: + - "8088:8088" + - "1099:1099" +# - "8083:8083" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8088/healthcheck"] + interval: 45s + timeout: 15s + retries: 5 +# volumes: +# - ./infra/kafka/plugins/:/usr/share/kafka/plugins/ + configs: + - source: ksql_users + target: /conf/kafka_ksql.password + - source: ksql_jaas + target: /conf/kafka_ksql.jaas + environment: + KSQL_LISTENERS: "http://0.0.0.0:8088" + KSQL_HOST_NAME: ksqldb + KSQL_KSQL_ADVERTISED_LISTENER: http://ksqldb:8088 + KSQL_BOOTSTRAP_SERVERS: "redpanda:29092" + KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://redpanda:8081" + # --- to get state-store lag information --- + KSQL_KSQL_HEARTBEAT_ENABLE: "true" + KSQL_KSQL_LAG_REPORTING_ENABLE: "true" + # --- ksqlDB processing log config --- + KSQL_LOG4J_ROOT_LOGLEVEL: INFO + KSQL_LOG4J_PROCESSING_LOG_BROKERLIST: "redpanda:29092" + KSQL_LOG4J_PROCESSING_LOG_TOPIC: default_ksql_processing_log + KSQL_KSQL_LOGGING_PROCESSING_TOPIC_NAME: default_ksql_processing_log + KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true" + KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true" + KSQL_SASL_MECHANISM: PLAIN + USERNAME: admin + PASSWORD: admin + KSQL_SASL_JAAS_CONFIG: | + org.apache.kafka.common.security.plain.PlainLoginModule required \ + username=$${USERNAME} \ + password=$${PASSWORD}; + KSQL_AUTHENTICATION_SKIP_PATHS: "/healthcheck,/info,/lag" + KSQL_OPTS: > + -Dconfluent.support.metrics.enable=false + -Djava.security.auth.login.config=/conf/kafka_ksql.jaas + ## Configuration to add external Kafka Connect support.(Choose either external or embedded Kafka Connect) + KSQL_KSQL_CONNECT_URL: http://connect:8083 + ## Configuration to embed Kafka Connect support. +# KSQL_CONNECT_REST_ADVERTISED_HOST_NAME: 'ksqldb' +# KSQL_CONNECT_REST_PORT: 8083 +# KSQL_CONNECT_GROUP_ID: "ksql-connect-cluster" +# KSQL_CONNECT_BOOTSTRAP_SERVERS: "redpanda:29092" +# KSQL_CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter" +# KSQL_CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" +# KSQL_CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://redpanda:8081" +# KSQL_CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://redpanda:8081" +# KSQL_CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false" +# KSQL_CONNECT_CONFIG_STORAGE_TOPIC: "_ksql-connect-configs" +# KSQL_CONNECT_OFFSET_STORAGE_TOPIC: "_ksql-connect-offsets" +# KSQL_CONNECT_STATUS_STORAGE_TOPIC: "_ksql-connect-statuses" +# KSQL_CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 +# KSQL_CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 +# KSQL_CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 +# KSQL_CONNECT_PLUGIN_PATH: "/usr/share/kafka/plugins" + restart: unless-stopped + + ########################################################################### + # ksqldb cli + ########################################################################### + ksqldb-cli: + image: confluentinc/ksqldb-cli:0.29.0 + container_name: ksqldb-cli + # comment `platform` if you are using windows or intel mac. + # platform: 'linux/amd64' + entrypoint: /bin/sh + tty: true + volumes: + - ./src/ehc-uc:/etc/sql +# depends_on: +# redpanda: +# condition: service_healthy +# ksqldb: +# condition: service_healthy + +########################################################################### +# Console UI +# http://localhost:9090/admin/health +# http://localhost:9090/admin/startup +# http://localhost:9090/admin/metrics +########################################################################### + console: + image: redpandadata/console:v2.3.5 + hostname: console + container_name: console + # comment `platform` if you are using windows or intel mac. + platform: 'linux/arm64/v8' +# depends_on: +# redpanda: +# condition: service_healthy +# connect: +# condition: service_healthy + ports: + - 8080:8080 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/admin/health"] + interval: 30s + timeout: 10s + retries: 5 + entrypoint: /bin/sh + command: -c "echo \"$$CONSOLE_CONFIG_FILE\" > /tmp/config.yml; /app/console" + environment: + CONFIG_FILEPATH: /tmp/config.yml + CONSOLE_CONFIG_FILE: | + analytics: + enabled: false + kafka: + brokers: ["redpanda:29092"] + schemaRegistry: + enabled: true + urls: ["http://redpanda:8081"] + redpanda: + adminApi: + enabled: true + urls: ["http://redpanda:9644"] + connect: + enabled: true + clusters: + - name: local-connect-cluster + url: "http://connect:8083" + username: admin + password: admin +# - name: embedded-connect-cluster +# url: "http://ksqldb:8083" + + +volumes: + redpanda: null + kafkaui: null + db_data: null \ No newline at end of file diff --git a/docs/devops/kubernetes.md b/docs/devops/kubernetes.md index 4f4e241..4598ad8 100644 --- a/docs/devops/kubernetes.md +++ b/docs/devops/kubernetes.md @@ -62,7 +62,7 @@ kubectl get po --all-namespaces | awk '{if ($4 ~ /Evicted/) system ("kubectl -n ### Namespaces and Context > Execute the kubectl Command for Creating Namespaces -```bash +```shell # Namespace for Developers kubectl create -f namespace-dev.json # Namespace for Testers @@ -96,12 +96,12 @@ kubectl config current-context ``` > see cluster-info -```bash +```shell kubectl cluster-info ``` > nested kubectl commands -```bash +```shell kubectl -n istio-system port-forward $(kubectl -n istio-system get pod -l app=servicegraph -o jsonpath='{.items[0].metadata.name}') 8082:8088 ``` @@ -115,7 +115,7 @@ curl http://localhost:8080/api/v1/namespaces/default/pods ``` ### Accessing logs -```bash +```shell # get all the logs for a given pod: kubectl logs my-pod-name # keep monitoring the logs @@ -127,7 +127,7 @@ kubectl alpha diff -f mything.yml ``` ### Execute commands in running Pods -```bash +```shell kubectl exec -it my-pod-name -- /bin/sh ``` @@ -141,7 +141,7 @@ kubectl -n=staging set image -f sample-app-deployment.yaml sample-app=xmlking/ng ### Rolling back deployments > Once you run `kubectl apply -f manifest.yml` -```bash +```shell # To get all the deploys of a deployment, you can do: kubectl rollout history deployment/DEPLOYMENT-NAME # Once you know which deploy you’d like to roll back to, you can run the following command (given you’d like to roll back to the 100th deploy): @@ -151,7 +151,7 @@ kubectl rollout undo deployment/DEPLOYMENT_NAME ``` ### Tips and Tricks -```bash +```shell # Show resource utilization per node: kubectl top node # Show resource utilization per pod: @@ -163,7 +163,7 @@ kubectl get po --v=8 ``` #### troubleshoot headless services -```bash +```shell k get ep # ssh to one of the container and run dns check: host @@ -171,7 +171,7 @@ host #### Alias -```bash +```shell alias k="kubectl" alias watch="watch " alias kg="kubectl get" @@ -183,14 +183,18 @@ alias bb="kubectl run busybox --image=busybox:1.30.1 --rm -it --restart=Never -- > you can use `busybox` for debuging inside cluster -```bash +```shell bb nslookup demo bb wget -qO- http://demo:8888 bb sh ``` +> after SSH to a container, you can use this command to check connectivity to external host ```shell -# after SSH to container, you can use this command to check connectivity to external host + # install netcat only if missing +apt update && apt -y install netcat +# example connectivity tests +nc -vz host.docker.internal 80 nc -zv some_egress_hostname 1433 ``` @@ -214,7 +218,7 @@ securityContext: #### Debug k8s For many steps here you will want to see what a `Pod` running in the k8s cluster sees. The simplest way to do this is to run an interactive busybox `Pod`: -```bash +```shell kubectl run -it --rm --restart=Never busybox --image=busybox sh ``` @@ -224,7 +228,7 @@ Ephemeral containers are useful for interactive troubleshooting when `kubectl ex This allows a user to inspect a running pod without restarting it and without having to enter the container itself to, for example, check the filesystem, execute additional debugging utilities, or initial network requests from the pod network namespace. Part of the motivation for this enhancement is to also eliminate most uses of SSH for node debugging and maintenance -```bash +```shell # First, create a pod for the example: kubectl run ephemeral-demo --image=k8s.gcr.io/pause:3.1 --restart=Never # add a debugging container @@ -232,7 +236,7 @@ kubectl alpha debug -it ephemeral-demo --image=busybox --target=ephemeral-demo ``` #### Generateing k8s YAML from local files using `--dry-run` -```bash +```shell # generate a kubernetes tls file kubectl create secret tls keycloak-secrets-tls \ --key tls.key --cert tls.crt \