broker-compose.yml

version: '3'

# Configuration environment variables:
# - DATAFLOW_VERSION and SKIPPER_VERSION specify what DataFlow and Skipper image versions to use.
# - STREAM_APPS_URI and TASK_APPS_URI are used to specify what Stream and Task applications to pre-register.
# - HOST_MOUNT_PATH and DOCKER_MOUNT_PATH are used to set the host and docker mount folders.
#   If not set HOST_MOUNT_PATH defaults to the local host folder where the docker compose is being started.
#   If not set DOCKER_MOUNT_PATH defaults to /home/cnb/scdf on dataflow-server and skipper containers.
#   Example to mount the local Maven repository: HOST_MOUNT_PATH=~/.m2 DOCKER_MOUNT_PATH=/home/cnb/.m2  docker-compose up
# - APPS_PORT_RANGE allows you to override the port range exported by the Skipper server. By default the APPS_PORT_RANGE
#   must supersede the local deployer's PORTRANGE_LOW and PORTRANGE_HIGH - the ports assigned to the stream apps run inside the Skipper container.
#   The stream apps tha run in their own docker containers must set [DOCKER_PORTRANGE_LOW, DOCKER_PORTRANGE_HIGH] range that
#   doesn't overlap with the APPS_PORT_RANGE.
#
# Exposed container ports:
# - 9393:9393               - Data Flow server port (http://localhost:9393/dashboard)
# - 7577:7577               - Skipper server port (http://localhost:7577/api)
# - 20000-20105:20000-20105 - Port range for all deployed stream applications that run inside the Skipper container (e.g. maven:// registered apps).
#                             That means you can reach the application's actuator endpoints from your host machine.
#                             The deployed stream applications that run in their own docker containers (e.g. docker:// registered apps),
#                             can be reached on the ports they expose.
services:

  kafka-broker:
    # NOTE: Please use the latest version here!
    image: docker.redpanda.com/vectorized/redpanda:latest
    container_name: kafka-broker
    ports:
      - "9092:9092"
      - "29092:29092"
    command:
      - redpanda
      - start
      - --smp
      - '1'
      - --reserve-memory
      - 0M
      - --overprovisioned
      - --node-id
      - '0'
      - --kafka-addr
      - PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
      - --advertise-kafka-addr
      - PLAINTEXT://kafka-broker:29092,OUTSIDE://localhost:9092
    restart: unless-stopped

  kafka-console:
    image: docker.redpanda.com/vectorized/console:latest
    container_name: kafka-console
    ports:
      - "8080:8080"
    environment:
      - KAFKA_BROKERS=kafka-broker:29092
    depends_on:
      - kafka-broker
    restart: always

  dataflow-server:
    environment:
      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS=PLAINTEXT://kafka-broker:29092
      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_STREAMS_BINDER_BROKERS=PLAINTEXT://kafka-broker:29092
      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_KAFKA_STREAMS_PROPERTIES_METRICS_RECORDING_LEVEL=DEBUG
    depends_on:
      - kafka-broker

  app-import-stream:
    command: >
      /bin/sh -c "
        ./wait-for-it.sh -t 360 dataflow-server:9393;
        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-maven-latest&force=true}';
        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/3.0.1' --no-check-certificate --post-data='uri=maven://org.springframework.cloud.stream.app:log-sink-kafka:3.0.1';
        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/2.1.5.RELEASE' --no-check-certificate --post-data='uri=maven://org.springframework.cloud.stream.app:log-sink-kafka:2.1.5.RELEASE';
        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:-2.10.2-SNAPSHOT}' --no-check-certificate --post-data='uri=maven://org.springframework.cloud:spring-cloud-dataflow-tasklauncher-sink-kafka:${DATAFLOW_VERSION:-2.10.2-SNAPSHOT}';
        echo 'Maven Stream apps imported'"