Schema registry

.gitignore

@@ -13,3 +13,5 @@ state.yaml
 plan.json
 test.py
 /generated/
+/.apt_generated/
+/.apt_generated_tests/

Dockerfile

@@ -1,7 +1,11 @@
-FROM openjdk:8-jre-slim
+FROM gradle:5.6.4-jdk8 AS build
+COPY --chown=gradle:gradle . /home/gradle/src
+WORKDIR /home/gradle/src
+RUN gradle clean build buildRelease -x test
 
+FROM openjdk:8-jre-slim
 RUN apt-get update && apt-get --yes upgrade && \
     apt-get install -y python3 python3-pip curl && \
     rm -rf /var/lib/apt/lists/*
 
-COPY ./build/output/kafka-gitops /usr/local/bin/kafka-gitops
+COPY --from=build /home/gradle/src/build/output/kafka-gitops /usr/local/bin/kafka-gitops

build.gradle

@@ -4,7 +4,7 @@ plugins {
     id 'application'
     id 'idea'
     id 'jacoco'
-    id 'org.inferred.processors' version '1.2.10'
+    id 'org.inferred.processors' version '2.1.0'
     id "net.ltgt.apt" version "0.21"
     id 'com.github.johnrengelman.shadow' version '4.0.4'
 }
@@ -19,25 +19,36 @@ sourceCompatibility = 1.8
 
 repositories {
     mavenCentral()
+    maven {
+        url "https://packages.confluent.io/maven/"
+    }
+    maven {
+        url "https://jitpack.io"
+    }
 }
 
 dependencies {
-    compile group: 'org.apache.kafka', name: 'kafka-clients', version: '2.4.0'
-    compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.1'
-    compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.8"
-    compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.10.2"
-    compile 'info.picocli:picocli:4.1.4'
-
-    compile 'org.slf4j:slf4j-api:1.7.30'
-    compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3'
-    compile group: 'ch.qos.logback', name: 'logback-core', version: '1.2.3'
-
-    processor 'org.inferred:freebuilder:2.5.0'
-
-    testCompile group: 'junit', name: 'junit', version: '4.12'
-    testCompile group: 'org.codehaus.groovy', name: 'groovy-all', version: '2.4.4'
-    testCompile group: 'org.spockframework', name: 'spock-core', version: '1.0-groovy-2.4'
-    testCompile group: 'cglib', name: 'cglib-nodep', version: '2.2'
+    compile group: 'org.apache.kafka', name: 'kafka-clients', version: '2.8.1'
+    compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.13.2.2'
+    compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.2"
+    compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.2"
+    compile 'info.picocli:picocli:4.6.3'
+
+    implementation ('io.confluent:kafka-schema-registry-client:7.3.3')
+    implementation ('io.confluent:kafka-json-schema-provider:7.3.3')
+    implementation ('io.confluent:kafka-protobuf-serializer:7.3.3')
+    implementation ('io.github.java-diff-utils:java-diff-utils:4.11')
+
+    compile 'org.slf4j:slf4j-api:1.7.36'
+    compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.11'
+    compile group: 'ch.qos.logback', name: 'logback-core', version: '1.2.11'
+
+    processor 'org.inferred:freebuilder:2.7.0'
+
+    testCompile group: 'junit', name: 'junit', version: '4.13.2'
+    testCompile group: 'org.codehaus.groovy', name: 'groovy-all', version: '2.5.16'
+    testCompile group: 'org.spockframework', name: 'spock-core', version: '1.3-groovy-2.5'
+    testCompile group: 'cglib', name: 'cglib-nodep', version: '2.2.2'
     testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: '1.19.0'
     testCompile group: 'org.skyscreamer', name: 'jsonassert', version: '1.5.0'
 }
@@ -46,7 +57,7 @@ jacocoTestReport {
     reports {
         xml.enabled = true
         html.enabled = true
-        xml.destination "${buildDir}/reports/jacoco/report.xml"
+        xml.destination file("${buildDir}/reports/jacoco/report.xml")
     }
 
     afterEvaluate {
@@ -77,4 +88,4 @@ task buildRelease(type: Zip, group: "build") {
 }
 
 buildRelease.dependsOn buildExecutableJar
-buildExecutableJar.dependsOn shadowJar
+buildExecutableJar.dependsOn shadowJar

docker/config/registry_jaas.conf

@@ -0,0 +1,5 @@
+KafkaClient {
+  org.apache.kafka.common.security.plain.PlainLoginModule required
+  username="test"
+  password="test-secret";
+};

docker/docker-compose.yml

@@ -1,24 +1,33 @@
-version: '2.1'
+version: '3.3'
 
 services:
   zoo1:
-    image: zookeeper:3.4.9
+    image: zookeeper:3.8.0
     hostname: zoo1
     ports:
       - "2181:2181"
+    healthcheck:
+      test: echo stat | nc localhost 2181
+      interval: 10s
+      timeout: 10s
+      retries: 3
     environment:
-      ZOO_MY_ID: 1
-      ZOO_PORT: 2181
-      ZOO_SERVERS: server.1=zoo1:2888:3888
-    volumes:
-      - ./data/zoo1/data:/data
-      - ./data/zoo1/datalog:/datalog
+      - ZOOKEEPER_SERVER_ID=1
+      - ZOOKEEPER_CLIENT_PORT=2181
+      - ZOOKEEPER_TICK_TIME=2000
+      - ZOOKEEPER_INIT_LIMIT=5
+      - ZOOKEEPER_SYNC_LIMIT=2
+      - ZOOKEEPER_SERVERS=zoo1:2888:3888
 
   kafka1:
-    image: confluentinc/cp-kafka:5.5.3
+    image: confluentinc/cp-kafka:7.3.3
+    user: "0:0"
     hostname: kafka1
     ports:
       - "9092:9092"
+    restart: on-failure:3
+    healthcheck:
+      test: ps augwwx | egrep [S]upportedKafka
     environment:
       KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:SASL_PLAINTEXT,LISTENER_DOCKER_EXTERNAL:SASL_PLAINTEXT
@@ -27,24 +36,28 @@ services:
       KAFKA_BROKER_ID: 1
       KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf"
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+      KAFKA_DELETE_TOPIC_ENABLE: "true"
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN
       KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
       ZOOKEEPER_SASL_ENABLED: "false"
-      KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer"
+      KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.authorizer.AclAuthorizer"
       KAFKA_SUPER_USERS: "User:test;User:kafka"
       KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
     volumes:
-      - ./data/kafka1/data:/var/lib/kafka/data
       - ./config/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
     depends_on:
       - zoo1
 
   kafka2:
-    image: confluentinc/cp-kafka:5.5.3
+    image: confluentinc/cp-kafka:7.3.3
+    user: "0:0"
     hostname: kafka2
     ports:
       - "9093:9093"
+    restart: on-failure:3
+    healthcheck:
+      test: ps augwwx | egrep [S]upportedKafka
     environment:
       KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:SASL_PLAINTEXT,LISTENER_DOCKER_EXTERNAL:SASL_PLAINTEXT
@@ -53,24 +66,28 @@ services:
       KAFKA_BROKER_ID: 2
       KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf"
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+      KAFKA_DELETE_TOPIC_ENABLE: "true"
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN
       KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
       ZOOKEEPER_SASL_ENABLED: "false"
-      KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer"
+      KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.authorizer.AclAuthorizer"
       KAFKA_SUPER_USERS: "User:test;User:kafka"
       KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
     volumes:
-      - ./data/kafka2/data:/var/lib/kafka/data
       - ./config/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
     depends_on:
       - zoo1
 
   kafka3:
-    image: confluentinc/cp-kafka:5.5.3
+    image: confluentinc/cp-kafka:7.3.3
+    user: "0:0"
     hostname: kafka3
     ports:
       - "9094:9094"
+    restart: on-failure:3
+    healthcheck:
+      test: ps augwwx | egrep [S]upportedKafka
     environment:
       KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:SASL_PLAINTEXT,LISTENER_DOCKER_EXTERNAL:SASL_PLAINTEXT
@@ -79,16 +96,38 @@ services:
       KAFKA_BROKER_ID: 3
       KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf"
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+      KAFKA_DELETE_TOPIC_ENABLE: "true"
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN
       KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
       ZOOKEEPER_SASL_ENABLED: "false"
-      KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer"
+      KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.authorizer.AclAuthorizer"
       KAFKA_SUPER_USERS: "User:test;User:kafka"
       KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
     volumes:
-      - ./data/kafka3/data:/var/lib/kafka/data
       - ./config/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
     depends_on:
       - zoo1
 
+  schema-registry:
+    image: confluentinc/cp-schema-registry:7.3.3
+    hostname: schema-registry
+    ports:
+      - "8082:8082"
+    restart: on-failure:5
+    environment:
+      SCHEMA_REGISTRY_HOST_NAME: schema-registry
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "kafka1:19092,kafka2:19092,kafka3:19092"
+      SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SASL_PLAINTEXT
+      SCHEMA_REGISTRY_KAFKASTORE_SASL_MECHANISM: PLAIN
+      SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8082"
+      SCHEMA_REGISTRY_GROUP_ID: "schema-registry-test"
+      KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/registry_jaas.conf"
+      SCHEMA_REGISTRY_OPTS: "-Djava.security.auth.login.config=/etc/kafka/registry_jaas.conf"
+    volumes:
+      - ./config/registry_jaas.conf:/etc/kafka/registry_jaas.conf
+    depends_on:
+      - kafka1
+      - kafka2
+      - kafka3
+

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.10-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.4-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -28,7 +28,7 @@ APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+DEFAULT_JVM_OPTS='"-Xmx64m"'
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
@@ -66,7 +66,7 @@ esac
 
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
-# Determine the Java cli to use to start the JVM.
+# Determine the Java command to use to start the JVM.
 if [ -n "$JAVA_HOME" ] ; then
     if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
         # IBM's JDK on AIX uses strange locations for the executables
@@ -82,7 +82,7 @@ location of your Java installation."
     fi
 else
     JAVACMD="java"
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' cli could be found in your PATH.
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
@@ -161,7 +161,7 @@ save () {
 }
 APP_ARGS=$(save "$@")
 
-# Collect all arguments for the java cli, following the shell quoting and substitution rules
+# Collect all arguments for the java command, following the shell quoting and substitution rules
 eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
 
 # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong

gradlew.bat

@@ -14,7 +14,7 @@ set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
 @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
+set DEFAULT_JVM_OPTS="-Xmx64m"
 
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome

src/main/java/com/devshawn/kafka/gitops/MainCommand.java

@@ -84,4 +84,4 @@ public static void main(String[] args) {
         int exitCode = new CommandLine(new MainCommand()).execute(args);
         System.exit(exitCode);
     }
-}
+}