diff --git a/Jenkinsfile b/Jenkinsfile index c8450f1..e0f7e4c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,23 +1,5 @@ #!groovy -node { - def mvnBuildNumber = "0.1.${env.BUILD_NUMBER}" +@Library('jenkins-pipeline') import com.github.jcustenborder.jenkins.pipeline.KafkaConnectPipeline - def mvnHome = tool 'M3' - - checkout scm - - if (env.BRANCH_NAME == 'master') { - stage 'versioning' - sh "${mvnHome}/bin/mvn -B versions:set -DgenerateBackupPoms=false -DnewVersion=${mvnBuildNumber}" - } - - stage 'build' - sh "${mvnHome}/bin/mvn -B -P maven-central clean verify package" - - junit '**/target/surefire-reports/TEST-*.xml' - - if (env.BRANCH_NAME == 'master') { - stage 'publishing' - sh "${mvnHome}/bin/mvn -B -P github,maven-central deploy" - } -} \ No newline at end of file +def pipe = new KafkaConnectPipeline() +pipe.execute() \ No newline at end of file diff --git a/README.md b/README.md index 00a3205..8c8061a 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ it will sit behind a load balancer nicely. ``` name=splunk-http-source tasks.max=1 -connector.class=io.confluent.kafka.connect.splunk.SplunkHttpSourceConnector +connector.class=com.github.jcustenborder.kafka.connect.splunk.SplunkHttpSourceConnector splunk.ssl.key.store.path=/etc/security/keystore.jks splunk.ssl.key.store.password=password splunk.collector.index.default=main @@ -60,7 +60,7 @@ This configuration will write to Splunk over SSL but will not verify the certifi name=splunk-http-sink topics=syslog-udp tasks.max=1 -connector.class=io.confluent.kafka.connect.splunk.SplunkHttpSinkConnector +connector.class=com.github.jcustenborder.kafka.connect.splunk.SplunkHttpSinkConnector splunk.remote.host=192.168.99.100 splunk.remote.port=8088 splunk.ssl.enabled=true @@ -80,7 +80,7 @@ object. Here is an example of an event generated by [Kafka Connect Syslog](https://github.com/jcustenborder/kafka-connect-syslog) written to Splunk. -``` +```json { "host": "vpn.example.com", "time": 1472342182, @@ -96,8 +96,15 @@ Here is an example of an event generated by [Kafka Connect Syslog](https://githu # Running in development +## Run the connector + +```bash +./bin/debug.sh ``` -mvn clean package -export CLASSPATH="$(find target/ -type f -name '*.jar'| grep '\-package' | tr '\n' ':')" -$CONFLUENT_HOME/bin/connect-standalone $CONFLUENT_HOME/etc/schema-registry/connect-avro-standalone.properties config/MySourceConnector.properties -``` + +## Suspend waiting on the debugger to attach. + +```bash +export SUSPEND='y' +./bin/debug.sh +``` \ No newline at end of file diff --git a/bin/debug.sh b/bin/debug.sh index 976f02c..10aa491 100755 --- a/bin/debug.sh +++ b/bin/debug.sh @@ -15,17 +15,12 @@ # limitations under the License. # -mvn clean package +: ${SUSPEND:='n'} -if [ $? -eq 0 ] -then - echo "maven finished" -else - echo "Maven failed" - exit 1 -fi +set -e -export CLASSPATH="$(find target/ -type f -name '*.jar'| grep '\-package' | tr '\n' ':')" -export KAFKA_JMX_OPTS='-Xdebug -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005' +mvn clean package +export KAFKA_JMX_OPTS="-Xdebug -agentlib:jdwp=transport=dt_socket,server=y,suspend=${SUSPEND},address=5005" +export CLASSPATH="$(find target/kafka-connect-target/share/java -type f -name '*.jar' | tr '\n' ':')" -$CONFLUENT_HOME/bin/connect-standalone connect/connect-avro-docker.properties config/SplunkHttpSource.properties config/SplunkHttpSink.properties \ No newline at end of file +$CONFLUENT_HOME/bin/connect-standalone \ No newline at end of file diff --git a/bin/suspend.sh b/bin/suspend.sh deleted file mode 100755 index d35a621..0000000 --- a/bin/suspend.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -mvn clean package - -if [ $? -eq 0 ] -then - echo "maven finished" -else - echo "Maven failed" - exit 1 -fi - -export CLASSPATH="$(find target/ -type f -name '*.jar'| grep '\-package' | tr '\n' ':')" -export KAFKA_JMX_OPTS='-Xdebug -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005' - -$CONFLUENT_HOME/bin/connect-standalone connect/connect-avro-docker.properties config/SplunkHttpSink.properties config/SplunkHttpSource.properties \ No newline at end of file diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml deleted file mode 100644 index 9190e45..0000000 --- a/checkstyle/checkstyle.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/connect/connect-avro-docker.properties b/config/connect-avro-docker.properties similarity index 100% rename from connect/connect-avro-docker.properties rename to config/connect-avro-docker.properties diff --git a/docker-compose.yml b/docker-compose.yml index 5c7c122..33c5188 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ # -# Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com) +# Copyright © 2017 Jeremy Custenborder (jcustenborder@gmail.com) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,31 +17,30 @@ version: "2" services: zookeeper: - image: confluent/zookeeper - ports: - - "2181:2181" + image: confluentinc/cp-zookeeper:3.1.1-1 environment: + ZOOKEEPER_CLIENT_PORT: 2181 zk_id: "1" - network_mode: "host" kafka: - image: confluent/kafka - depends_on: + hostname: kafka + image: confluentinc/cp-kafka:3.1.1-1 + links: - zookeeper ports: - "9092:9092" environment: - KAFKA_ZOOKEEPER_CONNECT: "confluent:2181" - network_mode: "host" + KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://:9092" schema-registry: - image: confluent/schema-registry - depends_on: + image: confluentinc/cp-schema-registry:3.1.1-1 + links: - kafka - zookeeper ports: - "8081:8081" environment: - SR_KAFKASTORE_CONNECTION_URL: "confluent:2181" - network_mode: "host" + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:2181" + SCHEMA_REGISTRY_HOST_NAME: schema-registry splunk: image: outcoldman/splunk:latest environment: