Apache Kafka Connect to Azure Event Hubs

$ tar -xzf kafka_2.13–3.1.0.tgz$ cd kafka_2.13–3.1.0
$ bin/zookeeper-server-start.sh config/zookeeper.properties
$ bin/kafka-server-start.sh config/server.properties
bootstrap.servers={NAMESPACE.NAME}.servicebus.windows.net:9093security.protocol=SASL_SSLsasl.mechanism=PLAINsasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username=”$ConnectionString” password=”Endpoint=sb://{NAMESPACE.NAME}.servicebus.windows.net/;SharedAccessKeyName={XXXXXX};SharedAccessKey={XXXXXX}”;
bootstrap.servers={NAMESPACE.NAME}.servicebus.windows.net:9093group.id=connect-cluster-group# connect internal topic names, automatically created by Kafka Connect with AdminClient API if not existsconfig.storage.topic=connect-cluster-configsoffset.storage.topic=connect-cluster-offsetsstatus.storage.topic=connect-cluster-status# internal topic replication factors — auto 3x replication in Azure Storageconfig.storage.replication.factor=1offset.storage.replication.factor=1status.storage.replication.factor=1rest.advertised.host.name=connectoffset.flush.interval.ms=10000connections.max.idle.ms=180000metadata.max.age.ms=180000key.converter=org.apache.kafka.connect.json.JsonConvertervalue.converter=org.apache.kafka.connect.json.JsonConverterinternal.key.converter=org.apache.kafka.connect.json.JsonConverterinternal.value.converter=org.apache.kafka.connect.json.JsonConverterinternal.key.converter.schemas.enable=falseinternal.value.converter.schemas.enable=false# required EH Kafka security settingssecurity.protocol=SASL_SSLsasl.mechanism=PLAINsasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username=”$ConnectionString” password=”Endpoint=sb://{NAMESPACE.NAME}.servicebus.windows.net/;SharedAccessKeyName={XXXXXX};SharedAccessKey={XXXXXX}”;producer.security.protocol=SASL_SSLproducer.sasl.mechanism=PLAINproducer.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username=”$ConnectionString” password=”Endpoint=sb://{NAMESPACE.NAME}.servicebus.windows.net/;SharedAccessKeyName={XXXXXX};SharedAccessKey={XXXXXX}”;consumer.security.protocol=SASL_SSLconsumer.sasl.mechanism=PLAINconsumer.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username=”$ConnectionString” password=”Endpoint=sb://{NAMESPACE.NAME}.servicebus.windows.net/;SharedAccessKeyName={XXXXXX};SharedAccessKey={XXXXXX}”;plugin.path={KAFKA.DIRECTORY}/libs # path to the libs directory within the Kafka release
$ bin/kafka-topics.sh — bootstrap-server {NAMESPACE.NAME}.servicebus.windows.net:9093 — command-config path/to/connector.properties — create — topic CONFIGS-TOPIC-NAME — config cleanup.policy=compact — partitions 1 — replication-factor 1
$ bin/kafka-topics.sh — bootstrap-server {NAMESPACE.NAME}.servicebus.windows.net:9093 — command-config path/to/connector.properties — create — topic OFFSETS-TOPIC-NAME — config cleanup.policy=compact — partitions 25 — replication-factor 1
$ bin/kafka-topics.sh — bootstrap-server {NAMESPACE.NAME}.servicebus.windows.net:9093 — command-config path/to/connector.properties — create — topic STATUS-TOPIC-NAME — config cleanup.policy=compact — partitions 5 — replication-factor 1
$ bin/connect-distributed.sh path/to/connect-distributed.properties
$ mkdir ~/connect-demo$ seq 1000 > ~/connect-demo/input.txt$ touch ~/connect-demo/output.txt
curl -s -X POST -H “Content-Type: application/json” — data ‘{“name”: “file-source”,”config”: {“connector.class”:”org.apache.kafka.connect.file.FileStreamSourceConnector”,”tasks.max”:”1",”topic”:”connect-demo”,”file”: “{YOUR/HOME/PATH}/connect-demo/input.txt”}}’ http://localhost:8083/connectors
curl -s http://localhost:8083/connectors/file-source/status
{“name”:”file-source”,”connector”:{“state”:”RUNNING”,”worker_id”:”connect:8083"},”tasks”:[{“id”:0,”state”:”RUNNING”,”worker_id”:”connect:8083"}],”type”:”source”}
curl -X POST -H “Content-Type: application/json” — data ‘{“name”: “file-sink”, “config”: {“connector.class”:”org.apache.kafka.connect.file.FileStreamSinkConnector”, “tasks.max”:”1", “topics”:”connect-demo”, “file”: “{YOUR/HOME/PATH}/connect-demo/output.txt”}}’ http://localhost:8083/connectors
curl -s http://localhost:8083/connectors/file-sink/status
cat ~/connect-demo/output.txt

--

--

I’m a keen traveler to see every country in the world, passionate about cutting edge technologies.

Love podcasts or audiobooks? Learn on the go with our new app.

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store