4.0.0 io.confluent kafka-connect-storage-common-parent 11.0.9 kafka-connect-hdfs 10.1.11 jar kafka-connect-hdfs Confluent, Inc. http://confluent.io http://confluent.io A Kafka Connect HDFS connector for copying data between Kafka and Hadoop HDFS. Confluent Community License http://www.confluent.io/confluent-community-license repo scm:git:git://github.com/confluentinc/kafka-connect-hdfs.git scm:git:git@github.com:confluentinc/kafka-connect-hdfs.git https://github.com/confluentinc/kafka-connect-hdfs v10.1.11 http://packages.confluent.io/maven/ 2.0.0-M2 1.2.17-cp8 0.11.1 2.5.3 11.0.9 3.2.2 0.13.0 2.17.1 6.1.6 confluent Confluent ${confluent.maven.repo} org.apache.kafka connect-api provided org.apache.kafka connect-json ${kafka.version} provided io.confluent kafka-connect-storage-common ${kafka.connect.storage.common.version} io.confluent kafka-connect-storage-core ${kafka.connect.storage.common.version} io.confluent kafka-connect-storage-format ${kafka.connect.storage.common.version} io.confluent kafka-connect-storage-partitioner ${kafka.connect.storage.common.version} io.confluent kafka-connect-storage-wal ${kafka.connect.storage.common.version} io.confluent kafka-connect-storage-hive ${kafka.connect.storage.common.version} io.netty netty io.netty netty-codec org.apache.htrace htrace-core4 org.apache.htrace htrace-core org.apache.calcite.avatica avatica org.apache.avro avro-ipc-jetty org.apache.logging.log4j log4j-core org.apache.calcite calcite-core 1.11.0 org.apache.calcite calcite-druid 1.11.0 commons-collections commons-collections ${commons.collections.version} org.apache.thrift libthrift ${libthrift.version} org.apache.logging.log4j log4j-core ${log4j2-api.version} org.apache.logging.log4j log4j-1.2-api ${log4j2-api.version} org.apache.logging.log4j log4j-slf4j-impl ${log4j2-api.version} org.apache.logging.log4j log4j-core com.lmax disruptor 3.4.2 com.github.spotbugs spotbugs-annotations com.fasterxml.jackson.core jackson-databind com.fasterxml.jackson.core jackson-core io.confluent kafka-connect-storage-common-htrace-core4-shaded ${kafka.connect.storage.common.version} io.confluent kafka-connect-storage-common-avatica-shaded ${kafka.connect.storage.common.version} com.google.guava guava 18.0 test org.apache.hadoop hadoop-minicluster ${hadoop.version} test io.netty netty io.netty netty-all org.apache.htrace htrace-core4 log4j log4j org.apache.hadoop hadoop-minikdc ${hadoop.version} test org.apache.directory.jdbm apacheds-jdbm1 log4j log4j org.apache.htrace htrace-core4 io.confluent confluent-log4j ${confluent-log4j.version} test org.apache.directory.jdbm apacheds-jdbm1 ${apacheds-jdbm1.version} test org.mockito mockito-core 2.28.2 test org.mockito mockito-all 1.10.19 test io.confluent ${kafka.connect.maven.plugin.version} kafka-connect-maven-plugin kafka-connect https://docs.confluent.io/kafka-connect-hdfs/current/index.html The HDFS connector allows you to export data from Kafka topics to HDFS files in a variety of formats and integrates with Hive to make data immediately available for querying with HiveQL. The connector periodically polls data from Kafka and writes them to HDFS. The data from each Kafka topic is partitioned by the provided partitioner and divided into chunks. Each chunk of data is represented as an HDFS file with topic, Kafka partition, start and end offsets of this data chunk in the filename. If no partitioner is specified in the configuration, the default partitioner which preserves the Kafka partitioning is used. The size of each data chunk is determined by the number of records written to HDFS, the time written to HDFS and schema compatibility. The HDFS connector integrates with Hive and when it is enabled, the connector automatically creates an external Hive partitioned table for each Kafka topic and updates the table according to the available data in HDFS. Confluent, Inc. supported by Confluent as part of a Confluent Platform subscription.]]> https://docs.confluent.io/current/ logos/confluent.png confluentinc organization Confluent, Inc. https://confluent.io/ logos/confluent.png sink hadoop hdfs hive true org.apache.maven.plugins maven-compiler-plugin -Xlint:all,-deprecation,-processing true false maven-assembly-plugin src/assembly/development.xml src/assembly/package.xml false make-assembly package single org.apache.maven.plugins maven-surefire-plugin false 1 true src/test/resources/conf org.apache.maven.plugins maven-checkstyle-plugin validate validate checkstyle/suppressions.xml check maven-clean-plugin 3.0.0 . derby.log metastore_db/ org.apache.maven.plugins maven-release-plugin ${maven.release.plugin.version} true false v@{project.version} src/main/resources true standalone maven-assembly-plugin src/assembly/standalone.xml