Documentation Index
Fetch the complete documentation index at: /llms.txt
Use this file to discover all available pages before exploring further.
Buffer data in Kafka before writing to ClickHouse
kafka write mode buffers data in Kafka before writing it to ClickHouse. This mode is useful when you have a high volume of data and want to avoid overwhelming ClickHouse. This mode takes advantage of ClickHouse’s kafka table engine.
kafka write mode, you can make use of the following environment variables:
# necessary to use the write mode
WRITE_MODE=kafka
# the kafka host and portnames to write to
KAFKA_BROKERS=host1:port1,host2:port2
# the username and password to use for the kafka connection
KAFKA_USERNAME=my-kafka-username
KAFKA_PASSWORD=my-kafka-password
# whether to use SSL for the kafka connection
KAFKA_SSL=true
# the kafka sasl mechanism to use
# possible values are: plain, scram-sha-256, scram-sha-512
KAFKA_SASL_MECHANISM=plain
# the number of partitions to use for the user events topic when bootstrapping
KAFKA_USER_EVENTS_PARTITIONS=1
# the replication factor to use for the user events topic when bootstrapping
KAFKA_USER_EVENTS_REPLICATION_FACTOR=1
# the name of the user events topic to create when bootstrapping
USER_EVENTS_TOPIC_NAME=dittofeed-user-events
<!-- clickhouse_config.xml -->
<clickhouse>
<kafka>
<security_protocol>sasl_plaintext</security_protocol>
<sasl_mechanism>PLAIN</sasl_mechanism>
<sasl_username>your-kafka-username</sasl_username>
<sasl_password>your-kafka-password</sasl_password>
</kafka>
</clickhouse>
<!-- clickhouse_config.xml -->
<clickhouse>
<kafka>
<security_protocol>sasl_ssl</security_protocol>
<sasl_mechanism>PLAIN</sasl_mechanism>
<sasl_username>your-kafka-username</sasl_username>
<sasl_password>your-kafka-password</sasl_password>
</kafka>
</clickhouse>
BOOTSTRAP_KAFKA=true ./admin.sh bootstrap