![]() |
1
2Nfree 10 小时 4 分钟前
只删除数据不重置消费组的情况下,对应 topic 的 offset 会变成负数,需要重置消费组
docker 部署给你一个参考,部署了两种集群: ```yaml version: '3' services: zookeeper1: image: 'bitnami/zookeeper:3.8' container_name: zookeeper1 ports: - '2181:2181' environment: # 匿名登录--必须开启 - ALLOW_ANONYMOUS_LOGIN=yes # 服务 ID 不重复 - ZOO_SERVER_ID=1 # 集群地址配置 - ZOO_SERVERS=zookeeper1:2888:3888,zookeeper2:2888:3888,zookeeper3:2888:3888 volumes: - ./zookeeper/zookeeper-data1:/bitnami/zookeeper zookeeper2: image: 'bitnami/zookeeper:3.8' container_name: zookeeper2 ports: - '2182:2181' environment: - ALLOW_ANONYMOUS_LOGIN=yes - ZOO_SERVER_ID=2 - ZOO_SERVERS=zookeeper1:2888:3888,zookeeper2:2888:3888,zookeeper3:2888:3888 volumes: - ./zookeeper/zookeeper-data2:/bitnami/zookeeper zookeeper3: image: 'bitnami/zookeeper:3.8' container_name: zookeeper3 ports: - '2183:2181' environment: - ALLOW_ANONYMOUS_LOGIN=yes - ZOO_SERVER_ID=3 - ZOO_SERVERS=zookeeper1:2888:3888,zookeeper2:2888:3888,zookeeper3:2888:3888 volumes: - ./zookeeper/zookeeper-data3:/bitnami/zookeeper kafka1: image: 'bitnami/kafka:3.4' container_name: kafka1 depends_on: - zookeeper1 - zookeeper2 - zookeeper3 ports: - '9092:9092' environment: ### zookeeper # 配置 zookeeper 地址 - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper1:2181,zookeeper2:2181,zookeeper3:2181 ### Listeners # 允许使用 PLAINTEXT 监听器 - ALLOW_PLAINTEXT_LISTENER=yes # 指定内部使用的 Broker 监听名称 - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT # 配置 Kafka 和 ZooKeeper 之间的通信协议 - KAFKA_ZOOKEEPER_PROTOCOL=PLAINTEXT # 定义安全协议 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT # 定义 kafka 服务端 socket 监听端口 - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092 # 配置外部访问地址 - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092 ### Cluster # 配置偏移量主题为 3 个副本 - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3 # 配置事务状态日志为 3 个副本 - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3 # 用于指定事务状态日志的最小 ISR 。ISR 是指与 Kafka Broker 保持同步的副本集合。如果 ISR 中的副本数量低于 2 ,则 Kafka Broker 将停止写入事务状态日志。 - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2 # 设置堆内存 - KAFKA_HEAP_OPTS=-Xmx512m -Xms512m volumes: - ./kafka-zookeeper/kafka-data1:/bitnami/kafka kafka2: image: 'bitnami/kafka:3.4' container_name: kafka2 depends_on: - zookeeper1 - zookeeper2 - zookeeper3 ports: - '9093:9092' environment: - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper1:2181,zookeeper2:2181,zookeeper3:2181 - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT - KAFKA_ZOOKEEPER_PROTOCOL=PLAINTEXT - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092 - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092 - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3 - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3 - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2 - KAFKA_HEAP_OPTS=-Xmx512m -Xms512m volumes: - ./kafka-zookeeper/kafka-data2:/bitnami/kafka kafka3: image: 'bitnami/kafka:3.4' container_name: kafka3 depends_on: - zookeeper1 - zookeeper2 - zookeeper3 ports: - '9094:9092' environment: - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper1:2181,zookeeper2:2181,zookeeper3:2181 - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT - KAFKA_ZOOKEEPER_PROTOCOL=PLAINTEXT - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092 - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092 - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3 - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3 - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2 - KAFKA_HEAP_OPTS=-Xmx512m -Xms512m volumes: - ./kafka-zookeeper/kafka-data3:/bitnami/kafka kafka-kraft1: container_name: "kafka-kraft1" image: 'bitnami/kafka:3.4' restart: always ports: - '19092:9092' environment: ### KRaft 配置 # 允许使用 kraft ,即 Kafka 替代 Zookeeper - KAFKA_ENABLE_KRAFT=yes # 配置节点唯一 ID - KAFKA_CFG_NODE_ID=1 # 配置 kafka 角色,做 broker ,也要做 controller - KAFKA_CFG_PROCESS_ROLES=controller,broker # 控制端集群地址 - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka-kraft1:9093,2@kafka-kraft2:9093,3@kafka-kraft3:9093 # 使用 Kafka 时的集群 id ,集群内的 Kafka 都要用这个 id 做初始化,生成一个 UUID 即可 - KAFKA_KRAFT_CLUSTER_ID=Mg-p_2D8RFKm5qYZPgq_hA ### Listeners # 允许使用 PLAINTEXT 监听器 - ALLOW_PLAINTEXT_LISTENER=yes # 指定控制端使用的监听名称 - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER # 指定内部使用的 Broker 监听名称 - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT # 定义安全协议 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT # 配置 kafka 服务端 socket 监听端口 - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 # 配置外部访问地址 - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092 ### Cluster # 配置偏移量主题为 3 个副本 - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3 # 配置事务状态日志为 3 个副本 - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3 # 用于指定事务状态日志的最小 ISR 。ISR 是指与 Kafka Broker 保持同步的副本集合。如果 ISR 中的副本数量低于 2 ,则 Kafka Broker 将停止写入事务状态日志。 - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2 # 设置堆内存 - KAFKA_HEAP_OPTS=-Xmx512m -Xms512m volumes: - ./kafka-kraft/kafka-data1:/bitnami/kafka kafka-kraft2: container_name: "kafka-kraft2" image: 'bitnami/kafka:3.4' restart: always ports: - '19093:9092' environment: - KAFKA_ENABLE_KRAFT=yes - KAFKA_CFG_NODE_ID=2 - KAFKA_CFG_PROCESS_ROLES=controller,broker - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka-kraft1:9093,2@kafka-kraft2:9093,3@kafka-kraft3:9093 - KAFKA_KRAFT_CLUSTER_ID=Mg-p_2D8RFKm5qYZPgq_hA - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092 - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3 - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3 - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2 - KAFKA_HEAP_OPTS=-Xmx512m -Xms512m volumes: - ./kafka-kraft/kafka-data2:/bitnami/kafka kafka-kraft3: container_name: "kafka-kraft3" image: 'bitnami/kafka:3.4' restart: always ports: - '19094:9092' environment: - KAFKA_ENABLE_KRAFT=yes - KAFKA_CFG_NODE_ID=3 - KAFKA_CFG_PROCESS_ROLES=controller,broker - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka-kraft1:9093,2@kafka-kraft2:9093,3@kafka-kraft3:9093 - KAFKA_KRAFT_CLUSTER_ID=Mg-p_2D8RFKm5qYZPgq_hA - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092 - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3 - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3 - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2 - KAFKA_HEAP_OPTS=-Xmx512m -Xms512m volumes: - ./kafka-kraft/kafka-data3:/bitnami/kafka kafka-ui: image: provectuslabs/kafka-ui:latest container_name: kafka-ui restart: always depends_on: - kafka1 - kafka2 - kafka3 - kafka-kraft1 - kafka-kraft2 - kafka-kraft3 ports: - 8080:8080 environment: - KAFKA_CLUSTERS_0_NAME=kafka-zookeeper - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=172.28.13.101:9092,172.28.13.101:9093,172.28.13.101:9094 - KAFKA_CLUSTERS_1_NAME=kafka-kraft - KAFKA_CLUSTERS_1_BOOTSTRAPSERVERS=172.28.13.101:19092,172.28.13.101:19093,172.28.13.101:19094 - SERVER_SERVLET_CONTEXT_PATH=/ - AUTH_TYPE= "LOGIN_FORM" - SPRING_SECURITY_USER_NAME=admin - SPRING_SECURITY_USER_PASSWORD=admin - DYNAMIC_CONFIG_ENABLED='true' kafka-map: image: dushixiang/kafka-map:v1.3.3 container_name: kafka-map restart: always depends_on: - kafka1 - kafka2 - kafka3 - kafka-kraft1 - kafka-kraft2 - kafka-kraft3 ports: - 8081:8080 environment: - DEFAULT_USERNAME=admin - DEFAULT_PASSWORD=admin ``` kraft 我们是没有用的,之前 zookeeper 用惯了 |