V2EX = way to explore
V2EX 是一个关于分享和探索的地方
现在注册
已注册用户请  登录
• 请不要在回答技术问题时复制粘贴 AI 生成的内容
qbmiller
V2EX  ›  程序员

kafka-server 出问题,删除所有数据,应用客户端没重启, offset 会乱吗

  •  
  •   qbmiller · 13 小时 35 分钟前 · 324 次点击
    kafka-server 出问题,删除所有数据,
    应用客户端没重启,自动连接上了,消费提交的 offset 会乱吗?还是自动从 头开始

    1. 大佬,谁有 docker 部署 kafka 集群的配置啊,
    2. KAFKA_ENABLE_KRAFT=yes 大家在用了吗?
    3. 目前测试的 客户端不升级(spring-kafka 2.8 kafka-client 3.0) , 服务端 kafka-server 3.6 也能用
    2Nfree
        1
    2Nfree  
       10 小时 4 分钟前
    只删除数据不重置消费组的情况下,对应 topic 的 offset 会变成负数,需要重置消费组
    docker 部署给你一个参考,部署了两种集群:
    ```yaml
    version: '3'
    services:
    zookeeper1:
    image: 'bitnami/zookeeper:3.8'
    container_name: zookeeper1
    ports:
    - '2181:2181'
    environment:
    # 匿名登录--必须开启
    - ALLOW_ANONYMOUS_LOGIN=yes
    # 服务 ID 不重复
    - ZOO_SERVER_ID=1
    # 集群地址配置
    - ZOO_SERVERS=zookeeper1:2888:3888,zookeeper2:2888:3888,zookeeper3:2888:3888
    volumes:
    - ./zookeeper/zookeeper-data1:/bitnami/zookeeper
    zookeeper2:
    image: 'bitnami/zookeeper:3.8'
    container_name: zookeeper2
    ports:
    - '2182:2181'
    environment:
    - ALLOW_ANONYMOUS_LOGIN=yes
    - ZOO_SERVER_ID=2
    - ZOO_SERVERS=zookeeper1:2888:3888,zookeeper2:2888:3888,zookeeper3:2888:3888
    volumes:
    - ./zookeeper/zookeeper-data2:/bitnami/zookeeper
    zookeeper3:
    image: 'bitnami/zookeeper:3.8'
    container_name: zookeeper3
    ports:
    - '2183:2181'
    environment:
    - ALLOW_ANONYMOUS_LOGIN=yes
    - ZOO_SERVER_ID=3
    - ZOO_SERVERS=zookeeper1:2888:3888,zookeeper2:2888:3888,zookeeper3:2888:3888
    volumes:
    - ./zookeeper/zookeeper-data3:/bitnami/zookeeper
    kafka1:
    image: 'bitnami/kafka:3.4'
    container_name: kafka1
    depends_on:
    - zookeeper1
    - zookeeper2
    - zookeeper3
    ports:
    - '9092:9092'
    environment:
    ### zookeeper
    # 配置 zookeeper 地址
    - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper1:2181,zookeeper2:2181,zookeeper3:2181

    ### Listeners
    # 允许使用 PLAINTEXT 监听器
    - ALLOW_PLAINTEXT_LISTENER=yes
    # 指定内部使用的 Broker 监听名称
    - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
    # 配置 Kafka 和 ZooKeeper 之间的通信协议
    - KAFKA_ZOOKEEPER_PROTOCOL=PLAINTEXT
    # 定义安全协议
    - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT
    # 定义 kafka 服务端 socket 监听端口
    - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
    # 配置外部访问地址
    - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092

    ### Cluster
    # 配置偏移量主题为 3 个副本
    - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
    # 配置事务状态日志为 3 个副本
    - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
    # 用于指定事务状态日志的最小 ISR 。ISR 是指与 Kafka Broker 保持同步的副本集合。如果 ISR 中的副本数量低于 2 ,则 Kafka Broker 将停止写入事务状态日志。
    - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2

    # 设置堆内存
    - KAFKA_HEAP_OPTS=-Xmx512m -Xms512m
    volumes:
    - ./kafka-zookeeper/kafka-data1:/bitnami/kafka
    kafka2:
    image: 'bitnami/kafka:3.4'
    container_name: kafka2
    depends_on:
    - zookeeper1
    - zookeeper2
    - zookeeper3
    ports:
    - '9093:9092'
    environment:
    - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
    - ALLOW_PLAINTEXT_LISTENER=yes
    - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
    - KAFKA_ZOOKEEPER_PROTOCOL=PLAINTEXT
    - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT
    - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
    - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092
    - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
    - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
    - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
    - KAFKA_HEAP_OPTS=-Xmx512m -Xms512m
    volumes:
    - ./kafka-zookeeper/kafka-data2:/bitnami/kafka
    kafka3:
    image: 'bitnami/kafka:3.4'
    container_name: kafka3
    depends_on:
    - zookeeper1
    - zookeeper2
    - zookeeper3
    ports:
    - '9094:9092'
    environment:
    - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
    - ALLOW_PLAINTEXT_LISTENER=yes
    - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
    - KAFKA_ZOOKEEPER_PROTOCOL=PLAINTEXT
    - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT
    - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
    - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092
    - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
    - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
    - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
    - KAFKA_HEAP_OPTS=-Xmx512m -Xms512m
    volumes:
    - ./kafka-zookeeper/kafka-data3:/bitnami/kafka
    kafka-kraft1:
    container_name: "kafka-kraft1"
    image: 'bitnami/kafka:3.4'
    restart: always
    ports:
    - '19092:9092'
    environment:
    ### KRaft 配置
    # 允许使用 kraft ,即 Kafka 替代 Zookeeper
    - KAFKA_ENABLE_KRAFT=yes
    # 配置节点唯一 ID
    - KAFKA_CFG_NODE_ID=1
    # 配置 kafka 角色,做 broker ,也要做 controller
    - KAFKA_CFG_PROCESS_ROLES=controller,broker
    # 控制端集群地址
    - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka-kraft1:9093,2@kafka-kraft2:9093,3@kafka-kraft3:9093
    # 使用 Kafka 时的集群 id ,集群内的 Kafka 都要用这个 id 做初始化,生成一个 UUID 即可
    - KAFKA_KRAFT_CLUSTER_ID=Mg-p_2D8RFKm5qYZPgq_hA

    ### Listeners
    # 允许使用 PLAINTEXT 监听器
    - ALLOW_PLAINTEXT_LISTENER=yes
    # 指定控制端使用的监听名称
    - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
    # 指定内部使用的 Broker 监听名称
    - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
    # 定义安全协议
    - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
    # 配置 kafka 服务端 socket 监听端口
    - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
    # 配置外部访问地址
    - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092

    ### Cluster
    # 配置偏移量主题为 3 个副本
    - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
    # 配置事务状态日志为 3 个副本
    - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
    # 用于指定事务状态日志的最小 ISR 。ISR 是指与 Kafka Broker 保持同步的副本集合。如果 ISR 中的副本数量低于 2 ,则 Kafka Broker 将停止写入事务状态日志。
    - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2

    # 设置堆内存
    - KAFKA_HEAP_OPTS=-Xmx512m -Xms512m
    volumes:
    - ./kafka-kraft/kafka-data1:/bitnami/kafka
    kafka-kraft2:
    container_name: "kafka-kraft2"
    image: 'bitnami/kafka:3.4'
    restart: always
    ports:
    - '19093:9092'
    environment:
    - KAFKA_ENABLE_KRAFT=yes
    - KAFKA_CFG_NODE_ID=2
    - KAFKA_CFG_PROCESS_ROLES=controller,broker
    - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka-kraft1:9093,2@kafka-kraft2:9093,3@kafka-kraft3:9093
    - KAFKA_KRAFT_CLUSTER_ID=Mg-p_2D8RFKm5qYZPgq_hA
    - ALLOW_PLAINTEXT_LISTENER=yes
    - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
    - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
    - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
    - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
    - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092
    - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
    - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
    - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
    - KAFKA_HEAP_OPTS=-Xmx512m -Xms512m
    volumes:
    - ./kafka-kraft/kafka-data2:/bitnami/kafka
    kafka-kraft3:
    container_name: "kafka-kraft3"
    image: 'bitnami/kafka:3.4'
    restart: always
    ports:
    - '19094:9092'
    environment:
    - KAFKA_ENABLE_KRAFT=yes
    - KAFKA_CFG_NODE_ID=3
    - KAFKA_CFG_PROCESS_ROLES=controller,broker
    - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka-kraft1:9093,2@kafka-kraft2:9093,3@kafka-kraft3:9093
    - KAFKA_KRAFT_CLUSTER_ID=Mg-p_2D8RFKm5qYZPgq_hA
    - ALLOW_PLAINTEXT_LISTENER=yes
    - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
    - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
    - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
    - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
    - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092
    - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
    - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
    - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
    - KAFKA_HEAP_OPTS=-Xmx512m -Xms512m
    volumes:
    - ./kafka-kraft/kafka-data3:/bitnami/kafka
    kafka-ui:
    image: provectuslabs/kafka-ui:latest
    container_name: kafka-ui
    restart: always
    depends_on:
    - kafka1
    - kafka2
    - kafka3
    - kafka-kraft1
    - kafka-kraft2
    - kafka-kraft3
    ports:
    - 8080:8080
    environment:
    - KAFKA_CLUSTERS_0_NAME=kafka-zookeeper
    - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=172.28.13.101:9092,172.28.13.101:9093,172.28.13.101:9094
    - KAFKA_CLUSTERS_1_NAME=kafka-kraft
    - KAFKA_CLUSTERS_1_BOOTSTRAPSERVERS=172.28.13.101:19092,172.28.13.101:19093,172.28.13.101:19094
    - SERVER_SERVLET_CONTEXT_PATH=/
    - AUTH_TYPE= "LOGIN_FORM"
    - SPRING_SECURITY_USER_NAME=admin
    - SPRING_SECURITY_USER_PASSWORD=admin
    - DYNAMIC_CONFIG_ENABLED='true'
    kafka-map:
    image: dushixiang/kafka-map:v1.3.3
    container_name: kafka-map
    restart: always
    depends_on:
    - kafka1
    - kafka2
    - kafka3
    - kafka-kraft1
    - kafka-kraft2
    - kafka-kraft3
    ports:
    - 8081:8080
    environment:
    - DEFAULT_USERNAME=admin
    - DEFAULT_PASSWORD=admin
    ```
    kraft 我们是没有用的,之前 zookeeper 用惯了
    关于   ·   帮助文档   ·   博客   ·   API   ·   FAQ   ·   实用小工具   ·   3157 人在线   最高记录 6679   ·     Select Language
    创意工作者们的社区
    World is powered by solitude
    VERSION: 3.9.8.5 · 23ms · UTC 13:21 · PVG 21:21 · LAX 06:21 · JFK 09:21
    Developed with CodeLauncher
    ♥ Do have faith in what you're doing.