python连接kafka-python连接kafka生产者，消费者脚本

最新推荐文章于 2024-09-24 17:00:03 发布

weixin_39831705

最新推荐文章于 2024-09-24 17:00:03 发布

阅读量439

点赞数

本文介绍如何使用 Kafka-Python 模块进行消息生产和消费。通过具体代码实例展示了配置 Kafka 生产者与消费者的方法，并实现了 JSON 数据的发送及接收。适用于希望快速上手 Kafka 的开发者。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

#-*- coding: utf-8 -*-

"""""

使用kafka-Python 1.3.3模块

# pip install kafka==1.3.5

# pip install kafka-python==1.3.5"""

importsysimporttimeimportjsonfrom kafka importKafkaProducerfrom kafka importKafkaConsumerfrom kafka.errors importKafkaError

KAFAKA_HOST= "101.236.51.235"KAFAKA_PORT= 9092KAFAKA_TOPIC= "test"

classKafka_producer():"""""

生产模块：根据不同的key，区分消息"""

def __init__(self, kafkahost,kafkaport, kafkatopic, key):

self.kafkaHost=kafkahost

self.kafkaPort=kafkaport

self.kafkatopic=kafkatopic

self.key=keyprint("producer:h,p,t,k",kafkahost,kafkaport,kafkatopic,key)

bootstrap_servers= "{kafka_host}:{kafka_port}".format(

kafka_host=self.kafkaHost,

kafka_port=self.kafkaPort

)print("boot svr:",bootstrap_servers)

self.producer= KafkaProducer(bootstrap_servers =bootstrap_servers

)defsendjsondata(self, params):try:

parmas_message= json.dumps(params,ensure_ascii=False)

producer=self.producerprint(parmas_message)

v= parmas_message.encode("utf-8")

k= key.encode("utf-8")print("send msg:(k,v)",k,v)

producer.send(self.kafkatopic, key=k, value=v)

producer.flush()exceptKafkaError as e:print(e)classKafka_consumer():"""""

消费模块: 通过不同groupid消费topic里面的消息"""

def __init__(self, kafkahost, kafkaport, kafkatopic, groupid):

self.kafkaHost=kafkahost

self.kafkaPort=kafkaport

self.kafkatopic=kafkatopic

self.groupid=groupid

self.key=key

self.consumer= KafkaConsumer(self.kafkatopic, group_id =self.groupid,

bootstrap_servers= "{kafka_host}:{kafka_port}".format(

kafka_host=self.kafkaHost,

kafka_port=self.kafkaPort )

)defconsume_data(self):try:for message inself.consumer:yieldmessageexceptKeyboardInterrupt as e:print(e)defmain(xtype, group, key):"""""

测试consumer和producer"""

if xtype == "p":#生产模块

producer =Kafka_producer(KAFAKA_HOST, KAFAKA_PORT, KAFAKA_TOPIC, key)print ("===========> producer:", producer)for _id in range(100):

params= "{"msg" : "%s"}" %str(_id)

params=[{"msg0" :_id},{"msg1":_id}]

producer.sendjsondata(params)

time.sleep(1)if xtype == "c":#消费模块

consumer =Kafka_consumer(KAFAKA_HOST, KAFAKA_PORT, KAFAKA_TOPIC, group)print ("===========> consumer:", consumer)

message=consumer.consume_data()for msg inmessage:print ("msg---------------->k,v", msg.key,msg.value)print ("offset---------------->", msg.offset)if __name__ == "__main__":

xtype= sys.argv[1]

group= sys.argv[2]

key= sys.argv[3]

main(xtype, group, key)