import threading
from queue import Queue
import time
import concurrent.futures
def get_detail_html(detail_url_list, id):
while True:
url = detail_url_list.get()
#设置退出机制
if(url == "STOP"):
break
#time.sleep(2)
print(f"thread {id}: get {url} detail finished")
print(f"html thread {id} end")
def get_detail_url(queue):
for i in range(10):
time.sleep(1)
queue.put(f"url: {i}")
print(f"get detail url {i} end")
#设置退出机制 不然线程内部queue读取循环无法结束
#写入数量>=读取线程数
for i in range(10):
queue.put("STOP")
print("url thread end")
if __name__ == "__main__":
detail_url_queue = Queue(maxsize=1000)
start_time = time.time()
'''
html_thread = []
for i in range(3):
thread2 = threading.Thread(target=get_detail_html, args=(detail_url_queue, i))
html_thread.append(thread2)
#启动所有queue的读取线程
for i in range(3):
html_thread[i].start()
'''
# 使用线程池 (线程池线程无法执行到queue.get卡住了?)
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
a = []
b = []
for i in range(3):
a.append(detail_url_queue)
b.append(i)
executor.map(get_detail_html,a,b)
#启动queue的写入线程
thread = threading.Thread(target=get_detail_url, args=(detail_url_queue,))
thread.start()
#等待所有线程结束
thread.join()
'''
for i in range(3):
html_thread[i].join()
'''
print(f"last time: {time.time() - start_time} s")
以上代码执行到queue.get的时候会卡住,无法正常执行循环。哪怕后面的线程往队列推入了数据。