genispan 2025-03-20 20:25 采纳率: 100%
浏览 5
已结题

python线程池无法正常执行 和queue.get冲突?


import threading
from queue import Queue
import time
import concurrent.futures
 
def get_detail_html(detail_url_list, id):
    while True:
        url = detail_url_list.get()
        #设置退出机制
        if(url == "STOP"):
            break
        #time.sleep(2)
        print(f"thread {id}: get {url} detail finished")
    print(f"html thread {id} end")
 
def get_detail_url(queue):
    for i in range(10):
        time.sleep(1)
        queue.put(f"url: {i}")
        print(f"get detail url {i} end")
    #设置退出机制 不然线程内部queue读取循环无法结束 
    #写入数量>=读取线程数
    for i in range(10):
        queue.put("STOP")    
    print("url thread end")

 
if __name__ == "__main__":
    detail_url_queue = Queue(maxsize=1000)
    start_time = time.time()
    '''
    html_thread = []
    for i in range(3):
        thread2 = threading.Thread(target=get_detail_html, args=(detail_url_queue, i))
        html_thread.append(thread2)

    #启动所有queue的读取线程    
    for i in range(3):
        html_thread[i].start()
      
    '''
   
    # 使用线程池 (线程池线程无法执行到queue.get卡住了?)
    with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
        a = []
        b = []
        for i in range(3):
            a.append(detail_url_queue)
            b.append(i)
        executor.map(get_detail_html,a,b)


   #启动queue的写入线程 
    thread = threading.Thread(target=get_detail_url, args=(detail_url_queue,))
    thread.start()



    #等待所有线程结束 
    thread.join()   
    '''
    for i in range(3):
        html_thread[i].join()
    ''' 

    print(f"last time: {time.time() - start_time} s")

以上代码执行到queue.get的时候会卡住,无法正常执行循环。哪怕后面的线程往队列推入了数据。

  • 写回答

4条回答 默认 最新

  • genispan 2025-03-20 20:54
    关注
    
    
    ```python
    import threading
    from queue import Queue
    import time
    import concurrent.futures
     
    def get_detail_html(detail_url_list, id):
        while True:
            url = detail_url_list.get()
            #设置退出机制
            if url is None:
                break
            time.sleep(2)
            print(f"thread {id}: get {url} detail finished")
        print(f"html thread {id} end")
     
    def get_detail_url(queue):
        for i in range(10):
            time.sleep(1)
            queue.put(f"url: {i}")
            print(f"get detail url {i} end")
        #设置退出机制 不然线程内部queue读取循环无法结束 
        #写入数量>=读取线程数
        for i in range(10):
            queue.put(None)    
        print("url thread end")
    
     
    if __name__ == "__main__":
        detail_url_queue = Queue(maxsize=1000)
        start_time = time.time()
        '''
        html_thread = []
        for i in range(3):
            thread2 = threading.Thread(target=get_detail_html, args=(detail_url_queue, i))
            html_thread.append(thread2)
    
        #启动所有queue的读取线程    
        for i in range(3):
            html_thread[i].start()
          
        '''
        # 修改重点!!!!!!!
        executor = concurrent.futures.ThreadPoolExecutor(max_workers=5)
        for i in range(5):
            future = executor.submit(get_detail_html, detail_url_queue,i)
       
    
       #启动queue的写入线程 
        thread = threading.Thread(target=get_detail_url, args=(detail_url_queue,))
        thread.start()
    
    
    
        #等待所有线程结束 
        thread.join()   
        executor.shutdown()
        '''
        for i in range(3):
            html_thread[i].join()
        ''' 
    
        print(f"last time: {time.time() - start_time} s")
    
    
    
    问题已解决,确认Queue是线程安全的。
    
    ```python
    
       # 修改重点!!!!!!!
        executor = concurrent.futures.ThreadPoolExecutor(max_workers=5)
        for i in range(5):
            future = executor.submit(get_detail_html, detail_url_queue,i)
    
    本回答被题主选为最佳回答 , 对您是否有帮助呢?
    评论
查看更多回答(3条)

报告相同问题?

问题事件

  • 系统已结题 3月28日
  • 已采纳回答 3月20日
  • 创建了问题 3月20日