python 多进程的3种方式

1. multiprocessing.Process

import time

import multiprocessing

from dataclasses import dataclass

@dataclass

class PersonInfo:

seconds: int

def do_something(person_info: PersonInfo, gap):

print(f"Sleeping {person_info.seconds + gap} second(s)...")

time.sleep(person_info.seconds + gap)

print("Done Sleep...")

if __name__ == '__main__':

start = time.perf_counter()

processes = []

for _ in range(10):

p = multiprocessing.Process(target=do_something, args=[PersonInfo(1.5), 0.5])

p.start()

processes.append(p)

for process in processes:

process.join()

finish = time.perf_counter()

print(f"Finished in {round(finish - start, 2)} second(s)")

日志：

Sleeping 2.0 second(s)...

Sleeping 2.0 second(s)...

Sleeping 2.0 second(s)...

Sleeping 2.0 second(s)...

Sleeping 2.0 second(s)...

Sleeping 2.0 second(s)...

Sleeping 2.0 second(s)...

Sleeping 2.0 second(s)...

Sleeping 2.0 second(s)...

Sleeping 2.0 second(s)...

Done Sleep...

Done Sleep...

Done Sleep...

Done Sleep...Done Sleep...Done Sleep...

Done Sleep...Done Sleep...

Done Sleep...

Done Sleep...

Finished in 2.13 second(s)

2. concurrent.futures.ProcessPoolExecutor()

executor.submit

import time

import concurrent.futures

from dataclasses import dataclass

@dataclass

class PersonInfo:

seconds: int

def do_something(person_info: PersonInfo, gap):

print(f"Sleeping {person_info.seconds + gap} second(s)...")

time.sleep(person_info.seconds + gap)

return f"Done Sleep...{gap}"

if __name__ == '__main__':

start = time.perf_counter()

with concurrent.futures.ProcessPoolExecutor() as executor:

secs = [5, 4, 3, 2, 1]

results = [executor.submit(do_something, PersonInfo(seconds=1), sec) for sec in secs]

# todo 按照结束的顺序打印

for f in concurrent.futures.as_completed(results):

print(f.result())

finish = time.perf_counter()

print(f"Finished in {round(finish - start, 2)} second(s)")

日志：

Sleeping 6 second(s)...

Sleeping 5 second(s)...

Sleeping 4 second(s)...

Sleeping 3 second(s)...

Sleeping 2 second(s)...

Done Sleep...1

Done Sleep...2

Done Sleep...3

Done Sleep...4

Done Sleep...5

Finished in 6.14 second(s)

3. concurrent.futures.ProcessPoolExecutor()

executor.map

import time

import concurrent.futures

from dataclasses import dataclass

@dataclass

class PersonInfo:

seconds: int

def run_task(args):

return do_something(*args)

def do_something(person_info: PersonInfo, gap):

print(f"Sleeping {person_info.seconds + gap} second(s)...")

time.sleep(person_info.seconds + gap)

return f"Done Sleep...{gap}"

if __name__ == '__main__':

start = time.perf_counter()

with concurrent.futures.ProcessPoolExecutor() as executor:

# with concurrent.futures.ThreadPoolExecutor() as executor:

secs = [5, 4, 3, 2, 1]

# results = [executor.submit(do_something, PersonInfo(seconds=1), sec) for sec in secs]

arg_iterable = [(PersonInfo(1), sec) for sec in secs]

# todo 如果arg_interable不是单一的参数 那就不好直接用 executor.map(do_something, arg_iterable)

# 会将每个arg_iterable的元组传给do_something, 办法是再写一个run_task，传入args元组，再返回do_something(*args) 不要写lambda，不能序列化

results = executor.map(run_task, arg_iterable)

# 打印的顺序并不是执行完的顺序 按开始的顺序打印的

for result in results:

print(result)

finish = time.perf_counter()

print(f"Finished in {round(finish - start, 2)} second(s)")

日志：

Sleeping 6 second(s)...

Sleeping 5 second(s)...

Sleeping 4 second(s)...

Sleeping 3 second(s)...

Sleeping 2 second(s)...

Done Sleep...5

Done Sleep...4

Done Sleep...3

Done Sleep...2

Done Sleep...1

Finished in 6.14 second(s)