简单说明：
一个可执行文件读取配置文件（非常多）在两秒内返回结果，即便修改 _work_queue 还是会因为无法关闭相应子进程导致内存溢出。
import shlex
from concurrent.futures import ThreadPoolExecutor
class BoundedThreadPoolExecutor(ThreadPoolExecutor):
def __init__(self, max_workers, max_waiting_tasks, *args, **kwargs):
super().__init__(max_workers=max_workers, *args, **kwargs)
self._work_queue = Queue(maxsize=max_waiting_tasks)
def func(conf_path):
try:
command = f"../some.exe -d -c {conf_path}"
args = shlex.split(command)
proc = subprocess.Popen(args=args, shell=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
stdout, stderr = proc.communicate(5)
except subprocess.TimeoutExpired:
logger.error(f"pid:{proc.pid} communicate timeout")
finally:
proc.terminate()
return proc
def callback2kill(res):
proc = res.result()
os.system(f"taskkill /F /pid {proc.pid}")
with BoundedThreadPoolExecutor(100, 500) as executor:
for endpoint in endpoints:
future = executor.submit(func, endpoint)
future.add_done_callback(callback2kill)
