import pathlib as pl import json import importlib.machinery as impmach import multiprocessing import threading import concurrent.futures as concfut import os from . import batch from . import plan def execute(exp_file): dispatcher = load(exp_file) dispatcher.start() dispatcher.join() def load(exp_file): exp_plan = plan.Plan(exp_file, multiprocessing.Lock()) with open(exp_file) as efile: exp_obj = json.loads(efile.read()) exp_obj["load"] = pl.Path(exp_obj["load"]) exp_mod = impmach.SourceFileLoader(exp_obj["load"].stem, str(exp_obj["load"])).load_module() return Dispatcher(exp_mod.run, exp_plan, os.cpu_count()) class Dispatcher (threading.Thread): def __init__(self, exp_func, exp_plan, num_workers): threading.Thread.__init__(self) self.__exp_func = exp_func self.__plan = exp_plan self.__num_workers = num_workers self.__workers = [] for i in range(self.__num_workers): self.__workers.append(multiprocessing.Process(target=self.__run_exp, args=(self.__exp_func, self.__plan))) def run(self): for worker in self.__workers: worker.start() for worker in self.__workers: worker.join() @staticmethod def __run_exp(exp_func, exp_plan): instance = exp_plan.next() while instance != None: exp_func(instance) exp_plan.done_with(instance) instance = exp_plan.next()