You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
61 lines
1.6 KiB
61 lines
1.6 KiB
import pathlib as pl
|
|
import json
|
|
import importlib.machinery as impmach
|
|
import multiprocessing
|
|
import threading
|
|
import concurrent.futures as concfut
|
|
import os
|
|
|
|
from . import batch
|
|
from . import plan
|
|
|
|
def execute(exp_file):
|
|
dispatcher = load(exp_file)
|
|
dispatcher.start()
|
|
dispatcher.join()
|
|
|
|
def load(exp_file):
|
|
exp_plan = plan.Plan(exp_file, multiprocessing.Lock())
|
|
|
|
with open(exp_file) as efile:
|
|
exp_obj = json.loads(efile.read())
|
|
exp_obj["load"] = pl.Path(exp_obj["load"])
|
|
|
|
exp_mod = impmach.SourceFileLoader(exp_obj["load"].stem,
|
|
str(exp_obj["load"])).load_module()
|
|
|
|
return Dispatcher(exp_mod.run, exp_plan, os.cpu_count())
|
|
|
|
class Dispatcher (threading.Thread):
|
|
def __init__(self, exp_func, exp_plan, num_workers):
|
|
threading.Thread.__init__(self)
|
|
|
|
self.__exp_func = exp_func
|
|
self.__plan = exp_plan
|
|
|
|
self.__num_workers = num_workers
|
|
self.__workers = []
|
|
|
|
for i in range(self.__num_workers):
|
|
self.__workers.append(multiprocessing.Process(target=self.__run_exp,
|
|
args=(self.__exp_func,
|
|
self.__plan)))
|
|
|
|
def run(self):
|
|
for worker in self.__workers:
|
|
worker.start()
|
|
|
|
for worker in self.__workers:
|
|
worker.join()
|
|
|
|
@staticmethod
|
|
def __run_exp(exp_func, exp_plan):
|
|
instance = exp_plan.next()
|
|
|
|
while instance != None:
|
|
exp_func(instance)
|
|
|
|
exp_plan.done_with(instance)
|
|
|
|
instance = exp_plan.next()
|
|
|