feat(builder): use multiprocessing instead of threading

This commit is contained in:
Haoyu Xu
2023-01-02 07:02:17 -05:00
parent 5d7cbc5df4
commit 60ff6a9f24

View File

@@ -1,4 +1,4 @@
import threading from multiprocessing import Process, Manager
import shutil import shutil
from lib.alpha_composite import AlphaComposite from lib.alpha_composite import AlphaComposite
from lib.atlas_reader import AtlasReader from lib.atlas_reader import AtlasReader
@@ -29,19 +29,17 @@ class Builder:
dict( dict(
target=self.__build_assets, target=self.__build_assets,
args=(operator_name,), args=(operator_name,),
daemon=True,
), ),
dict( dict(
target=self.__build_settings, target=self.__build_settings,
args=(operator_name,), args=(operator_name,),
daemon=True,
) )
] ]
threads = list() threads = list()
self.content_processor = ContentProcessor(self.config, operator_name) self.content_processor = ContentProcessor(self.config, operator_name)
for item in thread_map: for item in thread_map:
thread = threading.Thread(**item) thread = Process(**item)
threads.append(thread) threads.append(thread)
thread.start() thread.start()
@@ -75,7 +73,8 @@ class Builder:
print("Building operator data for {}...".format(operator_name)) print("Building operator data for {}...".format(operator_name))
prefix = "window.operatorAssets = " prefix = "window.operatorAssets = "
data = dict() manager = Manager()
data = manager.dict()
thread_map = [ thread_map = [
dict( dict(
@@ -84,7 +83,6 @@ class Builder:
file_paths, file_paths,
data data
), ),
daemon=True,
), ),
dict( dict(
target=self.__skeleton_binary_thread, target=self.__skeleton_binary_thread,
@@ -92,7 +90,6 @@ class Builder:
file_paths, file_paths,
data data
), ),
daemon=True,
), ),
dict( dict(
target=self.__fallback_thread, target=self.__fallback_thread,
@@ -100,12 +97,11 @@ class Builder:
file_paths, file_paths,
data data
), ),
daemon=True,
), ),
] ]
threads = list() threads = list()
for item in thread_map: for item in thread_map:
thread = threading.Thread(**item) thread = Process(**item)
threads.append(thread) threads.append(thread)
thread.start() thread.start()
@@ -134,22 +130,20 @@ class Builder:
alpha_composite_threads = list() alpha_composite_threads = list()
ar = AtlasReader(source_path + common_name, target_path + common_name) ar = AtlasReader(source_path + common_name, target_path + common_name)
images = ar.get_images() images = ar.get_images()
atlas_to_base64_thread = threading.Thread( atlas_to_base64_thread = Process(
target=self.__atlas_to_base64, target=self.__atlas_to_base64,
args=( args=(
target_path + common_name + ".atlas", target_path + common_name + ".atlas",
data, data,
self.config["server"]["operator_folder"] + common_name + ".atlas", self.config["server"]["operator_folder"] + common_name + ".atlas",
), ),
daemon=True,
) )
atlas_to_base64_thread.start() atlas_to_base64_thread.start()
for item in images: for item in images:
alpha_composite_thread = threading.Thread( alpha_composite_thread = Process(
target=AlphaComposite, target=AlphaComposite,
args=(source_path + item, target_path + item), args=(source_path + item, target_path + item),
daemon=True,
) )
alpha_composite_threads.append(alpha_composite_thread) alpha_composite_threads.append(alpha_composite_thread)
alpha_composite_thread.start() alpha_composite_thread.start()
@@ -157,14 +151,13 @@ class Builder:
thread.join() thread.join()
for item in images: for item in images:
png_to_base64_thread = threading.Thread( png_to_base64_thread = Process(
target=self.__png_to_base64, target=self.__png_to_base64,
args=( args=(
target_path + item, target_path + item,
data, data,
self.config["server"]["operator_folder"] + item, self.config["server"]["operator_folder"] + item,
), ),
daemon=True,
) )
png_to_base64_threads.append(png_to_base64_thread) png_to_base64_threads.append(png_to_base64_thread)