60cce66993
Tests / test (push) Has been cancelled
导入上游源码,为所有面向用户的品牌显示处添加「增强版 / Enhanced」标识;包含 unitypackage→VRM 的纯色 Toon 材质、兽耳 SpringBone、Unity 描边、UV 接缝白边补偿修复,以及默认不强制 T-pose。146 个单元测试通过。
241 lines
10 KiB
Python
241 lines
10 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""多进程并行转换:几个文件(或一个包里的几个模型)同时转。
|
||
|
||
转换是纯 Python、吃 CPU 的活,线程受 GIL 限制快不了,所以开子进程(spawn)。每个任务的日志经队列送回主进程,
|
||
交给这个任务自己的 log 回调(界面按文件分开显示);任务结束的回调在主进程里、并且在它的日志都送完之后调用。
|
||
|
||
results = parallel.run([(key, func, args, kwargs), ...], jobs, log_for, start=None, done=None, should_stop=None)
|
||
|
||
func 必须是模块顶层函数(子进程要能按名字导入),调用方式是 func(*args, log=..., **kwargs)。
|
||
子进程里不再开并行(in_worker()),免得「4 个文件 × 每个包 4 个模型」叠出 16 个进程。
|
||
单个转换内部贴图解码用的线程数见 threads()。
|
||
打包的 exe 要在入口调用 multiprocessing.freeze_support()(ui/app.py),否则子进程会再开一个程序。
|
||
"""
|
||
import concurrent.futures as cf
|
||
import functools
|
||
import multiprocessing as mp
|
||
import os
|
||
import queue
|
||
import threading
|
||
import traceback
|
||
|
||
WORKER_ENV = "VRMIFY_WORKER"
|
||
MAX_AUTO = 8 # 自动时最多同时跑几个:再多也快不了多少(内存带宽、硬盘、进程启动),还多占内存
|
||
GB_PER_JOB = 1.5 # 不知道任务多大时按这么多算(十几万顶点的模型转换时进程峰值约 0.5 GB)
|
||
MEM_SHARE = 0.8 # 并行的任务加起来最多占可用内存的这么多
|
||
|
||
|
||
def in_worker():
|
||
return os.environ.get(WORKER_ENV) == "1"
|
||
|
||
|
||
def free_memory_gb():
|
||
"""可用物理内存(GB);拿不到时按 8 GB 算。"""
|
||
try:
|
||
if os.name == "nt":
|
||
import ctypes
|
||
|
||
class MS(ctypes.Structure):
|
||
_fields_ = [("dwLength", ctypes.c_ulong), ("dwMemoryLoad", ctypes.c_ulong),
|
||
("ullTotalPhys", ctypes.c_ulonglong), ("ullAvailPhys", ctypes.c_ulonglong),
|
||
("ullTotalPageFile", ctypes.c_ulonglong), ("ullAvailPageFile", ctypes.c_ulonglong),
|
||
("ullTotalVirtual", ctypes.c_ulonglong), ("ullAvailVirtual", ctypes.c_ulonglong),
|
||
("ullAvailExtendedVirtual", ctypes.c_ulonglong)]
|
||
m = MS()
|
||
m.dwLength = ctypes.sizeof(m)
|
||
if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(m)):
|
||
return m.ullAvailPhys / 1024 ** 3
|
||
else:
|
||
return os.sysconf("SC_AVPHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") / 1024 ** 3
|
||
except (OSError, ValueError, AttributeError):
|
||
pass
|
||
return 8.0
|
||
|
||
|
||
@functools.lru_cache(maxsize=None)
|
||
def cores():
|
||
"""物理核数(超线程的两个逻辑核算一个);拿不到时按逻辑核数。"""
|
||
n = 0
|
||
try:
|
||
if os.name == "nt":
|
||
import ctypes
|
||
|
||
class INFO(ctypes.Structure): # SYSTEM_LOGICAL_PROCESSOR_INFORMATION
|
||
_fields_ = [("mask", ctypes.c_size_t), ("relationship", ctypes.c_int),
|
||
("_", ctypes.c_ulonglong * 2)]
|
||
k32, size = ctypes.windll.kernel32, ctypes.c_ulong(0)
|
||
k32.GetLogicalProcessorInformation(None, ctypes.byref(size))
|
||
buf = (INFO * (size.value // ctypes.sizeof(INFO)))()
|
||
if k32.GetLogicalProcessorInformation(buf, ctypes.byref(size)):
|
||
n = sum(1 for i in buf if i.relationship == 0) # RelationProcessorCore
|
||
elif os.path.exists("/proc/cpuinfo"):
|
||
seen, phys = set(), "0"
|
||
with open("/proc/cpuinfo", encoding="utf-8", errors="replace") as f:
|
||
for line in f:
|
||
key, _, val = line.partition(":")
|
||
if key.strip() == "physical id":
|
||
phys = val.strip()
|
||
elif key.strip() == "core id":
|
||
seen.add((phys, val.strip()))
|
||
n = len(seen)
|
||
except (OSError, ValueError, AttributeError):
|
||
n = 0
|
||
return n or os.cpu_count() or 1
|
||
|
||
|
||
def threads():
|
||
"""一个转换内部同时处理几张贴图(Pillow 在 C 里解码 / 编码、不占 GIL,线程就够):最多 4 个;
|
||
在并行转换的子进程里最多 2 个(核和内存已经被别的子进程占着,一张 4K 贴图解开约 64 MB)。"""
|
||
return max(1, min(2 if in_worker() else 4, os.cpu_count() or 1))
|
||
|
||
|
||
def auto_jobs():
|
||
"""自动时最多同时跑几个(还没看内存):物理核数 − 1(留一个给界面和系统),最多 MAX_AUTO。
|
||
8 核 16 线程上 20 个包:4 个 11.6 秒、8 个 9.1 秒、16 个 8.3 秒——超过物理核数快不了多少。"""
|
||
return max(1, min(MAX_AUTO, cores() - 1))
|
||
|
||
|
||
def jobs_for(n_tasks, setting=0, mem_gb=None):
|
||
"""同时跑几个:setting > 0 时按设置;0 = 自动(auto_jobs,再看可用内存)。
|
||
|
||
mem_gb:每个任务的内存峰值估计(GB,见 pipeline.memory_estimate)。给了就取「最大的几个加起来
|
||
不超过可用内存的 MEM_SHARE」——同时在跑的任意几个都不会超;不给按每个 GB_PER_JOB 算。
|
||
"""
|
||
if n_tasks <= 1 or in_worker():
|
||
return 1
|
||
try:
|
||
setting = int(setting or 0)
|
||
except (TypeError, ValueError):
|
||
setting = 0
|
||
if setting > 0:
|
||
return min(setting, n_tasks)
|
||
cap = max(1, min(auto_jobs(), n_tasks))
|
||
free = free_memory_gb()
|
||
if not mem_gb:
|
||
return max(1, min(cap, int(free // GB_PER_JOB)))
|
||
k, tot = 0, 0.0
|
||
for m in sorted(mem_gb, reverse=True)[:cap]:
|
||
tot += m
|
||
if tot > free * MEM_SHARE:
|
||
break
|
||
k += 1
|
||
return max(1, k)
|
||
|
||
|
||
_QUEUE = None
|
||
|
||
|
||
def _init(q):
|
||
global _QUEUE
|
||
_QUEUE = q
|
||
os.environ[WORKER_ENV] = "1"
|
||
_lower_priority()
|
||
|
||
|
||
def _lower_priority():
|
||
"""子进程降到「低于正常」:电脑空闲时照样用满 CPU,用户同时在用别的程序(或这个界面)时先让给它们。"""
|
||
try:
|
||
if os.name == "nt":
|
||
import ctypes
|
||
k32 = ctypes.windll.kernel32
|
||
k32.SetPriorityClass(k32.GetCurrentProcess(), 0x4000) # BELOW_NORMAL_PRIORITY_CLASS
|
||
else:
|
||
os.nice(5)
|
||
except (OSError, AttributeError):
|
||
pass
|
||
|
||
|
||
def _call(key, func, args, kwargs):
|
||
def log(msg, tag=None):
|
||
_QUEUE.put((key, str(msg), tag))
|
||
try:
|
||
out = (True, func(*args, log=log, **kwargs))
|
||
except Exception as e: # noqa: BLE001 一个任务失败不影响其它
|
||
out = (False, ("%s: %s" % (type(e).__name__, e), traceback.format_exc()))
|
||
_QUEUE.put((key, None, "__done__")) # 这个任务的日志到此为止
|
||
return out
|
||
|
||
|
||
def run(tasks, jobs, log_for, start=None, done=None, should_stop=None):
|
||
"""→ {key: (成功?, 结果 或 (错误摘要, 完整 traceback))}。should_stop() 为真后不再开始新任务。"""
|
||
ctx = mp.get_context("spawn")
|
||
q = ctx.Queue()
|
||
finished = {key: threading.Event() for key, *_ in tasks}
|
||
stop_pump = threading.Event()
|
||
|
||
def pump():
|
||
while not (stop_pump.is_set() and q.empty()):
|
||
try:
|
||
key, msg, tag = q.get(timeout=0.2)
|
||
except queue.Empty:
|
||
continue
|
||
if tag == "__stop__": # 收尾时放进来的,不用再等 0.2 秒超时
|
||
break
|
||
if tag == "__done__":
|
||
finished[key].set()
|
||
else:
|
||
log_for(key)(msg, tag)
|
||
|
||
th = threading.Thread(target=pump, daemon=True)
|
||
th.start()
|
||
by_key = {t[0]: t for t in tasks}
|
||
results, pending, running, started = {}, list(tasks), {}, set()
|
||
broken = False
|
||
try:
|
||
with cf.ProcessPoolExecutor(max_workers=jobs, mp_context=ctx, initializer=_init, initargs=(q,)) as ex:
|
||
while (pending or running) and not broken:
|
||
while pending and len(running) < jobs and not (should_stop and should_stop()):
|
||
key, func, args, kwargs = pending.pop(0)
|
||
if start and key not in started:
|
||
start(key)
|
||
started.add(key)
|
||
try:
|
||
running[ex.submit(_call, key, func, args, kwargs)] = key
|
||
except cf.process.BrokenProcessPool:
|
||
pending.insert(0, by_key[key])
|
||
broken = True
|
||
break
|
||
if broken or not running:
|
||
break # 停止了:剩下的不开始
|
||
fin, _ = cf.wait(running, return_when=cf.FIRST_COMPLETED)
|
||
for f in fin:
|
||
key = running.pop(f)
|
||
try:
|
||
res = f.result()
|
||
except cf.process.BrokenProcessPool:
|
||
# 子进程开不起来 / 被杀(杀毒软件拦截、内存不够):这个任务和剩下的都在本进程里转
|
||
pending.insert(0, by_key[key])
|
||
broken = True
|
||
continue
|
||
finished[key].wait(10) # 先把它的日志送完
|
||
results[key] = res
|
||
if done:
|
||
done(key, *res)
|
||
if broken:
|
||
pending = [by_key[k] for k in running.values()] + pending
|
||
running.clear()
|
||
except cf.process.BrokenProcessPool:
|
||
broken = True
|
||
finally:
|
||
stop_pump.set()
|
||
q.put((None, None, "__stop__")) # 子进程都已退出,它们的日志都在这之前
|
||
th.join(5)
|
||
if broken:
|
||
pending = [t for t in pending if t[0] not in results]
|
||
for key, func, args, kwargs in pending:
|
||
if should_stop and should_stop():
|
||
break
|
||
log = log_for(key)
|
||
if key == pending[0][0]:
|
||
log("无法启动子进程,改为逐个转换", "warn")
|
||
if start and key not in started:
|
||
start(key)
|
||
try:
|
||
res = (True, func(*args, log=log, **kwargs))
|
||
except Exception as e: # noqa: BLE001
|
||
res = (False, ("%s: %s" % (type(e).__name__, e), traceback.format_exc()))
|
||
results[key] = res
|
||
if done:
|
||
done(key, *res)
|
||
return results
|