Загрузка данных
import sys
import os
import time
import gc
import math
import ctypes
import subprocess
import statistics
import traceback
from pathlib import Path
from typing import Dict, Any, List, Tuple
import psutil
# --- Low-Level Windows API для замера тактов CPU ---
kernel32 = ctypes.windll.kernel32 if sys.platform == "win32" else None
ULONG64 = ctypes.c_uint64
def get_process_cycles(pid: int) -> int:
"""Возвращает точное количество аппаратных тактов CPU для процесса на Windows."""
if not kernel32:
return 0
PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
h_process = kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
if not h_process:
return 0
cycles = ULONG64()
success = kernel32.QueryProcessCycleTime(h_process, ctypes.byref(cycles))
kernel32.CloseHandle(h_process)
return cycles.value if success else 0
# --- Математический модуль: U-тест Манна — Уитни ---
def mann_whitney_u_test(sample1: List[float], sample2: List[float]) -> Tuple[float, float]:
"""Рассчитывает U-статистику и p-value для двух выборок без сторонних библиотек."""
n1, n2 = len(sample1), len(sample2)
if n1 < 3 or n2 < 3:
return 0.0, 1.0
ranked = sorted([(val, 1) for val in sample1] + [(val, 2) for val in sample2], key=lambda x: x[0])
r1 = sum(i + 1 for i, (val, group) in enumerate(ranked) if group == 1)
u1 = r1 - (n1 * (n1 + 1)) / 2
u2 = (n1 * n2) - u1
u = min(u1, u2)
mean_u = (n1 * n2) / 2
std_u = math.sqrt((n1 * n2 * (n1 + n2 + 1)) / 12)
if std_u == 0:
return u, 1.0
z = (u - mean_u) / std_u
p_value = 2 * (0.5 * (1 + math.erf(z / math.sqrt(2))))
return u, p_value
class AbsoluteBenchmark:
def __init__(self, script_path: str, iterations: int = 10, timeout_sec: float = 30.0):
self.script_path = Path(script_path)
self.iterations = iterations
self.timeout_sec = timeout_sec
if not self.script_path.exists():
raise FileNotFoundError(f"Файл не найден: {self.script_path}")
def _apply_isolation(self, process: psutil.Process):
try:
if sys.platform == "win32":
process.nice(psutil.HIGH_PRIORITY_CLASS)
cpus = process.cpu_affinity()
if cpus and len(cpus) > 1:
process.cpu_affinity([0])
except (psutil.AccessDenied, psutil.NoSuchProcess, AttributeError):
pass
def _run_single_pass(self, current_iter: int, is_warmup: bool = False, input_commands: List[str] = None, args: List[str] = None) -> Dict[str, Any]:
cmd = [sys.executable, str(self.script_path)] + (args or [])
env = os.environ.copy()
env["PYTHONIOENCODING"] = "utf-8"
env["PYTHONDONTWRITEBYTECODE"] = "1"
env["PYTHONUNBUFFERED"] = "1"
gc.collect()
gc.disable()
start_wall_ns = time.perf_counter_ns()
process = subprocess.Popen(
cmd,
stdin=subprocess.PIPE,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
env=env,
text=True
)
try:
p = psutil.Process(process.pid)
self._apply_isolation(p)
except psutil.NoSuchProcess:
gc.enable()
return {"wall_ns": 0, "cpu_cycles": 0, "max_rss_mb": 0.0, "exit_code": -1}
# Последовательный ввод команд
if input_commands and process.stdin:
try:
for cmd_item in input_commands:
process.stdin.write(cmd_item)
process.stdin.flush()
except (BrokenPipeError, OSError):
pass
rss_samples = []
while process.poll() is None:
elapsed_sec = (time.perf_counter_ns() - start_wall_ns) / 1e9
if elapsed_sec > self.timeout_sec:
process.kill()
gc.enable()
return {"wall_ns": 0, "cpu_cycles": 0, "max_rss_mb": 0.0, "exit_code": -2}
try:
mem = p.memory_info().rss
rss_samples.append(mem)
if not is_warmup:
status = f"\r [RUN] Прогон {current_iter}/{self.iterations} | Время: {elapsed_sec:.2f}s "
else:
status = f"\r [WARMUP] Холостой прогон... {elapsed_sec:.2f}s "
sys.stdout.write(status)
sys.stdout.flush()
except (psutil.NoSuchProcess, psutil.AccessDenied):
break
time.sleep(0.02)
end_wall_ns = time.perf_counter_ns()
wall_time_ns = end_wall_ns - start_wall_ns
# Захват ошибок stderr при ненулевом коде завершения
_, stderr_out = process.communicate()
if process.returncode != 0 and stderr_out:
sys.stdout.write(f"\n\n[ОШИБКА В {self.script_path.name} (Код: {process.returncode})]:\n{stderr_out.strip()}\n")
cpu_cycles = get_process_cycles(process.pid)
gc.enable()
max_rss_mb = (max(rss_samples) / (1024 * 1024)) if rss_samples else 0.0
return {
"wall_ns": wall_time_ns,
"cpu_cycles": cpu_cycles,
"max_rss_mb": max_rss_mb,
"exit_code": process.returncode
}
def execute(self, input_commands: List[str] = None, args: List[str] = None) -> Dict[str, Any]:
self._run_single_pass(current_iter=0, is_warmup=True, input_commands=input_commands, args=args)
time.sleep(0.5)
results = {"wall_sec": [], "cpu_cycles": [], "max_rss_mb": []}
for i in range(self.iterations):
m = self._run_single_pass(current_iter=i + 1, is_warmup=False, input_commands=input_commands, args=args)
if m["exit_code"] == 0:
results["wall_sec"].append(m["wall_ns"] / 1e9)
results["cpu_cycles"].append(m["cpu_cycles"])
results["max_rss_mb"].append(m["max_rss_mb"])
time.sleep(0.2)
sys.stdout.write("\r" + " " * 80 + "\r")
sys.stdout.flush()
if not results["wall_sec"]:
return {"error": "Ошибка выполнения скрипта"}
return results
def run_absolute_benchmark(script1: str, script2: str, input_commands: List[str] = None, iterations: int = 10):
print(f"=== АБСОЛЮТНЫЙ ДЕТЕРМИНИРОВАННЫЙ БЕНЧМАРК ({iterations} итераций) ===")
b1 = AbsoluteBenchmark(script1, iterations=iterations)
b2 = AbsoluteBenchmark(script2, iterations=iterations)
print(f"Тестирование: {script1}")
res1 = b1.execute(input_commands=input_commands)
print(f"Тестирование: {script2}")
res2 = b2.execute(input_commands=input_commands)
if "error" in res1 or "error" in res2:
print("\n[ОШИБКА] Бенчмарк прерван из-за сбоя в выполнении. См. лог ошибок выше.")
return
w1, w2 = res1["wall_sec"], res2["wall_sec"]
c1, c2 = res1["cpu_cycles"], res2["cpu_cycles"]
u_stat, p_val = mann_whitney_u_test(w1, w2)
med_w1, med_w2 = statistics.median(w1), statistics.median(w2)
med_c1, med_c2 = int(statistics.median(c1)), int(statistics.median(c2))
print("\n" + "="*75)
print(f"{'Метрика / Показатель':<32} | {Path(script1).name:<18} | {Path(script2).name:<18}")
print("="*75)
print(f"{'Wall Time: Медиана (сек)':<32} | {med_w1:<18.6f} | {med_w2:<18.6f}")
print(f"{'Wall Time: Мин (сек)':<32} | {min(w1):<18.6f} | {min(w2):<18.6f}")
print(f"{'CPU Cycles: Медиана (такты)':<32} | {med_c1:<18,d} | {med_c2:<18,d}")
print(f"{'CPU Cycles: Мин (такты)':<32} | {int(min(c1)):<18,d} | {int(min(c2)):<18,d}")
print(f"{'Пик RAM (МБ)':<32} | {max(res1['max_rss_mb']):<18.2f} | {max(res2['max_rss_mb']):<18.2f}")
print("="*75)
print(f"Математическая статистика (Mann-Whitney U-test):")
print(f" - U-Statistic: {u_stat:.2f}")
print(f" - p-value: {p_val:.5f}")
print("-" * 75)
if p_val < 0.05:
winner = script1 if med_w1 < med_w2 else script2
print(f"РЕЗУЛЬТАТ: Разница статистически значима (p < 0.05). Победитель: {winner}")
else:
print("РЕЗУЛЬТАТ: Разница находится внутри пределов случайного шума (p >= 0.05). Скрипты идентичны по производительности.")
print("="*75)
if __name__ == "__main__":
try:
SCRIPT_A = "script1.exe"
SCRIPT_B = "script2.exe"
# Команды ввода передаются последовательным списком
INPUT_COMMANDS = ["1\n", "n\n", "\n"]
run_absolute_benchmark(SCRIPT_A, SCRIPT_B, input_commands=INPUT_COMMANDS, iterations=10)
except Exception as e:
print("\n[КРИТИЧЕСКАЯ ОШИБКА ВЫПОЛНЕНИЯ]:")
traceback.print_exc()
finally:
input("\nНажмите Enter для выхода...")