Complete VIDEO-001 through VIDEO-004 software video studies
This commit is contained in:
LittleSam129
274
tools/video_gate.py
Normal file
274
tools/video_gate.py
Normal file
@@ -0,0 +1,274 @@
|
||||
"""VIDEO-001/002/003: проверки без данных аппаратных лабораторий и без RF."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import av
|
||||
import numpy as np
|
||||
|
||||
from tools.video003_design import ROOT, run as calculate_design
|
||||
|
||||
|
||||
MODULES = (
|
||||
"protocol.packet", "protocol.bpsk_radio", "protocol.convolutional_fec",
|
||||
"protocol.video_stream_transport", "protocol.punctured_fec",
|
||||
"protocol.video003_modem", "protocol.video003_compact_header",
|
||||
"protocol.video004_viterbi", "protocol.video004_erasure",
|
||||
"tools.video_compare", "tools.video002_radio_budget",
|
||||
"tools.video002_simulation", "tools.video003_design",
|
||||
"tools.video003_synthetic", "tools.video003_stream",
|
||||
"tools.video003_codec", "tools.video003_transport",
|
||||
"tools.video004_benchmark", "tools.video004_transport",
|
||||
"tools.video004_interleaving",
|
||||
"tools.video004_viterbi_benchmark", "tools.video004_longrun",
|
||||
"tools.video004_pipeline_benchmark",
|
||||
"tools.video004_keyframe", "tools.video004_seed_sweep",
|
||||
)
|
||||
TESTS = (
|
||||
"tests/test_packet.py", "tests/test_bpsk_radio.py",
|
||||
"tests/test_video_001_compare.py", "tests/test_video002_codec.py",
|
||||
"tests/test_video002_transport.py", "tests/test_video003_modem.py",
|
||||
"tests/test_video003_design.py", "tests/test_video003_transport.py",
|
||||
"tests/test_video003_stream.py",
|
||||
"tests/test_video004_viterbi.py", "tests/test_video004_erasure.py",
|
||||
"tests/test_video004_transport.py",
|
||||
"tests/test_video004_pipeline.py",
|
||||
"tests/test_video004_keyframe.py",
|
||||
)
|
||||
ARTIFACTS = {
|
||||
"video001_160": "data/processed/video001/comparison_traffic_160k.json",
|
||||
"video001_320": "data/processed/video001/comparison_traffic_320k.json",
|
||||
"video002": "data/processed/video002/results.json",
|
||||
"design": "data/processed/video003/design_space.json",
|
||||
"synthetic": "data/processed/video003/synthetic_phy.json",
|
||||
"stream": "data/processed/video003/continuous_stream.json",
|
||||
"codec": "data/processed/video003/codec_results.json",
|
||||
"transport": "data/processed/video003/transport_results.json",
|
||||
"benchmark_manifest": "data/processed/video004/canonical_manifest.json",
|
||||
"benchmark_before": "data/processed/video004/benchmark_before.json",
|
||||
"benchmark_after": "data/processed/video004/benchmark_after.json",
|
||||
"transport004": "data/processed/video004/transport_results.json",
|
||||
"interleaving004": "data/processed/video004/interleaving_results.json",
|
||||
"viterbi004": "data/processed/video004/viterbi_benchmark.json",
|
||||
"long004": "data/processed/video004/benchmark_256.json",
|
||||
"pipeline_manifest004": "data/processed/video004/pipeline_manifest.json",
|
||||
"pipeline004": "data/processed/video004/pipeline_benchmark.json",
|
||||
"keyframe004": "data/processed/video004/keyframe_recovery.json",
|
||||
"seed_sweep004": "data/processed/video004/normal_seed_sweep.json",
|
||||
}
|
||||
FORBIDDEN_IMPORT_ROOTS = {"iio", "adi", "SoapySDR", "hackrf", "pylibiio", "pyhackrf"}
|
||||
|
||||
|
||||
def _check_imports_and_hardware_boundary() -> None:
|
||||
for name in MODULES:
|
||||
module = importlib.import_module(name)
|
||||
source_path = Path(module.__file__).resolve()
|
||||
if not source_path.is_relative_to(ROOT.resolve()):
|
||||
raise AssertionError(f"локальный модуль вне worktree: {name}")
|
||||
tree = ast.parse(source_path.read_text(encoding="utf-8"), filename=str(source_path))
|
||||
for node in ast.walk(tree):
|
||||
imported = []
|
||||
if isinstance(node, ast.Import):
|
||||
imported = [alias.name for alias in node.names]
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
imported = [node.module]
|
||||
for imported_name in imported:
|
||||
if imported_name.split(".")[0] in FORBIDDEN_IMPORT_ROOTS:
|
||||
raise AssertionError(f"аппаратный импорт {imported_name} в {name}")
|
||||
|
||||
|
||||
def _load_artifacts() -> dict[str, dict]:
|
||||
documents = {}
|
||||
for label, relative in ARTIFACTS.items():
|
||||
path = ROOT / relative
|
||||
if not path.is_file():
|
||||
raise AssertionError(f"нет результата: {relative}")
|
||||
documents[label] = json.loads(path.read_text(encoding="utf-8"))
|
||||
source = ROOT / "data/raw/video001/udacity_project_video.mp4"
|
||||
digest = hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
for label in ("video001_160", "video001_320", "video002", "codec", "transport"):
|
||||
if documents[label]["source_sha256"] != digest:
|
||||
raise AssertionError(f"SHA исходного ролика не совпал: {label}")
|
||||
return documents
|
||||
|
||||
|
||||
def _check_results(documents: dict[str, dict]) -> None:
|
||||
design = documents["design"]
|
||||
if design != calculate_design():
|
||||
raise AssertionError("расчётная таблица отличается от сохранённой")
|
||||
if design["best_calculated"]["useful_video_rate_bps"] >= 600_000:
|
||||
raise AssertionError("таблица неверно объявила достижение 600 кбит/с")
|
||||
synthetic = documents["synthetic"]
|
||||
if len(synthetic["records"]) != 7 * 4 * synthetic["trials_per_point"]:
|
||||
raise AssertionError("число реализаций AWGN не совпадает")
|
||||
if len(synthetic["summary"]) != 28:
|
||||
raise AssertionError("неполная сводка AWGN")
|
||||
if not all(item["packet_crc_pass"] and item["packet_exact"]
|
||||
for item in synthetic["rrc_controls"]):
|
||||
raise AssertionError("RRC-контроль не прошёл")
|
||||
if not all(item["crc_pass_count"] == item["trials"]
|
||||
for item in synthetic["summary"] if item["ebn0_db"] == 9.0):
|
||||
raise AssertionError("эталонный AWGN-контроль 9 dB изменился")
|
||||
stream = documents["stream"]
|
||||
if len(stream["results"]) != 2 or stream["packet_count"] != 32:
|
||||
raise AssertionError("неполный непрерывный поток")
|
||||
for item in stream["results"]:
|
||||
if (item["crc_pass_count"] != 32 or item["exact_packet_count"] != 32
|
||||
or item["verified_video_fragment_count"] != 32
|
||||
or abs(item["offline_verified_video_bps"]
|
||||
- item["calculated_zero_loss_video_bps"]) > 1e-6):
|
||||
raise AssertionError("непрерывный поток не совпал с расчётом")
|
||||
codec = documents["codec"]
|
||||
if len(codec["cases"]) != 60 or av.__version__ != "16.0.1":
|
||||
raise AssertionError("неполный кодековый опыт или иная версия PyAV")
|
||||
for case in codec["cases"]:
|
||||
settings = case["settings"]
|
||||
if (settings["ffmpeg_cli_used"] or settings["profile_actual"] is None
|
||||
or settings["codec_name"] != "libx264"
|
||||
or case["keyframe_positions"][0] != 0
|
||||
or case["frame_count"] != len(case["frame_sizes_bytes"])):
|
||||
raise AssertionError("неполная конфигурация или размеры H.264")
|
||||
calculated_average = sum(case["frame_sizes_bytes"]) * 8 * case["settings"]["frame_rate"] / case["frame_count"]
|
||||
if abs(calculated_average - case["actual_average_bitrate_bps"]) > 1e-6:
|
||||
raise AssertionError("средний поток H.264 не совпал с размерами кадров")
|
||||
for case in (item for item in codec["cases"] if item["profile"] == "CURRENT"):
|
||||
originals = [item for item in documents["video001_320"]["cases"]
|
||||
if item["window"] == case["window"]
|
||||
and item["resolution"] == case["settings"]["resolution"]
|
||||
and item["fps"] == case["settings"]["frame_rate"]
|
||||
and item["codec"] == "H264_LIBX264"]
|
||||
if len(originals) != 1 or case["frame_sizes_bytes"] != originals[0]["frame_sizes_bytes"]:
|
||||
raise AssertionError("CURRENT не повторил покадровые размеры VIDEO-001")
|
||||
transport = documents["transport"]
|
||||
if len(transport["cases"]) != 784:
|
||||
raise AssertionError("неполная матрица доставки")
|
||||
for case in transport["cases"]:
|
||||
if not case["no_old_frame_replay"]:
|
||||
raise AssertionError("повторно показан старый кадр")
|
||||
if case["p95_end_to_end_age_ms"] is not None:
|
||||
if case["p95_end_to_end_age_ms"] > case["max_end_to_end_age_ms"]:
|
||||
raise AssertionError("P95 возраста больше максимума")
|
||||
selected_clean = [case for case in transport["cases"]
|
||||
if case["codec_profile"] == "SHORT_GOP_400K"
|
||||
and case["link"] == "QPSK_500_R23_1012_0MS"
|
||||
and case["scenario"] == "random" and case["level"] == 0]
|
||||
if len(selected_clean) != 6 or any(case["displayed_frames"] != case["input_frames"]
|
||||
for case in selected_clean):
|
||||
raise AssertionError("контрольный видеотракт не показал все кадры без потерь")
|
||||
if any(documents[label]["rf_tx"] != 0
|
||||
for label in ("video002", "design", "synthetic", "stream", "codec", "transport",
|
||||
"benchmark_manifest", "benchmark_before", "benchmark_after",
|
||||
"transport004", "interleaving004", "viterbi004", "long004",
|
||||
"pipeline_manifest004", "pipeline004", "keyframe004",
|
||||
"seed_sweep004")):
|
||||
raise AssertionError("в результатах найден ненулевой RF TX")
|
||||
|
||||
from tools.video004_benchmark import load_fixture, run as benchmark_run
|
||||
_, manifest = load_fixture()
|
||||
before, after = documents["benchmark_before"], documents["benchmark_after"]
|
||||
if (before["signal_sha256"] != after["signal_sha256"]
|
||||
or before["signal_sha256"] != manifest["signal_sha256"]
|
||||
or before["outcomes"] != after["outcomes"]):
|
||||
raise AssertionError("эталонные выходы приёмника до/после изменились")
|
||||
measured = benchmark_run("after", repetitions=3)
|
||||
if (measured["outcomes"] != before["outcomes"]
|
||||
or measured["available_path_realtime_factor"] > 0.67):
|
||||
raise AssertionError("производительность или строгий приём VIDEO-004 изменились")
|
||||
protected = documents["transport004"]
|
||||
interleaving = documents["interleaving004"]
|
||||
if (len(protected["cases"]) != 1440 or len(interleaving["cases"]) != 48
|
||||
or protected["source_sha256"] != documents["codec"]["source_sha256"]
|
||||
or interleaving["source_sha256"] != documents["codec"]["source_sha256"]):
|
||||
raise AssertionError("неполная матрица внешнего FEC")
|
||||
if any(not case["no_old_frame_replay"] for case in
|
||||
protected["cases"] + interleaving["cases"]):
|
||||
raise AssertionError("повторно показан старый кадр VIDEO-004")
|
||||
if any(case["max_frame_age_ms"] is not None and case["max_frame_age_ms"] >= 300
|
||||
for case in protected["cases"] + interleaving["cases"]):
|
||||
raise AssertionError("показан просроченный кадр VIDEO-004")
|
||||
selected = [case for case in protected["cases"]
|
||||
if case["mode"] == "NORMAL" and case["fec_rate"] == "2/3"
|
||||
and case["data_count"] == 8 and case["parity_count"] == 2
|
||||
and not case["keyframe_boost"] and case["scenario"] == "random"]
|
||||
if len(selected) != 12 or sum(case["displayed_frames"] for case in selected
|
||||
if case["level"] == 0) != 120:
|
||||
raise AssertionError("выбранный внешний FEC теряет кадры без потерь канала")
|
||||
viterbi = documents["viterbi004"]
|
||||
long_run = documents["long004"]
|
||||
pipeline = documents["pipeline004"]
|
||||
if (not viterbi["bits_equal"] or viterbi["signal_sha256"] != manifest["signal_sha256"]
|
||||
or not long_run["strict_outputs_equal"] or long_run["packet_count"] != 256
|
||||
or not pipeline["exact_packets_and_frames"] or pipeline["parity_recoveries"] != 4
|
||||
or pipeline["numba"]["realtime_factor"] > 0.67):
|
||||
raise AssertionError("длинный или полный программный тракт VIDEO-004 не прошёл")
|
||||
from tools.video004_pipeline_benchmark import SIGNAL, decode
|
||||
samples = np.load(SIGNAL, allow_pickle=False)
|
||||
if hashlib.sha256(samples.tobytes()).hexdigest() != documents["pipeline_manifest004"]["signal_sha256"]:
|
||||
raise AssertionError("контрольный сигнал внешнего FEC изменился")
|
||||
started = time.perf_counter()
|
||||
current = decode(samples, "numba")
|
||||
elapsed = time.perf_counter() - started
|
||||
if (current["packet_sha256"] != documents["pipeline_manifest004"]["expected_packet_sha256"]
|
||||
or current["frame_sha256"] != documents["pipeline_manifest004"]["expected_frame_sha256"]
|
||||
or elapsed / pipeline["signal_duration_s"] > 0.67):
|
||||
raise AssertionError("полный доступный программный тракт изменился")
|
||||
keyframe = documents["keyframe004"]
|
||||
sweep = documents["seed_sweep004"]
|
||||
if (len(keyframe["cases"]) != 24 or len(sweep["cases"]) != 180
|
||||
or sweep["seeds"] != list(range(20260916, 20260921))):
|
||||
raise AssertionError("неполная проверка опорного кадра или затравок")
|
||||
base_double = [case for case in keyframe["cases"] if case["fec_rate"] == "2/3"
|
||||
and case["data_count"] == 8 and case["parity_count"] == 1
|
||||
and not case["keyframe_boost"] and case["scenario"] == "keyframe_double"]
|
||||
boosted_double = [case for case in keyframe["cases"] if case["fec_rate"] == "2/3"
|
||||
and case["data_count"] == 8 and case["parity_count"] == 1
|
||||
and case["keyframe_boost"] and case["scenario"] == "keyframe_double"]
|
||||
if (len(base_double) != 2 or len(boosted_double) != 2
|
||||
or sum(x["displayed_frames"] for x in base_double) != 110
|
||||
or sum(x["displayed_frames"] for x in boosted_double) != 120):
|
||||
raise AssertionError("защита опорного кадра изменила результат")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--numba-packages", type=Path)
|
||||
args = parser.parse_args()
|
||||
if args.numba_packages is not None:
|
||||
if not args.numba_packages.is_dir():
|
||||
parser.error("каталог локальных пакетов Numba отсутствует")
|
||||
sys.path.insert(0, str(args.numba_packages.resolve()))
|
||||
_check_imports_and_hardware_boundary()
|
||||
documents = _load_artifacts()
|
||||
_check_results(documents)
|
||||
output_root = (ROOT / "data/processed/video003").resolve()
|
||||
if not output_root.is_relative_to(ROOT.resolve()):
|
||||
raise AssertionError("временный каталог вне worktree")
|
||||
environment = os.environ.copy()
|
||||
environment["PYTHONDONTWRITEBYTECODE"] = "1"
|
||||
if args.numba_packages is not None:
|
||||
environment["PYTHONPATH"] = str(args.numba_packages.resolve()) + os.pathsep + environment.get("PYTHONPATH", "")
|
||||
with tempfile.TemporaryDirectory(prefix="video_gate_pytest_", dir=output_root) as name:
|
||||
temporary = Path(name).resolve()
|
||||
if not temporary.is_relative_to(output_root):
|
||||
raise AssertionError("временный каталог вне результата VIDEO-003")
|
||||
command = [sys.executable, "-B", "-m", "pytest", *TESTS,
|
||||
"-q", "-p", "no:cacheprovider", "--basetemp", str(temporary)]
|
||||
completed = subprocess.run(command, cwd=ROOT, env=environment, check=False)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(completed.returncode)
|
||||
print("VIDEO-SPECIFIC GATE: PASS; результаты, импорты, аппаратная граница и тесты")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user