Lab051: accelerate 500 ksymbol/s streaming receiver

experiments/lab051_receiver_profile.py

@@ -0,0 +1,246 @@
+"""Lab051: воспроизводимое профилирование приёмника, без оборудования.
+
+Измерение времени отделено от медленного наблюдения массивов/tracemalloc.
+Классификация valid/false происходит только после штатного PHY/FEC/CRC.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from contextlib import contextmanager
+from dataclasses import asdict, dataclass
+import ctypes
+from ctypes import wintypes
+import json
+from pathlib import Path
+import statistics
+import sys
+import time
+import tracemalloc
+import weakref
+
+if __package__ in (None, ""):
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import numpy as np
+from PIL import Image
+
+from experiments import lab046c_offline_recovery as offline
+from experiments import lab050_video_stream as video
+from experiments.lab050_streaming_receiver import streaming_mode
+
+
+ROOT = Path(__file__).resolve().parents[1]
+OUTPUT = ROOT / "data/processed/lab051"
+lab045 = video.phy.lab045
+lab043 = lab045.lab044.lab043
+
+
+def process_memory() -> dict:
+    """Working set Windows без psutil; чужие процессы не перечисляются."""
+    if sys.platform != "win32":
+        return {"rss_bytes": None, "peak_rss_bytes": None}
+
+    class Counters(ctypes.Structure):
+        _fields_ = [("cb", wintypes.DWORD), ("PageFaultCount", wintypes.DWORD)] + [
+            (name, ctypes.c_size_t) for name in (
+                "PeakWorkingSetSize", "WorkingSetSize", "QuotaPeakPagedPoolUsage",
+                "QuotaPagedPoolUsage", "QuotaPeakNonPagedPoolUsage", "QuotaNonPagedPoolUsage",
+                "PagefileUsage", "PeakPagefileUsage")]
+
+    kernel = ctypes.WinDLL("kernel32", use_last_error=True)
+    psapi = ctypes.WinDLL("psapi", use_last_error=True)
+    kernel.GetCurrentProcess.restype = wintypes.HANDLE
+    psapi.GetProcessMemoryInfo.argtypes = [wintypes.HANDLE, ctypes.POINTER(Counters), wintypes.DWORD]
+    counters = Counters()
+    counters.cb = ctypes.sizeof(counters)
+    if not psapi.GetProcessMemoryInfo(kernel.GetCurrentProcess(), ctypes.byref(counters), counters.cb):
+        return {"rss_bytes": None, "peak_rss_bytes": None, "error": ctypes.get_last_error()}
+    return {"rss_bytes": counters.WorkingSetSize, "peak_rss_bytes": counters.PeakWorkingSetSize}
+
+
+@dataclass
+class Stage:
+    calls: int = 0
+    seconds: float = 0.0
+    exclusive_seconds: float = 0.0
+
+
+class StageProfiler:
+    """Только последовательное профилирование; подмены возвращаются в finally."""
+
+    def __init__(self):
+        self.rows = []
+        self.current = None
+        self.shared = defaultdict(Stage)
+        self.stack = []
+        self.examples = {}
+
+    @contextmanager
+    def measure(self, *, optimized: bool = False):
+        patches = []
+        targets = (
+            (offline, "_correlation_scores", "coarse_localization"),
+            (lab043, "_refine_frame_start", "refinement"),
+            (lab043, "_matched_waveform", "matched_filter"),
+            (lab043, "_preamble_timing_offset", "timing"),
+            (lab043, "_timing_offsets", "timing_fallback"),
+            (lab043, "_estimate_cfo_gain", "cfo_gain"),
+            (lab043, "_evaluate_symbols", "phase_diagnostics"),
+            (lab045, "decode_phy_header_soft", "phy_header"),
+            (lab045, "analyze_post_preamble_soft", "body_pipeline"),
+            (lab045.conv, "viterbi_decode_soft", "viterbi"),
+            (lab045, "parse_packet", "crc_parser"),
+        )
+        if optimized:
+            from experiments import lab051_streaming_receiver as fast
+            targets += ((fast.BlockFrontEnd, "refine", "refinement"),
+                        (fast.BlockFrontEnd, "coarse_scores", "coarse_localization"),
+                        (fast.BlockFrontEnd, "matched_window", "matched_filter"),
+                        (fast.BlockFrontEnd, "timing", "timing"),
+                        (fast, "fast_viterbi", "viterbi"))
+
+        def wrap(function, name):
+            def measured(*args, **kwargs):
+                scope = self.shared if self.current is None else self.current
+                stage = scope[name]
+                started = time.perf_counter()
+                entry = [0.0]
+                self.stack.append(entry)
+                try:
+                    return function(*args, **kwargs)
+                finally:
+                    elapsed = time.perf_counter() - started
+                    self.stack.pop()
+                    if self.stack:
+                        self.stack[-1][0] += elapsed
+                    stage.calls += 1
+                    stage.seconds += elapsed
+                    stage.exclusive_seconds += elapsed - entry[0]
+            return measured
+
+        for owner, name, stage in targets:
+            original = getattr(owner, name)
+            patches.append((owner, name, original))
+            setattr(owner, name, wrap(original, stage))
+        original_decode = offline.decode_candidate_from_memmap
+
+        def candidate(raw, sample_count, coarse, number, **kwargs):
+            self.current = defaultdict(Stage)
+            started = time.perf_counter()
+            try:
+                outcome = original_decode(raw, sample_count, coarse, number, **kwargs)
+                elapsed = time.perf_counter() - started
+                kind = "valid" if outcome.accepted else "false"
+                self.rows.append({"kind": kind, "seconds": elapsed, "reason": outcome.reason,
+                                  "offset": coarse.sample_offset, "refined": outcome.refined_start_sample,
+                                  "stages": {name: asdict(row) for name, row in self.current.items()}})
+                # По одному примеру после решения, только для отдельного replay.
+                if kind not in self.examples and (kind == "valid" or outcome.reason.startswith("PHY_HEADER_MAGIC")):
+                    self.examples[kind] = (raw.copy(), sample_count, coarse)
+                return outcome
+            finally:
+                self.current = None
+
+        offline.decode_candidate_from_memmap = candidate
+        try:
+            yield self
+        finally:
+            offline.decode_candidate_from_memmap = original_decode
+            for owner, name, original in reversed(patches):
+                setattr(owner, name, original)
+
+    def summary(self) -> dict:
+        result = {"shared_stages": {name: asdict(row) for name, row in self.shared.items()}}
+        for kind in ("valid", "false"):
+            selected = [row for row in self.rows if row["kind"] == kind]
+            stages = defaultdict(Stage)
+            for row in selected:
+                for name, stage in row["stages"].items():
+                    stages[name].calls += stage["calls"]
+                    stages[name].seconds += stage["seconds"]
+                    stages[name].exclusive_seconds += stage["exclusive_seconds"]
+            times = [row["seconds"] for row in selected]
+            result[kind] = {"count": len(times), "mean_seconds": statistics.mean(times) if times else None,
+                            "median_seconds": statistics.median(times) if times else None,
+                            "total_seconds": sum(times),
+                            "stages": {name: asdict(row) for name, row in stages.items()}}
+        return result
+
+
+def array_observation(function) -> dict:
+    """Нижняя граница числа массивов в локальных переменных проектного Python.
+
+    Временные выражения и внутренности C/SciPy не видны line tracer-у.
+    tracemalloc отдельно считает пик отслеживаемой памяти, включая NumPy.
+    Эти проходы НИКОГДА не используются как измерение производительности.
+    """
+    seen = {}
+    counters = {"observed_array_objects": 0, "observed_owned_arrays": 0, "observed_owned_bytes_total": 0}
+
+    def trace(frame, event, argument):
+        filename = frame.f_code.co_filename.replace("\\", "/")
+        if not ("/experiments/" in filename or "/protocol/" in filename):
+            return None
+        if event in ("line", "return"):
+            for value in frame.f_locals.values():
+                if not isinstance(value, np.ndarray):
+                    continue
+                identity = id(value)
+                if identity in seen and seen[identity]() is value:
+                    continue
+                seen[identity] = weakref.ref(value)
+                counters["observed_array_objects"] += 1
+                if value.flags.owndata:
+                    counters["observed_owned_arrays"] += 1
+                    counters["observed_owned_bytes_total"] += value.nbytes
+        return trace
+
+    previous = sys.gettrace()
+    try:
+        sys.settrace(trace)
+        function()
+    finally:
+        sys.settrace(previous)
+    tracemalloc.start()
+    try:
+        function()
+        current, peak = tracemalloc.get_traced_memory()
+    finally:
+        tracemalloc.stop()
+    return {**counters, "tracemalloc_retained_bytes": current, "tracemalloc_peak_bytes": peak,
+            "array_count_is_lower_bound": True}
+
+
+def run_functional_tests() -> None:
+    assert video.phy.PhyMode(500_000).samples_per_symbol == 16
+
+
+def main() -> None:
+    OUTPUT.mkdir(parents=True, exist_ok=True)
+    with Image.open(ROOT / "data/raw/lab011_scene.jpg") as source:
+        sequence = video.encode_sequence(source, video.PROFILES[2], 3)
+    profiler = StageProfiler()
+    with profiler.measure():
+        benchmark = video.benchmark_receiver(sequence, 500_000)
+    result = {"benchmark": benchmark, "profile": profiler.summary(), "rf_tx": 0}
+    (OUTPUT / "baseline_profile.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps(result["profile"], indent=2), flush=True)
+    # Небольшие ограниченные окна, не полная запись. В Git не включать.
+    metadata = {}
+    for kind, (raw, count, coarse) in profiler.examples.items():
+        np.save(OUTPUT / f"baseline_{kind}_window.npy", raw)
+        metadata[kind] = {"sample_count": count, "offset": coarse.sample_offset, "score": coarse.score}
+        with streaming_mode(video.phy.PhyMode(500_000)):
+            call = lambda: offline.decode_candidate_from_memmap(
+                raw, count, coarse, 1,
+                read_iq_window=lambda data, start, end: data[2*start:2*end].view(np.complex64))
+            result.setdefault("array_observation", {})[kind] = array_observation(call)
+    (OUTPUT / "baseline_examples.json").write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8")
+    result["process_memory_after_profiling"] = process_memory()
+    (OUTPUT / "baseline_profile.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps(result["array_observation"]), flush=True)
+
+
+if __name__ == "__main__":
+    main()