Lab052: suppress false preamble candidates

experiments/lab052_candidate_detection.py

@@ -0,0 +1,287 @@
+"""Lab052: локальные группы пиков и дешёвая проверка частей PRBS7.
+
+Кандидат никогда не становится пакетом без прежних PHY/FEC/CRC. Параметры
+заданы длинами PHY и моделью шума, без ожидаемых стартов и содержимого TX.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from contextlib import contextmanager
+from dataclasses import dataclass
+import math
+from pathlib import Path
+import sys
+import time
+
+if __package__ in (None, ""):
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import numpy as np
+from scipy.signal import fftconvolve
+
+from experiments import lab051_streaming_receiver as previous
+
+phy = previous.phy
+offline = previous.offline
+SAMPLES_PER_SYMBOL = 16
+PREAMBLE_SAMPLES = phy.lab045.PREAMBLE_BIT_COUNT * SAMPLES_PER_SYMBOL
+COARSE_ERROR_SAMPLES = 2000 + 8  # исходное уточнение ±2000 плюс сетка децимации
+MIN_FRAME_SAMPLES = (phy.lab045.PREAMBLE_BIT_COUNT + phy.lab045.PHY_HEADER_BIT_COUNT
+                     + phy.lab045.conv.encoded_bit_count(phy.lab045.MIN_PACKET_BYTES * 8)
+                     + 2 * phy.lab045.GUARD_SYMBOL_COUNT + phy.lab045.RRC_SPAN_SYMBOLS) * SAMPLES_PER_SYMBOL
+CLUSTER_WIDTH_SAMPLES = PREAMBLE_SAMPLES
+PART_FALSE_ALARM = .01
+
+
+@dataclass(frozen=True)
+class DetectorRule:
+    """parts=0 оставляет только группировку; 1/2/3 нужны для сравнения правил."""
+
+    parts: int = 2
+    cluster_width_samples: int = CLUSTER_WIDTH_SAMPLES
+
+    def __post_init__(self):
+        if type(self.parts) is not int or not 0 <= self.parts <= 3:
+            raise ValueError("Нужно от нуля до трёх частей")
+        if (type(self.cluster_width_samples) is not int or self.cluster_width_samples < 0
+                or self.cluster_width_samples % 8
+                or self.cluster_width_samples >= MIN_FRAME_SAMPLES - 2 * COARSE_ERROR_SAMPLES):
+            raise ValueError("Окно должно различать два минимальных физических кадра")
+
+
+def peak_groups(indexes: np.ndarray, width: int):
+    """Полная ширина от первого пика, без продолжения по цепочке соседей.
+
+    На границе IQ-блока группа обрывается: возможна лишняя проверка, но ни
+    один следующий пик не теряется из-за представителя из другого блока.
+    """
+    cursor = 0
+    while cursor < len(indexes):
+        end = int(np.searchsorted(indexes, indexes[cursor] + width, side="right"))
+        yield indexes[cursor:end]
+        cursor = end
+
+
+def cluster_peak_indexes(scores: np.ndarray, width: int, threshold: float) -> np.ndarray:
+    """Первые представители ограниченных групп; равенство → правый пик."""
+    peaks = offline._all_local_peak_indexes(scores)
+    peaks = peaks[scores[peaks] >= threshold]
+    return np.asarray([group[len(group) - 1 - int(np.argmax(scores[group[::-1]]))]
+                       for group in peak_groups(peaks, width)], dtype=np.int64)
+
+
+def part_correlations(values: np.ndarray, reference: np.ndarray, count: int) -> np.ndarray:
+    """Несколько vdot на уже децимированном согласованном входе, без timing.
+
+    Модуль сохраняет неизвестную фазу/полярность и не вводит ограничения CFO
+    через согласование фаз между половинами. Каждая часть нормируется своей
+    энергией, поэтому усиление и смена мощности фона не требуют TX reference.
+    """
+    if values.size != reference.size or count < 1:
+        return np.zeros(max(count, 0))
+    boundaries = np.linspace(0, reference.size, count + 1, dtype=int)
+    output = np.zeros(count)
+    for number, (low, high) in enumerate(zip(boundaries[:-1], boundaries[1:])):
+        part = values[low:high]
+        expected = reference[low:high]
+        energy = float(np.vdot(part, part).real) * float(np.vdot(expected, expected).real)
+        output[number] = abs(np.vdot(expected, part)) / math.sqrt(max(energy, 1e-30))
+    return output
+
+
+def part_threshold(count: int) -> float:
+    """Номинальная H0-модель: независимые комплексные отсчёты по символам.
+
+    |rho|²~Beta(1,N-1), alpha=0.01 на часть. Это исходная инженерная настройка,
+    не обещание точной вероятности после RRC/выбора пиков: фон проверяется отдельно.
+    Она фиксируется до положительных опытов и не подгоняется под их offsets.
+    """
+    symbols = phy.lab045.PREAMBLE_BIT_COUNT // count
+    return math.sqrt(1 - PART_FALSE_ALARM ** (1 / (symbols - 1)))
+
+
+class CandidateFrontEnd(previous.BlockFrontEnd):
+    """Общая корреляция — согласованный фильтр преамбулы на двух выборках/символ.
+
+    Промежуточному этапу достаточно его исходных данных; полный общий RRC-
+    выход создаётся прежним строгим трактом только при прошедшем кандидате.
+    """
+
+    def coarse_scores(self, samples, reference):
+        raw = self.decimator.get(samples, self.base)
+        mean = np.mean(raw, dtype=np.complex128).astype(np.complex64)
+        needed = math.ceil(self.owned_end / 8) + reference.size + 1
+        self.detection_values = raw[:needed].copy()
+        self.detection_values -= mean
+        self.detection_values *= previous.detection_rotation(self.detection_values.size, 8)
+        if self.detection_values.size < reference.size:
+            return np.empty(0, np.float32)
+        scores = previous.lab043._normalized_sliding_correlation(self.detection_values, reference).astype(np.float32)
+        scores[~np.isfinite(scores)] = 0
+        np.clip(scores, 0, 1, out=scores)
+        return scores
+
+    def split_scores(self, reference, count, score_count):
+        """Общие корреляции частей для всех кандидатов, без per-candidate FFT.
+
+        Проверка выполняется ДО выбора представителя: сильный случайный пик
+        одной половины не может скрыть согласованный слабый кандидат рядом.
+        """
+        output = np.empty((score_count, count), np.float32)
+        boundaries = np.linspace(0, reference.size, count + 1, dtype=int)
+        energy = np.concatenate(([0.], np.cumsum(np.abs(self.detection_values) ** 2, dtype=np.float64)))
+        for number, (low, high) in enumerate(zip(boundaries[:-1], boundaries[1:])):
+            expected = reference[low:high]
+            correlation = fftconvolve(self.detection_values[low:], np.conj(expected[::-1]), mode="valid")[:score_count]
+            local_energy = energy[high:high + score_count] - energy[low:low + score_count]
+            denominator = np.sqrt(np.maximum(local_energy * float(np.vdot(expected, expected).real), 1e-30))
+            output[:, number] = np.abs(correlation) / denominator
+        return output
+
+
+class StreamingReceiver(previous.StreamingReceiver):
+    """Ограниченный IQ и кандидаты только одного блока, API получает только IQ."""
+
+    def __init__(self, symbol_rate: int = 500000, *, block_samples: int = 65536,
+                 rule: DetectorRule = DetectorRule(), observer=None):
+        super().__init__(symbol_rate, block_samples=block_samples)
+        self.rule, self.observer = rule, observer
+        self.detection_statistics = Counter()
+        self.cheap_seconds = self.cluster_seconds = 0.0
+        self.true_strict_seconds = self.false_strict_seconds = 0.0
+        self.peak_cluster_count = self.peak_eligible_count = self.candidate_state_count = 0
+        self.working_detection_arrays_peak_bytes = 0
+        with previous.previous.streaming_mode(self.mode):
+            self.raw_reference = phy.lab045.lab044._preamble_reference_samples().copy()
+            self.matched_taps = previous.lab043._matched_filter_taps().copy()
+            self.preamble_bits = phy.lab045.lab044.preamble_bits().copy()
+        for values in (self.raw_reference, self.matched_taps, self.preamble_bits):
+            values.flags.writeable = False
+
+    @contextmanager
+    def constant_templates(self):
+        """Старый адаптер очищает lru на каждом push; эти три константы локальны.
+
+        Возвращаем именно функции до выхода streaming_mode, чтобы его
+        cache_clear и соседний старый режим работали по прежним правилам.
+        """
+        targets = ((phy.lab045.lab044, "_preamble_reference_samples", lambda: self.raw_reference),
+                   (previous.lab043, "_matched_filter_taps", lambda: self.matched_taps),
+                   (phy.lab045.lab044, "preamble_bits", lambda: self.preamble_bits))
+        originals = [(owner, name, getattr(owner, name)) for owner, name, function in targets]
+        try:
+            for owner, name, function in targets:
+                setattr(owner, name, function)
+            yield
+        finally:
+            for owner, name, function in reversed(originals):
+                setattr(owner, name, function)
+
+    def _scan(self, owned_end):
+        with self.constant_templates():
+            return self._scan_with_templates(owned_end)
+
+    def _scan_with_templates(self, owned_end):
+        width = self.rule.cluster_width_samples // self.decimation
+        front = CandidateFrontEnd(self.buffer, owned_end - self.base_sample,
+                                  self.front_statistics, decimator=self.decimator, base=self.base_sample)
+        started = time.perf_counter()
+        scores = front.coarse_scores(self.buffer, self.reference)
+        self.coarse_seconds += time.perf_counter() - started
+        threshold = np.float32(phy.lab047.FIXED_HARDWARE_CANDIDATE_THRESHOLD)
+        peaks = offline._all_local_peak_indexes(scores)
+        offsets = self.base_sample + peaks * self.decimation
+        owned = (offsets >= self.scan_sample) & (offsets < owned_end)
+        owned_peaks = peaks[owned]
+        first_stage = owned_peaks[scores[owned_peaks] >= threshold]
+        self.detection_statistics["total_local_peaks"] += len(owned_peaks)
+        self.detection_statistics["first_stage_candidates"] += len(first_stage)
+        self.peak_candidate_count = max(self.peak_candidate_count, len(first_stage))
+        if self.observer is not None:
+            self.observer.scores(scores, self.base_sample, self.scan_sample, owned_end, self.decimation)
+        started = time.perf_counter()
+        parts = (front.split_scores(self.reference, self.rule.parts, len(scores))
+                 if self.rule.parts and len(scores) else np.empty((len(scores), 0)))
+        passed = (np.all(parts[first_stage] >= part_threshold(self.rule.parts), axis=1)
+                  if self.rule.parts else np.ones(len(first_stage), dtype=bool))
+        self.cheap_seconds += time.perf_counter() - started
+        self.detection_statistics["cheap_stage_checked"] += len(first_stage)
+        self.detection_statistics["cheap_stage_rejected"] += int(np.sum(~passed))
+        indexes = first_stage[passed]
+        if self.observer is not None and hasattr(self.observer, "cheap"):
+            for index, keep in zip(first_stage, passed):
+                self.observer.cheap(self.base_sample + int(index) * self.decimation,
+                                    float(scores[index]), parts[index], bool(keep))
+        started = time.perf_counter()
+        groups = list(peak_groups(indexes, width))
+        self.cluster_seconds += time.perf_counter() - started
+        self.detection_statistics["clusters"] += len(groups)
+        self.peak_cluster_count = max(self.peak_cluster_count, len(groups))
+        self.candidate_state_count = len(indexes)
+        self.peak_eligible_count = max(self.peak_eligible_count, len(indexes))
+        self.working_detection_arrays_peak_bytes = max(self.working_detection_arrays_peak_bytes,
+            front.detection_values.nbytes + scores.nbytes * 4 + parts.nbytes + (len(peaks) + len(indexes)) * 24)
+        emitted = []
+        try:
+            with previous.optimized_context(front):
+                # Внутри группы — самый сильный первым. Отказ не даёт права
+                # подавить альтернативы: пробуем следующий. Только CRC PASS
+                # разрешает прежний пропуск уже принятого физического кадра.
+                ordered = [group[np.argsort(scores[group], kind="stable")[::-1]] for group in groups]
+                attempted_groups = set()
+                for group_number, index in ((number, item) for number, group in enumerate(ordered) for item in group):
+                    absolute = self.base_sample + int(index) * self.decimation
+                    # Сетка грубого поиска может округлить следующий стык на
+                    # одну выборку назад. Не терять соседний физический кадр.
+                    if absolute < self.accepted_until - self.decimation:
+                        self.detection_statistics["inside_accepted_frame"] += 1
+                        continue
+                    self.detection_statistics["alternative_checks" if group_number in attempted_groups
+                                              else "representative_checks"] += 1
+                    attempted_groups.add(group_number)
+                    self.candidate_count += 1
+                    candidate = offline.CoarseCandidate(int(index) * self.decimation, float(scores[index]))
+                    started = time.perf_counter()
+                    outcome = offline.decode_candidate_from_memmap(
+                        self.buffer.view(np.float32), self.buffer.size, candidate, self.candidate_count,
+                        read_iq_window=lambda raw, low, high: raw[2 * low:2 * high].view(np.complex64))
+                    elapsed = time.perf_counter() - started
+                    self.strict_seconds += elapsed
+                    if self.observer is not None:
+                        self.observer.outcome(self.base_sample, candidate, outcome)
+                    if outcome.accepted and outcome.crc == phy.lab045.CRC_PASS and outcome.frame is not None:
+                        self.true_strict_seconds += elapsed
+                        self.accepted_until = self.base_sample + outcome.next_sample
+                        self.packet_count += 1
+                        emitted.append(previous.previous.ReceivedPacket(
+                            self.base_sample + outcome.refined_start_sample, self.accepted_until,
+                            self.total_samples, bytes(outcome.frame.reception.payload_bytes)))
+                    else:
+                        self.false_strict_seconds += elapsed
+                        self.rejected_reasons[outcome.reason] += 1
+        finally:
+            self.candidate_state_count = 0
+        self.blocks_processed += 1
+        self.scan_sample = owned_end
+        retain_start = max(0, self.scan_sample - self.margin)
+        trim = retain_start - self.base_sample
+        if trim > 0:
+            self.buffer = self.buffer[trim:].copy()
+            self.base_sample = retain_start
+        return emitted
+
+
+def run_functional_tests() -> None:
+    assert CLUSTER_WIDTH_SAMPLES < MIN_FRAME_SAMPLES - 2 * COARSE_ERROR_SAMPLES
+    assert .08 < part_threshold(2) < .09
+    assert StreamingReceiver().finish() == []
+
+
+def main() -> None:
+    run_functional_tests()
+    print("Lab052 detector functional checks PASS; RF TX=0")
+
+
+if __name__ == "__main__":
+    main()

experiments/lab052_detection_statistics.py

@@ -0,0 +1,157 @@
+"""Lab052: распределения и численные проверки после независимых решений RX."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import sys
+
+if __package__ in (None, ""):
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import numpy as np
+
+from experiments.lab052_detector_study import OUTPUT, quantiles, write_result
+from experiments import lab052_candidate_detection as detection
+from experiments.lab051_receiver_profile import array_observation
+
+
+def measure_arrays() -> dict:
+    """Отдельный короткий RX replay, без смешивания tracing с замером скорости.
+
+    ndarray в Python-locals даёт нижнюю границу наблюдаемых массивов; C/SciPy
+    могут иметь невидимые временные выделения. tracemalloc peak — отдельная
+    измеренная величина, не RSS и не универсальная верхняя граница памяти.
+    """
+    physical = detection.phy.lab045
+    payload = bytes(range(256))
+    plan = physical.build_transmission_plan(payload)
+    raw = detection.phy.build_parameterized_tx_cs8(plan, detection.phy.PhyMode(500000))
+    components = np.frombuffer(raw, np.int8).astype(np.float32)
+    values = components[0::2] + 1j * components[1::2]
+    random_generator = np.random.default_rng(52040)
+    sigma = np.sqrt(float(np.sum(abs(values)**2)) / plan.frame_symbol_count / (2 * 10**.7))
+    values += sigma * (random_generator.normal(size=len(values)) + 1j*random_generator.normal(size=len(values)))
+    samples = .2 * (random_generator.normal(size=63217 + len(values) + 350000)
+                    + 1j * random_generator.normal(size=63217 + len(values) + 350000))
+    samples[63217:63217 + len(values)] = values
+    samples = samples.astype(np.complex64)
+
+    def replay():
+        receiver = detection.StreamingReceiver()
+        packets = []
+        for start in range(0, len(samples), receiver.block_samples):
+            packets.extend(receiver.push_iq_block(samples[start:start + receiver.block_samples]))
+        packets.extend(receiver.finish())
+        assert len(packets) == 1 and packets[0].payload == payload
+
+    result = {**array_observation(replay), "iq_seconds": len(samples) / 8000000,
+              "packets": 1, "interpretation": "observed Python ndarray lower bound; tracked RX allocation peak; input IQ prepared before tracing",
+              "timing_not_used_as_benchmark": True, "rf_tx": 0}
+    write_result("allocations.json", result)
+    return result
+
+
+def analyze() -> dict:
+    baseline = json.loads((OUTPUT / "baseline_profile.json").read_text(encoding="utf-8"))
+    optimized = json.loads((OUTPUT / "parts2.json").read_text(encoding="utf-8"))
+    before = np.load(OUTPUT / "baseline_observations.npz")
+    after = np.load(OUTPUT / "parts2_observations.npz")
+    peaks, outcomes = before["peaks"], before["outcomes"]
+    true = outcomes[outcomes[:, 3] == 1]
+    false = outcomes[outcomes[:, 3] == 0]
+    starts = np.sort(true[:, 2].astype(np.int64))
+    new_starts = np.sort(after["outcomes"][after["outcomes"][:, 3] == 1, 2].astype(np.int64))
+    missing = np.setdiff1d(starts, new_starts)
+    unexpected = np.setdiff1d(new_starts, starts)
+    main_scores = []
+    for start in starts:
+        low, high = np.searchsorted(peaks[:, 0], [start - detection.COARSE_ERROR_SAMPLES,
+                                                 start + detection.COARSE_ERROR_SAMPLES], side="left")
+        main_scores.append(float(peaks[low:high, 1].max()))
+    # Окна чистого фона выделяются ТОЛЬКО после независимых принятых кадров.
+    # В этом опыте decoded PHY packet 268B => waveform 92256 samples.
+    frame_samples = (detection.phy.lab045.PREAMBLE_BIT_COUNT + detection.phy.lab045.PHY_HEADER_BIT_COUNT
+                     + detection.phy.lab045.conv.encoded_bit_count((256 + detection.phy.lab045.MIN_PACKET_BYTES) * 8)
+                     + 42) * 16
+    blocks = baseline["detection_profile"]["blocks"]
+    noise_blocks = []
+    for block in blocks:
+        low, high = block["start_sample"], block["end_sample"] + (1280 + 42) * 16
+        if not np.any((starts < high) & (starts + frame_samples > low)):
+            noise_blocks.append(block)
+    noise_statistics = {name: quantiles([block["score_statistics"][name] for block in noise_blocks])
+                        for name in ("median", "mad", "p90", "p99", "p999")}
+    cheap = np.load(OUTPUT / "parts2_cheap.npy")
+    accepted_coarse = after["outcomes"][after["outcomes"][:, 3] == 1, 0]
+    is_true = np.isin(cheap[:, 0], accepted_coarse)
+    part_min = cheap[:, 3:].min(axis=1)
+    false_count = len(false)
+    new_false = optimized["detector"]["strict_false_candidates"]
+    result = {
+        "classification": "strict PHY/FEC/CRC verdict first; no diagnostic feedback into RX",
+        "baseline_true_unique_starts": len(np.unique(starts)),
+        "optimized_true_unique_starts": len(np.unique(new_starts)),
+        "missing_true_starts": missing.tolist(), "unexpected_true_starts": unexpected.tolist(),
+        "true_main_peak_scores": quantiles(main_scores), "false_strict_scores": quantiles(false[:, 1]),
+        "baseline_false_refined_positions": len(np.unique(false[:, 2])),
+        "baseline_false_repeated_refinements": len(false) - len(np.unique(false[:, 2])),
+        "noise_only_blocks": len(noise_blocks), "noise_block_statistics": noise_statistics,
+        "split_minimum_true_strict": quantiles(part_min[is_true]),
+        "split_minimum_other_candidates": quantiles(part_min[~is_true]),
+        "strict_false_reduction": false_count / new_false,
+        "strict_false_reduction_fraction": 1 - new_false / false_count,
+        "first_stage_threshold_unchanged": detection.phy.lab047.FIXED_HARDWARE_CANDIDATE_THRESHOLD,
+        "part_threshold": detection.part_threshold(2), "rf_tx": 0,
+    }
+    write_result("statistics.json", result)
+    # Научные статические рисунки сохраняются самостоятельными артефактами.
+    import matplotlib
+    matplotlib.use("Agg")
+    import matplotlib.pyplot as plt
+    figure, axes = plt.subplots(2, 2, figsize=(12, 8), constrained_layout=True)
+    axis = axes[0, 0]
+    bins = np.linspace(0, 1, 101)
+    axis.hist(false[:, 1], bins, histtype="step", label="Ложные строгие кандидаты", log=True)
+    axis.hist(main_scores, bins, histtype="step", label="Главный пик настоящей преамбулы", log=True)
+    axis.axvline(result["first_stage_threshold_unchanged"], color="gray", linestyle="--", label="Первый порог")
+    axis.set(xlabel="Нормированная корреляция", ylabel="Количество", title="Классификация после PHY/FEC/CRC")
+    axis.legend(fontsize=8)
+    axis = axes[0, 1]
+    for name, label in (("median", "Медиана"), ("mad", "MAD"), ("p99", "Квантиль 99%")):
+        axis.plot([block["start_sample"] / 8000000 for block in noise_blocks],
+                  [block["score_statistics"][name] for block in noise_blocks], ".", markersize=2, label=label)
+    axis.set(xlabel="Время IQ, с", ylabel="Корреляционная статистика", title="Блоки чистого фона, определённые после RX")
+    axis.legend(fontsize=8)
+    axis = axes[1, 0]
+    axis.hist(part_min[~is_true], bins, histtype="step", log=True, label="Остальные локальные кандидаты")
+    axis.hist(part_min[is_true], bins, histtype="step", log=True, label="Кандидаты с последующим CRC PASS")
+    axis.axvline(detection.part_threshold(2), color="gray", linestyle="--", label="Порог каждой половины")
+    axis.set(xlabel="Меньшая корреляция двух половин", ylabel="Количество", title="Дешёвая проверка обеих частей")
+    axis.legend(fontsize=8)
+    axis = axes[1, 1]
+    rule_data = json.loads((OUTPUT / "rule_comparison.json").read_text(encoding="utf-8"))["rules"]
+    counts = [row["detector"]["strict_false_candidates"] for row in rule_data]
+    bars = axis.bar(["Группы", "1 часть", "2 части", "3 части"], counts)
+    axis.set_yscale("log")
+    for bar, count in zip(bars, counts):
+        axis.text(bar.get_x() + bar.get_width()/2, count*1.15, str(count), ha="center", fontsize=9)
+    axis.set(ylabel="Ложные строгие проверки", title="Все четыре варианта сохранили 240 пакетов")
+    figure.savefig(OUTPUT / "detector_distributions.png", dpi=160)
+    figure.savefig(OUTPUT / "detector_distributions.svg")
+    plt.close(figure)
+    return result
+
+
+def run_functional_tests() -> None:
+    assert detection.part_threshold(2) < detection.part_threshold(3)
+
+
+def main() -> None:
+    result = analyze()
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+    print(json.dumps(measure_arrays(), ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()

experiments/lab052_detector_study.py

@@ -0,0 +1,374 @@
+"""Lab052: воспроизводимый профиль обнаружителя, только программный IQ.
+
+Диагностическая история принадлежит эксперименту. Приёмник не получает её
+обратно; эталон используется только после независимых решений PHY/FEC/CRC.
+"""
+
+from __future__ import annotations
+
+import argparse
+import cProfile
+from collections import Counter
+from dataclasses import replace
+import hashlib
+import json
+from pathlib import Path
+import sys
+import time
+from types import SimpleNamespace
+
+if __package__ in (None, ""):
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import numpy as np
+from PIL import Image
+
+from experiments import lab051_streaming_receiver as previous
+from experiments import lab051_receiver_performance as performance
+from experiments.lab051_receiver_profile import process_memory
+
+ROOT = Path(__file__).resolve().parents[1]
+OUTPUT = ROOT / "data/processed/lab052"
+
+
+def write_result(name: str, value: dict) -> None:
+    OUTPUT.mkdir(parents=True, exist_ok=True)
+    (OUTPUT / name).write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+
+def sequence_fixture():
+    """Тот же генератор Lab051, готовность JPEG фиксируется первым замером.
+
+    Прежний опыт не сохранял encode_seconds, поэтому побитную историческую
+    временную сетку восстановить нельзя. Все новые сравнения используют одну
+    сохранённую сетку, один seed и те же байты JPEG, включая реальную encode cost.
+    """
+    with Image.open(ROOT / "data/raw/lab011_scene.jpg") as source:
+        sequence = performance.video.encode_sequence(source, performance.video.PROFILES[2], 30)
+    path = OUTPUT / "source_fixture.json"
+    if path.exists():
+        fixture = json.loads(path.read_text(encoding="utf-8"))
+        for item, recorded in zip(sequence, fixture["images"], strict=True):
+            assert hashlib.sha256(item.jpeg).hexdigest() == recorded["sha256"]
+        return [replace(item, encode_seconds=recorded["encode_seconds"])
+                for item, recorded in zip(sequence, fixture["images"], strict=True)]
+    write_result(path.name, {"seed": 50050, "description": "Lab051 generator; frozen measured JPEG readiness",
+                            "images": [{"image_id": item.image_id, "bytes": len(item.jpeg),
+                                        "encode_seconds": item.encode_seconds,
+                                        "sha256": hashlib.sha256(item.jpeg).hexdigest()} for item in sequence]})
+    return sequence
+
+
+def quantiles(values) -> dict:
+    values = np.asarray(values, dtype=np.float64)
+    if not values.size:
+        return {"count": 0}
+    median = float(np.median(values))
+    return {"count": values.size, "min": float(values.min()), "median": median,
+            "mad": float(np.median(np.abs(values - median))),
+            "p90": float(np.quantile(values, .9)), "p99": float(np.quantile(values, .99)),
+            "p999": float(np.quantile(values, .999)), "max": float(values.max())}
+
+
+class DetectionObserver:
+    """Полная история только для конечного лабораторного профиля, вне RX."""
+
+    def __init__(self):
+        self.peaks = []
+        self.blocks = []
+        self.outcomes = []
+        self.cheap_rows = []
+
+    def scores(self, scores, base, low, high, decimation):
+        indexes = previous.offline._all_local_peak_indexes(scores)
+        offsets = base + indexes * decimation
+        selected = (offsets >= low) & (offsets < high)
+        self.peaks.append(np.column_stack((offsets[selected], scores[indexes[selected]])))
+        first = max(0, (low - base + decimation - 1) // decimation)
+        last = min(len(scores), (high - base + decimation - 1) // decimation)
+        self.blocks.append({"start_sample": low, "end_sample": high,
+                            "score_statistics": quantiles(scores[first:last]),
+                            "peak_statistics": quantiles(scores[indexes[selected]])})
+
+    def outcome(self, base, candidate, outcome):
+        self.outcomes.append((base + candidate.sample_offset, candidate.score,
+                              base + outcome.refined_start_sample, int(outcome.accepted)))
+
+    def cheap(self, absolute, score, parts, passed):
+        self.cheap_rows.append([absolute, score, int(passed), *parts])
+
+    def save(self, name, result):
+        peaks = np.concatenate(self.peaks) if self.peaks else np.empty((0, 2))
+        outcomes = np.asarray(self.outcomes).reshape(-1, 4)
+        np.savez_compressed(OUTPUT / f"{name}_observations.npz", peaks=peaks, outcomes=outcomes)
+        if self.cheap_rows:
+            np.save(OUTPUT / f"{name}_cheap.npy", np.asarray(self.cheap_rows))
+        threshold = previous.phy.lab047.FIXED_HARDWARE_CANDIDATE_THRESHOLD
+        valid = outcomes[:, 3] == 1
+        # Сначала строгий вердикт, затем классификация корреляционной статистики.
+        result["detection_profile"] = {
+            "total_local_peaks": len(peaks), "first_stage_candidates": int(np.sum(peaks[:, 1] >= threshold)),
+            "strict_candidates": len(outcomes), "strict_true_candidates": int(np.sum(valid)),
+            "strict_false_candidates": int(np.sum(~valid)),
+            "candidates_per_rf_second": len(outcomes) / result["iq_seconds"],
+            "false_candidates_per_rf_second": int(np.sum(~valid)) / result["iq_seconds"],
+            "all_local_peaks": quantiles(peaks[:, 1]),
+            "strict_true_scores": quantiles(outcomes[valid, 1]),
+            "strict_false_scores": quantiles(outcomes[~valid, 1]),
+            "blocks": self.blocks,
+        }
+
+
+class ProfileBaseline(previous.StreamingReceiver):
+    """Штатный Lab051 с наблюдением после корреляции и после решения."""
+
+    def __init__(self, observer):
+        super().__init__()
+        self.observer = observer
+
+    def _scan(self, owned_end):
+        front = previous.BlockFrontEnd(self.buffer, owned_end - self.base_sample, self.front_statistics,
+                                       decimator=self.decimator, base=self.base_sample)
+        original_scores = front.coarse_scores
+
+        def scores(samples, reference):
+            values = original_scores(samples, reference)
+            self.observer.scores(values, self.base_sample, self.scan_sample, owned_end, self.decimation)
+            return values
+
+        front.coarse_scores = scores
+        original_decode = previous.offline.decode_candidate_from_memmap
+
+        def decode(raw, count, candidate, number, **kwargs):
+            outcome = original_decode(raw, count, candidate, number, **kwargs)
+            self.observer.outcome(self.base_sample, candidate, outcome)
+            return outcome
+
+        try:
+            previous.offline.decode_candidate_from_memmap = decode
+            with previous.optimized_context(front):
+                return previous.previous.StreamingReceiver._scan(self, owned_end)
+        finally:
+            previous.offline.decode_candidate_from_memmap = original_decode
+
+
+def measured_stream(sequence, factory):
+    """Повторно использовать исходную событийную шкалу с фактическими push times."""
+    instances = []
+
+    def create():
+        receiver = factory()
+        instances.append(receiver)
+        return receiver
+
+    original = performance.fast
+    try:
+        performance.fast = SimpleNamespace(StreamingReceiver=create)
+        result = performance.run_stream(sequence)
+    finally:
+        performance.fast = original
+    receiver = instances[0]
+    result["coarse_seconds"] = receiver.coarse_seconds
+    result["strict_seconds"] = receiver.strict_seconds
+    result["peak_candidates_in_block"] = receiver.peak_candidate_count
+    if hasattr(receiver, "detection_statistics"):
+        result["detector"] = {**dict(receiver.detection_statistics),
+                              "strict_true_candidates": receiver.packet_count,
+                              "strict_false_candidates": receiver.candidate_count - receiver.packet_count,
+                              "strict_checks_per_rf_second": receiver.candidate_count / result["iq_seconds"],
+                              "cheap_seconds": receiver.cheap_seconds, "cluster_seconds": receiver.cluster_seconds,
+                              "true_strict_seconds": receiver.true_strict_seconds,
+                              "false_strict_seconds": receiver.false_strict_seconds,
+                              "peak_cluster_count": receiver.peak_cluster_count,
+                              "peak_eligible_count": receiver.peak_eligible_count,
+                              "candidate_state_after_finish": receiver.candidate_state_count,
+                              "detection_arrays_peak_estimate_bytes": receiver.working_detection_arrays_peak_bytes}
+    return result, receiver
+
+
+def profile_baseline(sequence):
+    observer = DetectionObserver()
+    result, receiver = measured_stream(sequence, lambda: ProfileBaseline(observer))
+    observer.save("baseline", result)
+    result["timing_includes_diagnostic_observer"] = True
+    write_result("baseline_profile.json", result)
+    print(json.dumps({key: value for key, value in result.items() if key in (
+        "candidate_count", "received_images", "rx_physical_packets", "processing_per_rf_second",
+        "coarse_seconds", "strict_seconds")}), flush=True)
+    print(json.dumps({key: value for key, value in result["detection_profile"].items() if key != "blocks"}), flush=True)
+
+
+def unprofiled_baseline(sequence):
+    result, receiver = measured_stream(sequence, previous.StreamingReceiver)
+    result["timing_includes_diagnostic_observer"] = False
+    write_result("baseline_stream.json", result)
+    print(json.dumps({key: result[key] for key in ("candidate_count", "processing_seconds",
+        "processing_per_rf_second", "rx_physical_packets", "received_images", "object_jpeg_integrity")}), flush=True)
+
+
+def run_functional_tests() -> None:
+    assert quantiles([1, 2, 3])["median"] == 2
+
+
+def compare_rules(sequence):
+    from experiments import lab052_candidate_detection as detection
+    baseline = json.loads((OUTPUT / "baseline_profile.json").read_text(encoding="utf-8"))
+    results = []
+    for parts in (0, 1, 2, 3):
+        observer = DetectionObserver()
+        rule = detection.DetectorRule(parts=parts)
+        result, receiver = measured_stream(sequence, lambda: detection.StreamingReceiver(rule=rule, observer=observer))
+        observer.save(f"parts{parts}", result)
+        result["rule"] = {"parts": parts, "width_samples": rule.cluster_width_samples,
+                          "part_threshold": detection.part_threshold(parts) if parts else None}
+        result["true_candidates_lost"] = baseline["rx_physical_packets"] - result["rx_physical_packets"]
+        false_count = result["detector"]["strict_false_candidates"]
+        baseline_false = baseline["detection_profile"]["strict_false_candidates"]
+        result["false_candidate_reduction"] = baseline_false / false_count if false_count else None
+        result["false_candidate_reduction_fraction"] = 1 - false_count / baseline_false
+        result["timing_includes_diagnostic_observer"] = True
+        write_result(f"parts{parts}.json", result)
+        compact = {key: result[key] for key in ("rule", "received_images", "object_jpeg_integrity",
+                                               "true_candidates_lost", "false_candidate_reduction", "detector",
+                                               "processing_per_rf_second", "fresh_display_fps")}
+        results.append(compact)
+        print(json.dumps(compact), flush=True)
+    write_result("rule_comparison.json", {"rules": results, "rf_tx": 0})
+
+
+def optimized_stream(sequence):
+    from experiments import lab052_candidate_detection as detection
+    results = []
+    for number in range(1, 4):
+        result, receiver = measured_stream(sequence, detection.StreamingReceiver)
+        result["timing_includes_diagnostic_observer"] = False
+        write_result(f"stream_repeat_{number}.json", result)
+        results.append(result)
+        print(json.dumps({"repeat": number, "ratio": result["processing_per_rf_second"],
+                          "fresh_fps": result["fresh_display_fps"], "packets": result["rx_physical_packets"]}), flush=True)
+    result = sorted(results, key=lambda row: row["processing_seconds"])[1]
+    result["selection"] = "median processing time of three complete measured passes; one whole timeline"
+    write_result("stream.json", result)
+    print(json.dumps({key: value for key, value in result.items() if key not in (
+        "blocks", "images", "rss_samples", "fresh_image_age_seconds")}), flush=True)
+
+
+def profile_cpu(sequence):
+    from experiments import lab052_candidate_detection as detection
+    import pstats
+    from io import StringIO
+    profiler = cProfile.Profile()
+
+    class ProfiledReceiver(detection.StreamingReceiver):
+        def push_iq_block(self, samples):
+            return profiler.runcall(super().push_iq_block, samples)
+
+        def finish(self):
+            return profiler.runcall(super().finish)
+
+    result, receiver = measured_stream(sequence, ProfiledReceiver)
+    profiler.dump_stats(str(OUTPUT / "receiver_cpu.prof"))
+    report = StringIO()
+    stats = pstats.Stats(profiler, stream=report).strip_dirs()
+    stats.sort_stats("tottime").print_stats(30)
+    stats.sort_stats("cumtime").print_stats(30)
+    (OUTPUT / "receiver_cpu.txt").write_text(report.getvalue(), encoding="utf-8")
+    result["timing_includes_cprofile"] = True
+    write_result("cpu_profile.json", result)
+    print(report.getvalue(), flush=True)
+
+
+def long_state_test(sequence=None, block_count=2048):
+    """16,78 с распределённого шума, меняющаяся мощность, без истории в RX."""
+    from experiments import lab052_candidate_detection as detection
+    receiver = detection.StreamingReceiver()
+    random_generator = np.random.default_rng(52030)
+    checkpoints = []
+    accepted = 0
+    for number in range(block_count):
+        components = random_generator.standard_normal((2, receiver.block_samples), dtype=np.float32)
+        scale = (.002, .2, 20., .02)[(number // 257) % 4]
+        values = (components[0] + 1j * components[1]) * scale
+        accepted += len(receiver.push_iq_block(values))
+        assert receiver.buffer.size <= receiver.buffer_limit_samples
+        assert receiver.candidate_state_count == 0
+        if (number + 1) % 256 == 0:
+            checkpoints.append({"block": number + 1, "iq_bytes": receiver.buffer.nbytes,
+                                "decimated_bytes": receiver.decimator.values.nbytes,
+                                "retained_candidates": receiver.candidate_state_count,
+                                "peak_eligible_candidates": receiver.peak_eligible_count,
+                                "counter_keys": len(receiver.detection_statistics) + len(receiver.rejected_reasons),
+                                **process_memory()})
+    accepted += len(receiver.finish())
+    result = {"blocks": block_count, "iq_seconds": block_count * receiver.block_samples / 8000000,
+              "background_scales": [.002, .2, 20, .02], "accepted_noise_packets": accepted,
+              "peak_retained_iq_bytes": receiver.peak_buffer_samples * 8,
+              "iq_limit_bytes": receiver.buffer_limit_samples * 8,
+              "retained_after_finish_bytes": receiver.buffer.nbytes + receiver.decimator.values.nbytes,
+              "constant_template_bytes": receiver.raw_reference.nbytes + receiver.matched_taps.nbytes + receiver.preamble_bits.nbytes,
+              "candidate_state_after_finish": receiver.candidate_state_count,
+              "peak_eligible_candidates": receiver.peak_eligible_count,
+              "peak_cluster_count": receiver.peak_cluster_count,
+              "detector": dict(receiver.detection_statistics), "strict_candidates": receiver.candidate_count,
+              "processing_seconds": receiver.processing_seconds, "checkpoints": checkpoints,
+              "process_memory": process_memory(), "rf_tx": 0}
+    write_result("memory.json", result)
+    print(json.dumps(result), flush=True)
+
+
+def sensitivity_test(sequence=None):
+    """До/после на одинаковом IQ около границы работоспособности FEC."""
+    from experiments import lab052_candidate_detection as detection
+    physical = previous.phy.lab045
+    payload = bytes(range(256))
+    plan = physical.build_transmission_plan(payload)
+    raw = previous.phy.build_parameterized_tx_cs8(plan, previous.phy.PhyMode(500000))
+    components = np.frombuffer(raw, np.int8).astype(np.float32)
+    clean = components[0::2] + 1j * components[1::2]
+    energy = float(np.sum(np.abs(clean) ** 2)) / plan.frame_symbol_count
+    rows = []
+    for ebn0 in (0, 1, 2, 3, 7):
+        for cfo in (-100, 10, 100):
+            for seed, scale in ((52031, .002), (52032, .2), (52033, 1.)):
+                random_generator = np.random.default_rng(seed)
+                sigma = np.sqrt(energy / (2 * 10 ** (ebn0 / 10)))
+                phase = .73 + (seed % 2) * np.pi
+                values = clean * np.exp(1j * (phase + 2*np.pi*cfo*np.arange(len(clean))/8000000))
+                values += sigma * (random_generator.normal(size=len(clean)) + 1j*random_generator.normal(size=len(clean)))
+                samples = np.concatenate((np.zeros(63217, np.complex64), (values * scale).astype(np.complex64),
+                                          np.zeros(350000, np.complex64)))
+                outcomes = []
+                for constructor in (previous.StreamingReceiver, detection.StreamingReceiver):
+                    receiver = constructor()
+                    packets = []
+                    for start in range(0, len(samples), receiver.block_samples):
+                        packets.extend(receiver.push_iq_block(samples[start:start + receiver.block_samples]))
+                    packets.extend(receiver.finish())
+                    # Оценка содержимого только после завершения независимого RX.
+                    outcomes.append({"accepted": len(packets),
+                                     "payload_match": len(packets) == 1 and packets[0].payload == payload,
+                                     "reasons": dict(receiver.rejected_reasons)})
+                rows.append({"ebn0_db": ebn0, "cfo_hz": cfo, "seed": seed, "amplitude_scale": scale,
+                             "baseline": outcomes[0], "optimized": outcomes[1],
+                             "lost_baseline_packet": outcomes[0]["payload_match"] and not outcomes[1]["payload_match"]})
+    result = {"cases": rows, "count": len(rows),
+              "baseline_correct": sum(row["baseline"]["payload_match"] for row in rows),
+              "optimized_correct": sum(row["optimized"]["payload_match"] for row in rows),
+              "lost_baseline_packets": sum(row["lost_baseline_packet"] for row in rows), "rf_tx": 0}
+    write_result("sensitivity.json", result)
+    print(json.dumps({key: value for key, value in result.items() if key != "cases"}), flush=True)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--stage", choices=("profile", "baseline", "rules", "stream", "cpu", "memory", "sensitivity"), default="profile")
+    arguments = parser.parse_args()
+    OUTPUT.mkdir(parents=True, exist_ok=True)
+    sequence = sequence_fixture()
+    print(f"Lab052 {arguments.stage}: START; RF TX=0", flush=True)
+    {"profile": profile_baseline, "baseline": unprofiled_baseline, "rules": compare_rules, "stream": optimized_stream,
+     "cpu": profile_cpu, "memory": long_state_test, "sensitivity": sensitivity_test}[arguments.stage](sequence)
+
+
+if __name__ == "__main__":
+    main()