Lab053: achieve real-time 5 fps software video receiver

2026-09-09 10:26:55 +03:00
parent cc38900cae
commit 0368c6d3e9
44 changed files with 472121 additions and 0 deletions
--- a/experiments/lab053_streaming_receiver.py
+++ b/experiments/lab053_streaming_receiver.py
@@ -0,0 +1,287 @@
+"""Lab053: прежняя политика Lab052 с независимыми строгими вычислениями."""
+
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+import sys
+import time
+
+if __package__ in (None, ""):
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import numpy as np
+
+from experiments import lab052_candidate_detection as detection
+from experiments.lab053_packet_processing import PacketProcessor
+
+
+class CandidateQueueOverflow(RuntimeError):
+    pass
+
+
+class DetectionWindow(detection.CandidateFrontEnd):
+    """Те же coarse/split функции Lab052; строгие FFT здесь не создаются."""
+
+    def __init__(self, samples, owned_end, decimator, base):
+        self.samples, self.owned_end = samples, owned_end
+        self.decimator, self.base = decimator, base
+
+
+class MeasuredDecimation(detection.previous.DecimationCache):
+    def __init__(self, decimation):
+        super().__init__(decimation)
+        self.seconds = 0.0
+
+    def get(self, samples, base):
+        started = time.perf_counter()
+        try:
+            return super().get(samples, base)
+        finally:
+            self.seconds += time.perf_counter() - started
+
+
+class StreamingReceiver(detection.StreamingReceiver):
+    """Порядок принятия прежний; опережающие результаты применяются по очереди.
+
+    Только представители групп могут вычисляться одновременно. Альтернативы
+    остаются после FAIL, пропуск внутри кадра разрешает только фактический CRC.
+    Неиспользованные опережающие проверки отдельно учтены, их verdict игнорируется.
+    """
+
+    def __init__(self, *, workers=1, candidate_capacity=4, observer=None, packet_observer=None,
+                 processor=None, block_samples=65536):
+        if type(workers) is not int or workers not in (1, 2, 4):
+            raise ValueError("Допустимы 1, 2 или 4 работника")
+        if type(candidate_capacity) is not int or candidate_capacity < workers:
+            raise ValueError("Вместимость кандидатов должна быть не меньше числа работников")
+        super().__init__(block_samples=block_samples, observer=observer)
+        self.processor = processor or PacketProcessor()
+        self.packet_observer = packet_observer
+        self.workers, self.candidate_capacity = workers, candidate_capacity
+        self.executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="lab053_strict") if workers > 1 else None
+        self.candidate_queue_peak = self.candidate_overflow = self.pending_jobs = 0
+        self.speculative_checks = self.discarded_speculative_checks = 0
+        self.speculative_seconds = self.discarded_speculative_seconds = 0.0
+        self.last_packet_rows = []
+        self.ring_capacity = self.buffer_limit_samples
+        self.ring = np.empty(2 * self.ring_capacity, np.complex64)
+        self.decimator = MeasuredDecimation(self.decimation)
+        self.input_preparation_seconds = 0.0
+
+    def submit(self, candidate):
+        if self.closed:
+            raise RuntimeError("Поток уже закрыт")
+        if self.pending_jobs >= self.candidate_capacity:
+            self.candidate_overflow += 1
+            raise CandidateQueueOverflow("Очередь строгих кандидатов заполнена")
+        self.pending_jobs += 1
+        self.candidate_queue_peak = max(self.candidate_queue_peak, self.pending_jobs)
+        try:
+            return self.executor.submit(self.processor.decode, self.buffer, candidate)
+        except BaseException:
+            self.pending_jobs -= 1
+            raise
+
+    def _scan(self, owned_end):
+        self.buffer.flags.writeable = False
+        front = DetectionWindow(self.buffer, owned_end - self.base_sample, self.decimator, self.base_sample)
+        started = time.perf_counter()
+        scores = front.coarse_scores(self.buffer, self.reference)
+        self.coarse_seconds += time.perf_counter() - started
+        peaks = detection.offline._all_local_peak_indexes(scores)
+        offsets = self.base_sample + peaks * self.decimation
+        owned_peaks = peaks[(offsets >= self.scan_sample) & (offsets < owned_end)]
+        threshold = np.float32(detection.phy.lab047.FIXED_HARDWARE_CANDIDATE_THRESHOLD)
+        first_stage = owned_peaks[scores[owned_peaks] >= threshold]
+        self.detection_statistics["total_local_peaks"] += len(owned_peaks)
+        self.detection_statistics["first_stage_candidates"] += len(first_stage)
+        self.peak_candidate_count = max(self.peak_candidate_count, len(first_stage))
+        if self.observer is not None:
+            self.observer.scores(scores, self.base_sample, self.scan_sample, owned_end, self.decimation)
+        started = time.perf_counter()
+        parts = front.split_scores(self.reference, self.rule.parts, len(scores)) if len(scores) else np.empty((0, 2))
+        passed = np.all(parts[first_stage] >= detection.part_threshold(self.rule.parts), axis=1)
+        self.cheap_seconds += time.perf_counter() - started
+        self.detection_statistics["cheap_stage_checked"] += len(first_stage)
+        self.detection_statistics["cheap_stage_rejected"] += int(np.sum(~passed))
+        indexes = first_stage[passed]
+        if self.observer is not None and hasattr(self.observer, "cheap"):
+            for index, keep in zip(first_stage, passed):
+                self.observer.cheap(self.base_sample + int(index) * self.decimation,
+                                    float(scores[index]), parts[index], bool(keep))
+        started = time.perf_counter()
+        groups = list(detection.peak_groups(indexes, self.rule.cluster_width_samples // self.decimation))
+        self.cluster_seconds += time.perf_counter() - started
+        self.detection_statistics["clusters"] += len(groups)
+        self.peak_cluster_count = max(self.peak_cluster_count, len(groups))
+        self.candidate_state_count = len(indexes)
+        self.peak_eligible_count = max(self.peak_eligible_count, len(indexes))
+        self.working_detection_arrays_peak_bytes = max(self.working_detection_arrays_peak_bytes,
+            front.detection_values.nbytes + scores.nbytes * 4 + parts.nbytes + (len(peaks) + len(indexes)) * 24)
+        ordered = [group[np.argsort(scores[group], kind="stable")[::-1]] for group in groups]
+        futures = {}
+        emitted = []
+
+        def candidate_at(index):
+            return detection.offline.CoarseCandidate(int(index) * self.decimation, float(scores[index]))
+
+        try:
+            if self.executor:
+                for group in ordered:
+                    if len(futures) == self.candidate_capacity:
+                        break
+                    index = int(group[0])
+                    if self.base_sample + index * self.decimation >= self.accepted_until - self.decimation:
+                        futures[index] = self.submit(candidate_at(index))
+                        self.speculative_checks += 1
+            attempted_groups = set()
+            for group_number, index in ((number, int(item)) for number, group in enumerate(ordered) for item in group):
+                absolute = self.base_sample + index * self.decimation
+                if absolute < self.accepted_until - self.decimation:
+                    self.detection_statistics["inside_accepted_frame"] += 1
+                    continue
+                self.detection_statistics["alternative_checks" if group_number in attempted_groups else "representative_checks"] += 1
+                attempted_groups.add(group_number)
+                self.candidate_count += 1
+                candidate = candidate_at(index)
+                if index in futures:
+                    future = futures.pop(index)
+                    try:
+                        outcome, row = future.result()
+                    finally:
+                        self.pending_jobs -= 1
+                    self.speculative_seconds += row["seconds"]
+                else:
+                    if self.pending_jobs >= self.candidate_capacity:
+                        # Освободить один уже вычисленный, но пропущенный результат.
+                        skipped = next((key for key in futures
+                                        if self.base_sample + key*self.decimation < self.accepted_until-self.decimation), None)
+                        if skipped is None:
+                            self.candidate_overflow += 1
+                            raise CandidateQueueOverflow("Нет места для альтернативного кандидата")
+                        unused, discarded = futures.pop(skipped).result()
+                        self.pending_jobs -= 1
+                        self.discarded_speculative_checks += 1
+                        self.discarded_speculative_seconds += discarded["seconds"]
+                    self.candidate_queue_peak = max(self.candidate_queue_peak, self.pending_jobs + 1)
+                    if self.executor:
+                        future = self.submit(candidate)
+                        try:
+                            outcome, row = future.result()
+                        finally:
+                            self.pending_jobs -= 1
+                    else:
+                        outcome, row = self.processor.decode(self.buffer, candidate, self.candidate_count)
+                self.strict_seconds += row["seconds"]
+                row.update(base_sample=self.base_sample, absolute_refined=self.base_sample + outcome.refined_start_sample)
+                self.last_packet_rows.append(row)
+                if self.packet_observer is not None:
+                    self.packet_observer(row, outcome)
+                if self.observer is not None:
+                    self.observer.outcome(self.base_sample, candidate, outcome)
+                if outcome.accepted and outcome.crc == detection.phy.lab045.CRC_PASS and outcome.frame is not None:
+                    self.true_strict_seconds += row["seconds"]
+                    self.accepted_until = self.base_sample + outcome.next_sample
+                    self.packet_count += 1
+                    emitted.append(detection.previous.previous.ReceivedPacket(
+                        self.base_sample + outcome.refined_start_sample, self.accepted_until,
+                        self.total_samples, bytes(outcome.frame.reception.payload_bytes)))
+                else:
+                    self.false_strict_seconds += row["seconds"]
+                    self.rejected_reasons[outcome.reason] += 1
+        finally:
+            active_error = sys.exc_info()[0] is not None
+            cleanup_error = None
+            try:
+                for future in futures.values():
+                    try:
+                        unused, row = future.result()
+                        self.discarded_speculative_checks += 1
+                        self.discarded_speculative_seconds += row["seconds"]
+                    except BaseException as error:
+                        if cleanup_error is None:
+                            cleanup_error = error
+            finally:
+                self.pending_jobs = self.candidate_state_count = 0
+            if active_error or cleanup_error is not None:
+                self.close()
+            if not active_error and cleanup_error is not None:
+                raise cleanup_error
+        self.blocks_processed += 1
+        self.scan_sample = owned_end
+        retain_start = max(0, self.scan_sample - self.margin)
+        trim = retain_start - self.base_sample
+        if trim > 0:
+            self.base_sample = retain_start
+            start = self.base_sample % self.ring_capacity
+            self.buffer = self.ring[start:start + self.total_samples - self.base_sample]
+        return emitted
+
+    def push_iq_block(self, samples):
+        preparation_started = time.perf_counter()
+        if self.closed:
+            raise RuntimeError("Поток уже закрыт")
+        samples = np.asarray(samples, dtype=np.complex64)
+        if samples.ndim != 1 or samples.size > self.block_samples or not np.all(np.isfinite(samples)):
+            raise ValueError("Нужен конечный одномерный IQ-блок не больше block_samples")
+        self.last_packet_rows = []
+        started = time.perf_counter()
+        if self.total_samples + samples.size - self.base_sample > self.buffer_limit_samples:
+            raise RuntimeError("Превышен предел буфера IQ")
+        # Две копии только НОВОГО блока дают непрерывный view через стык кольца.
+        # Старое окно не переносится. Все строгие jobs предыдущего push завершены.
+        position = self.total_samples % self.ring_capacity
+        first = min(samples.size, self.ring_capacity - position)
+        self.ring[position:position + first] = samples[:first]
+        self.ring[position + self.ring_capacity:position + self.ring_capacity + first] = samples[:first]
+        remaining = samples.size - first
+        if remaining:
+            self.ring[:remaining] = samples[first:]
+            self.ring[self.ring_capacity:self.ring_capacity + remaining] = samples[first:]
+        self.total_samples += samples.size
+        start = self.base_sample % self.ring_capacity
+        self.buffer = self.ring[start:start + self.total_samples - self.base_sample]
+        self.peak_buffer_samples = max(self.peak_buffer_samples, self.buffer.size)
+        if self.buffer.size > self.buffer_limit_samples:
+            raise RuntimeError("Превышен предел буфера IQ")
+        emitted = []
+        self.input_preparation_seconds += time.perf_counter() - preparation_started
+        while self.total_samples >= self.scan_sample + self.core_samples + self.lookahead:
+            emitted.extend(self._scan(self.scan_sample + self.core_samples))
+        self.processing_seconds += time.perf_counter() - started
+        return emitted
+
+    def finish(self):
+        if self.closed:
+            raise RuntimeError("Поток уже закрыт")
+        self.last_packet_rows = []
+        started = time.perf_counter()
+        try:
+            return self._scan(self.total_samples) if self.scan_sample < self.total_samples else []
+        finally:
+            self.close()
+            self.processing_seconds += time.perf_counter() - started
+
+    def close(self):
+        """Дождаться всех выполняющихся jobs до освобождения IQ и счётчиков."""
+        if self.executor:
+            self.executor.shutdown(wait=True, cancel_futures=True)
+        self.pending_jobs = self.candidate_state_count = 0
+        self.buffer = np.empty(0, np.complex64)
+        self.ring = np.empty(0, np.complex64)
+        self.decimator.values = np.empty(0, np.complex64)
+        self.closed = True
+
+
+def run_functional_tests() -> None:
+    assert StreamingReceiver().finish() == []
+
+
+def main() -> None:
+    run_functional_tests()
+    print("Lab053 receiver functional checks PASS; RF TX=0")
+
+
+if __name__ == "__main__":
+    main()