"""Потоковое чтение больших cs8-захватов и оконный приём RF-001A.""" from __future__ import annotations from dataclasses import dataclass, replace from datetime import datetime, timezone import ctypes from pathlib import Path import threading from typing import Iterable import numpy as np from protocol.video003_modem import SAMPLE_RATE_HZ, build_frame, samples_per_symbol from protocol.video005_iq_waveform import NOMINAL_CARRIER_HZ, SYMBOL_RATE, shape_packet from protocol.video005_receiver import IqStreamResult, ReceivedIqPacket, receive_iq_stream from tools.qpsk_rf_common import PAYLOAD_BYTES, SEED, payload_for_sequence DEFAULT_CHUNK_SAMPLES = 4_000_000 LOCATOR_BLOCK_SAMPLES = 400_000 TIMESTAMP_MARGIN_SECONDS = 2.0 @dataclass(frozen=True) class SampleWindow: start: int stop: int @property def sample_count(self) -> int: return self.stop - self.start @dataclass(frozen=True) class StreamingReception: reception: IqStreamResult windows: tuple[SampleWindow, ...] window_source: str overlap_samples: int maximum_loaded_samples: int clipping_codes: int class MemoryPeakSampler: """Измеряет пиковый working set процесса во время анализа.""" def __init__(self, poll_seconds: float = 0.02) -> None: self.poll_seconds = poll_seconds self.baseline_bytes = current_process_rss_bytes() self.peak_bytes = self.baseline_bytes self._stop = threading.Event() self._thread = threading.Thread(target=self._sample, daemon=True) def _sample(self) -> None: while not self._stop.wait(self.poll_seconds): self.peak_bytes = max(self.peak_bytes, current_process_rss_bytes()) def __enter__(self) -> MemoryPeakSampler: self._thread.start() return self def __exit__(self, *_exc: object) -> None: self.peak_bytes = max(self.peak_bytes, current_process_rss_bytes()) self._stop.set() self._thread.join(timeout=1.0) @property def additional_peak_bytes(self) -> int: return max(0, self.peak_bytes - self.baseline_bytes) def current_process_rss_bytes() -> int: """Возвращает текущий working set процесса без сторонних библиотек.""" if not hasattr(ctypes, "windll"): return 0 class ProcessMemoryCounters(ctypes.Structure): _fields_ = [ ("cb", ctypes.c_ulong), ("page_fault_count", ctypes.c_ulong), ("peak_working_set_size", ctypes.c_size_t), ("working_set_size", ctypes.c_size_t), ("quota_peak_paged_pool_usage", ctypes.c_size_t), ("quota_paged_pool_usage", ctypes.c_size_t), ("quota_peak_non_paged_pool_usage", ctypes.c_size_t), ("quota_non_paged_pool_usage", ctypes.c_size_t), ("pagefile_usage", ctypes.c_size_t), ("peak_pagefile_usage", ctypes.c_size_t), ] counters = ProcessMemoryCounters() counters.cb = ctypes.sizeof(counters) get_current_process = ctypes.windll.kernel32.GetCurrentProcess # type: ignore[attr-defined] get_current_process.restype = ctypes.c_void_p get_process_memory_info = ctypes.windll.psapi.GetProcessMemoryInfo # type: ignore[attr-defined] get_process_memory_info.argtypes = [ ctypes.c_void_p, ctypes.POINTER(ProcessMemoryCounters), ctypes.c_ulong, ] get_process_memory_info.restype = ctypes.c_int succeeded = get_process_memory_info( get_current_process(), ctypes.byref(counters), counters.cb, ) return int(counters.working_set_size) if succeeded else 0 def cs8_sample_count(path: Path) -> int: size = path.stat().st_size if size == 0 or size % 2: raise ValueError("cs8 должен содержать целое ненулевое число пар I/Q") return size // 2 def read_cs8_window(raw: np.memmap, start: int, stop: int) -> np.ndarray: """Преобразует в complex64 только заданное окно cs8.""" total_samples = raw.size // 2 if start < 0 or stop <= start or stop > total_samples: raise ValueError("границы окна cs8 выходят за пределы файла") values = np.asarray(raw[start * 2:stop * 2], dtype=np.int8) real = values[0::2].astype(np.float32) imaginary = values[1::2].astype(np.float32) return ((real + 1j * imaginary) / 128).astype(np.complex64) def rf001_packet_waveform_samples() -> int: """Вычисляет длину волны из настоящего 256-байтного пакета RF-001.""" if PAYLOAD_BYTES != 256: raise AssertionError("канонический RF-001 payload неожиданно изменился") payload = payload_for_sequence(SEED, 0) frame = build_frame(payload, "QPSK", "2/3", sequence_number=0) return int(shape_packet(frame).size) def required_overlap_samples() -> int: """Перекрывает целую максимальную волну одного пакета RF-001.""" return rf001_packet_waveform_samples() def _parse_utc(value: str) -> datetime: parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) if parsed.tzinfo is None: raise ValueError("timestamp должен содержать часовой пояс") return parsed.astimezone(timezone.utc) def timestamp_sample_window( *, capture_started_at: str, tx_started_at: str, tx_finished_at: str, total_samples: int, sample_rate: int = SAMPLE_RATE_HZ, margin_seconds: float = TIMESTAMP_MARGIN_SECONDS, ) -> SampleWindow: """Переводит UTC-границы TX в безопасное окно отсчётов RX.""" if total_samples <= 0 or sample_rate <= 0 or margin_seconds < 0: raise ValueError("параметры timestamp-окна должны быть положительными") capture = _parse_utc(capture_started_at) started = _parse_utc(tx_started_at) finished = _parse_utc(tx_finished_at) if finished < started: raise ValueError("TX_FINISHED_AT не может предшествовать TX_STARTED_AT") margin_samples = round(margin_seconds * sample_rate) start_offset = round((started - capture).total_seconds() * sample_rate) stop_offset = round((finished - capture).total_seconds() * sample_rate) start = max(0, start_offset - margin_samples) stop = min(total_samples, stop_offset + margin_samples) if stop <= start: raise ValueError("TX timestamps не пересекаются с RX capture") return SampleWindow(start, stop) def _merge_windows(windows: Iterable[SampleWindow]) -> tuple[SampleWindow, ...]: ordered = sorted(windows, key=lambda item: item.start) merged: list[SampleWindow] = [] for window in ordered: if not merged or window.start > merged[-1].stop: merged.append(window) else: merged[-1] = SampleWindow( merged[-1].start, max(merged[-1].stop, window.stop), ) return tuple(merged) def locate_power_windows( path: Path, *, block_samples: int = LOCATOR_BLOCK_SAMPLES, padding_seconds: float = 1.5, ) -> tuple[SampleWindow, ...]: """Ищет повышенную мощность по всему cs8-файлу ограниченными блоками.""" total_samples = cs8_sample_count(path) if block_samples <= 0 or padding_seconds < 0: raise ValueError("параметры locator должны быть неотрицательными") powers: list[float] = [] with path.open("rb") as stream: while raw_bytes := stream.read(block_samples * 2): values = np.frombuffer(raw_bytes, dtype=np.int8).astype(np.float32) powers.append(float(np.mean(values * values))) if not powers: return () power_array = np.asarray(powers, dtype=np.float64) median = float(np.median(power_array)) mad = float(np.median(np.abs(power_array - median))) threshold = median + max(6.0 * mad, median * 0.15, 1.0) active = np.flatnonzero(power_array > threshold) if active.size == 0: return () padding = round(padding_seconds * SAMPLE_RATE_HZ) candidates = ( SampleWindow( max(0, int(index) * block_samples - padding), min(total_samples, (int(index) + 1) * block_samples + padding), ) for index in active ) return _merge_windows(candidates) def count_cs8_clipping(path: Path, *, block_samples: int = DEFAULT_CHUNK_SAMPLES) -> int: """Считает rail-коды во всём файле, сохраняя ограниченный объём памяти.""" cs8_sample_count(path) clipping = 0 with path.open("rb") as stream: while raw_bytes := stream.read(block_samples * 2): values = np.frombuffer(raw_bytes, dtype=np.int8) clipping += int(np.count_nonzero((values == -128) | (values == 127))) return clipping def _is_overlap_duplicate( packets: list[ReceivedIqPacket], candidate: ReceivedIqPacket, ) -> bool: tolerance = samples_per_symbol(SYMBOL_RATE) return any( packet.packet == candidate.packet and abs(packet.start_sample - candidate.start_sample) <= tolerance for packet in packets ) def receive_cs8_windows( path: Path, windows: Iterable[SampleWindow], *, decoder_backend: str = "numba", chunk_samples: int = DEFAULT_CHUNK_SAMPLES, overlap_samples: int | None = None, ) -> StreamingReception: """Принимает заданные окна частями с перекрытием и удаляет повторы.""" selected = _merge_windows(windows) overlap = required_overlap_samples() if overlap_samples is None else overlap_samples if chunk_samples <= overlap or overlap < required_overlap_samples(): raise ValueError("chunk должен быть длиннее полного overlap RF-001") raw = np.memmap(path, dtype=np.int8, mode="r") packets: list[ReceivedIqPacket] = [] candidate_count = 0 rejected_candidates = 0 stage_time: dict[str, float] = {} maximum_loaded = 0 for selected_window in selected: for primary_start in range(selected_window.start, selected_window.stop, chunk_samples): primary_stop = min(selected_window.stop, primary_start + chunk_samples) read_start = max(selected_window.start, primary_start - overlap) read_stop = min(selected_window.stop, primary_stop + overlap) iq = read_cs8_window(raw, read_start, read_stop) maximum_loaded = max(maximum_loaded, int(iq.size)) received = receive_iq_stream( iq, carrier_hz=NOMINAL_CARRIER_HZ, decoder_backend=decoder_backend, ) candidate_count += received.candidate_count rejected_candidates += received.rejected_candidates for name, duration in received.stage_time_s.items(): stage_time[name] = stage_time.get(name, 0.0) + duration for packet in received.packets: global_packet = replace( packet, start_sample=packet.start_sample + read_start, ) if not primary_start <= global_packet.start_sample < primary_stop: continue if not _is_overlap_duplicate(packets, global_packet): packets.append(global_packet) del iq packets.sort(key=lambda packet: packet.start_sample) return StreamingReception( IqStreamResult( packets=tuple(packets), candidate_count=candidate_count, rejected_candidates=rejected_candidates, stage_time_s=stage_time, ), selected, "EXPLICIT", overlap, maximum_loaded, count_cs8_clipping(path), ) def decode_cs8_streaming( path: Path, *, capture_started_at: str | None = None, tx_started_at: str | None = None, tx_finished_at: str | None = None, decoder_backend: str = "numba", chunk_samples: int = DEFAULT_CHUNK_SAMPLES, margin_seconds: float = TIMESTAMP_MARGIN_SECONDS, ) -> StreamingReception: """Выбирает timestamp-окно либо потоковый power locator и запускает приём.""" total_samples = cs8_sample_count(path) timestamp_values = (capture_started_at, tx_started_at, tx_finished_at) if all(value is not None for value in timestamp_values): windows = ( timestamp_sample_window( capture_started_at=str(capture_started_at), tx_started_at=str(tx_started_at), tx_finished_at=str(tx_finished_at), total_samples=total_samples, margin_seconds=margin_seconds, ), ) source = "TX_TIMESTAMPS" elif any(value is not None for value in timestamp_values): raise ValueError("для timestamp-окна нужны все три времени RX/TX") else: windows = locate_power_windows(path) source = "STREAMING_POWER_LOCATOR" if not windows: return StreamingReception( IqStreamResult((), 0, 0, {}), (), source, required_overlap_samples(), 0, count_cs8_clipping(path), ) reception = receive_cs8_windows( path, windows, decoder_backend=decoder_backend, chunk_samples=chunk_samples, ) return replace(reception, window_source=source)