Harden RF handoff and stream OTA decode
This commit is contained in:
LittleSam129
363
tools/rf001a_streaming.py
Normal file
363
tools/rf001a_streaming.py
Normal file
@@ -0,0 +1,363 @@
|
||||
"""Потоковое чтение больших cs8-захватов и оконный приём RF-001A."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime, timezone
|
||||
import ctypes
|
||||
from pathlib import Path
|
||||
import threading
|
||||
from typing import Iterable
|
||||
|
||||
import numpy as np
|
||||
|
||||
from protocol.video003_modem import SAMPLE_RATE_HZ, build_frame, samples_per_symbol
|
||||
from protocol.video005_iq_waveform import NOMINAL_CARRIER_HZ, SYMBOL_RATE, shape_packet
|
||||
from protocol.video005_receiver import IqStreamResult, ReceivedIqPacket, receive_iq_stream
|
||||
from tools.qpsk_rf_common import PAYLOAD_BYTES, SEED, payload_for_sequence
|
||||
|
||||
|
||||
DEFAULT_CHUNK_SAMPLES = 4_000_000
|
||||
LOCATOR_BLOCK_SAMPLES = 400_000
|
||||
TIMESTAMP_MARGIN_SECONDS = 2.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SampleWindow:
|
||||
start: int
|
||||
stop: int
|
||||
|
||||
@property
|
||||
def sample_count(self) -> int:
|
||||
return self.stop - self.start
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StreamingReception:
|
||||
reception: IqStreamResult
|
||||
windows: tuple[SampleWindow, ...]
|
||||
window_source: str
|
||||
overlap_samples: int
|
||||
maximum_loaded_samples: int
|
||||
clipping_codes: int
|
||||
|
||||
|
||||
class MemoryPeakSampler:
|
||||
"""Измеряет пиковый working set процесса во время анализа."""
|
||||
|
||||
def __init__(self, poll_seconds: float = 0.02) -> None:
|
||||
self.poll_seconds = poll_seconds
|
||||
self.baseline_bytes = current_process_rss_bytes()
|
||||
self.peak_bytes = self.baseline_bytes
|
||||
self._stop = threading.Event()
|
||||
self._thread = threading.Thread(target=self._sample, daemon=True)
|
||||
|
||||
def _sample(self) -> None:
|
||||
while not self._stop.wait(self.poll_seconds):
|
||||
self.peak_bytes = max(self.peak_bytes, current_process_rss_bytes())
|
||||
|
||||
def __enter__(self) -> MemoryPeakSampler:
|
||||
self._thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *_exc: object) -> None:
|
||||
self.peak_bytes = max(self.peak_bytes, current_process_rss_bytes())
|
||||
self._stop.set()
|
||||
self._thread.join(timeout=1.0)
|
||||
|
||||
@property
|
||||
def additional_peak_bytes(self) -> int:
|
||||
return max(0, self.peak_bytes - self.baseline_bytes)
|
||||
|
||||
|
||||
def current_process_rss_bytes() -> int:
|
||||
"""Возвращает текущий working set процесса без сторонних библиотек."""
|
||||
|
||||
if not hasattr(ctypes, "windll"):
|
||||
return 0
|
||||
|
||||
class ProcessMemoryCounters(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("cb", ctypes.c_ulong),
|
||||
("page_fault_count", ctypes.c_ulong),
|
||||
("peak_working_set_size", ctypes.c_size_t),
|
||||
("working_set_size", ctypes.c_size_t),
|
||||
("quota_peak_paged_pool_usage", ctypes.c_size_t),
|
||||
("quota_paged_pool_usage", ctypes.c_size_t),
|
||||
("quota_peak_non_paged_pool_usage", ctypes.c_size_t),
|
||||
("quota_non_paged_pool_usage", ctypes.c_size_t),
|
||||
("pagefile_usage", ctypes.c_size_t),
|
||||
("peak_pagefile_usage", ctypes.c_size_t),
|
||||
]
|
||||
|
||||
counters = ProcessMemoryCounters()
|
||||
counters.cb = ctypes.sizeof(counters)
|
||||
get_current_process = ctypes.windll.kernel32.GetCurrentProcess # type: ignore[attr-defined]
|
||||
get_current_process.restype = ctypes.c_void_p
|
||||
get_process_memory_info = ctypes.windll.psapi.GetProcessMemoryInfo # type: ignore[attr-defined]
|
||||
get_process_memory_info.argtypes = [
|
||||
ctypes.c_void_p,
|
||||
ctypes.POINTER(ProcessMemoryCounters),
|
||||
ctypes.c_ulong,
|
||||
]
|
||||
get_process_memory_info.restype = ctypes.c_int
|
||||
succeeded = get_process_memory_info(
|
||||
get_current_process(),
|
||||
ctypes.byref(counters),
|
||||
counters.cb,
|
||||
)
|
||||
return int(counters.working_set_size) if succeeded else 0
|
||||
|
||||
|
||||
def cs8_sample_count(path: Path) -> int:
|
||||
size = path.stat().st_size
|
||||
if size == 0 or size % 2:
|
||||
raise ValueError("cs8 должен содержать целое ненулевое число пар I/Q")
|
||||
return size // 2
|
||||
|
||||
|
||||
def read_cs8_window(raw: np.memmap, start: int, stop: int) -> np.ndarray:
|
||||
"""Преобразует в complex64 только заданное окно cs8."""
|
||||
|
||||
total_samples = raw.size // 2
|
||||
if start < 0 or stop <= start or stop > total_samples:
|
||||
raise ValueError("границы окна cs8 выходят за пределы файла")
|
||||
values = np.asarray(raw[start * 2:stop * 2], dtype=np.int8)
|
||||
real = values[0::2].astype(np.float32)
|
||||
imaginary = values[1::2].astype(np.float32)
|
||||
return ((real + 1j * imaginary) / 128).astype(np.complex64)
|
||||
|
||||
|
||||
def rf001_packet_waveform_samples() -> int:
|
||||
"""Вычисляет длину волны из настоящего 256-байтного пакета RF-001."""
|
||||
|
||||
if PAYLOAD_BYTES != 256:
|
||||
raise AssertionError("канонический RF-001 payload неожиданно изменился")
|
||||
payload = payload_for_sequence(SEED, 0)
|
||||
frame = build_frame(payload, "QPSK", "2/3", sequence_number=0)
|
||||
return int(shape_packet(frame).size)
|
||||
|
||||
|
||||
def required_overlap_samples() -> int:
|
||||
"""Перекрывает целую максимальную волну одного пакета RF-001."""
|
||||
|
||||
return rf001_packet_waveform_samples()
|
||||
|
||||
|
||||
def _parse_utc(value: str) -> datetime:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
if parsed.tzinfo is None:
|
||||
raise ValueError("timestamp должен содержать часовой пояс")
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def timestamp_sample_window(
|
||||
*,
|
||||
capture_started_at: str,
|
||||
tx_started_at: str,
|
||||
tx_finished_at: str,
|
||||
total_samples: int,
|
||||
sample_rate: int = SAMPLE_RATE_HZ,
|
||||
margin_seconds: float = TIMESTAMP_MARGIN_SECONDS,
|
||||
) -> SampleWindow:
|
||||
"""Переводит UTC-границы TX в безопасное окно отсчётов RX."""
|
||||
|
||||
if total_samples <= 0 or sample_rate <= 0 or margin_seconds < 0:
|
||||
raise ValueError("параметры timestamp-окна должны быть положительными")
|
||||
capture = _parse_utc(capture_started_at)
|
||||
started = _parse_utc(tx_started_at)
|
||||
finished = _parse_utc(tx_finished_at)
|
||||
if finished < started:
|
||||
raise ValueError("TX_FINISHED_AT не может предшествовать TX_STARTED_AT")
|
||||
margin_samples = round(margin_seconds * sample_rate)
|
||||
start_offset = round((started - capture).total_seconds() * sample_rate)
|
||||
stop_offset = round((finished - capture).total_seconds() * sample_rate)
|
||||
start = max(0, start_offset - margin_samples)
|
||||
stop = min(total_samples, stop_offset + margin_samples)
|
||||
if stop <= start:
|
||||
raise ValueError("TX timestamps не пересекаются с RX capture")
|
||||
return SampleWindow(start, stop)
|
||||
|
||||
|
||||
def _merge_windows(windows: Iterable[SampleWindow]) -> tuple[SampleWindow, ...]:
|
||||
ordered = sorted(windows, key=lambda item: item.start)
|
||||
merged: list[SampleWindow] = []
|
||||
for window in ordered:
|
||||
if not merged or window.start > merged[-1].stop:
|
||||
merged.append(window)
|
||||
else:
|
||||
merged[-1] = SampleWindow(
|
||||
merged[-1].start,
|
||||
max(merged[-1].stop, window.stop),
|
||||
)
|
||||
return tuple(merged)
|
||||
|
||||
|
||||
def locate_power_windows(
|
||||
path: Path,
|
||||
*,
|
||||
block_samples: int = LOCATOR_BLOCK_SAMPLES,
|
||||
padding_seconds: float = 1.5,
|
||||
) -> tuple[SampleWindow, ...]:
|
||||
"""Ищет повышенную мощность по всему cs8-файлу ограниченными блоками."""
|
||||
|
||||
total_samples = cs8_sample_count(path)
|
||||
if block_samples <= 0 or padding_seconds < 0:
|
||||
raise ValueError("параметры locator должны быть неотрицательными")
|
||||
powers: list[float] = []
|
||||
with path.open("rb") as stream:
|
||||
while raw_bytes := stream.read(block_samples * 2):
|
||||
values = np.frombuffer(raw_bytes, dtype=np.int8).astype(np.float32)
|
||||
powers.append(float(np.mean(values * values)))
|
||||
if not powers:
|
||||
return ()
|
||||
power_array = np.asarray(powers, dtype=np.float64)
|
||||
median = float(np.median(power_array))
|
||||
mad = float(np.median(np.abs(power_array - median)))
|
||||
threshold = median + max(6.0 * mad, median * 0.15, 1.0)
|
||||
active = np.flatnonzero(power_array > threshold)
|
||||
if active.size == 0:
|
||||
return ()
|
||||
padding = round(padding_seconds * SAMPLE_RATE_HZ)
|
||||
candidates = (
|
||||
SampleWindow(
|
||||
max(0, int(index) * block_samples - padding),
|
||||
min(total_samples, (int(index) + 1) * block_samples + padding),
|
||||
)
|
||||
for index in active
|
||||
)
|
||||
return _merge_windows(candidates)
|
||||
|
||||
|
||||
def count_cs8_clipping(path: Path, *, block_samples: int = DEFAULT_CHUNK_SAMPLES) -> int:
|
||||
"""Считает rail-коды во всём файле, сохраняя ограниченный объём памяти."""
|
||||
|
||||
cs8_sample_count(path)
|
||||
clipping = 0
|
||||
with path.open("rb") as stream:
|
||||
while raw_bytes := stream.read(block_samples * 2):
|
||||
values = np.frombuffer(raw_bytes, dtype=np.int8)
|
||||
clipping += int(np.count_nonzero((values == -128) | (values == 127)))
|
||||
return clipping
|
||||
|
||||
|
||||
def _is_overlap_duplicate(
|
||||
packets: list[ReceivedIqPacket],
|
||||
candidate: ReceivedIqPacket,
|
||||
) -> bool:
|
||||
tolerance = samples_per_symbol(SYMBOL_RATE)
|
||||
return any(
|
||||
packet.packet == candidate.packet
|
||||
and abs(packet.start_sample - candidate.start_sample) <= tolerance
|
||||
for packet in packets
|
||||
)
|
||||
|
||||
|
||||
def receive_cs8_windows(
|
||||
path: Path,
|
||||
windows: Iterable[SampleWindow],
|
||||
*,
|
||||
decoder_backend: str = "numba",
|
||||
chunk_samples: int = DEFAULT_CHUNK_SAMPLES,
|
||||
overlap_samples: int | None = None,
|
||||
) -> StreamingReception:
|
||||
"""Принимает заданные окна частями с перекрытием и удаляет повторы."""
|
||||
|
||||
selected = _merge_windows(windows)
|
||||
overlap = required_overlap_samples() if overlap_samples is None else overlap_samples
|
||||
if chunk_samples <= overlap or overlap < required_overlap_samples():
|
||||
raise ValueError("chunk должен быть длиннее полного overlap RF-001")
|
||||
raw = np.memmap(path, dtype=np.int8, mode="r")
|
||||
packets: list[ReceivedIqPacket] = []
|
||||
candidate_count = 0
|
||||
rejected_candidates = 0
|
||||
stage_time: dict[str, float] = {}
|
||||
maximum_loaded = 0
|
||||
for selected_window in selected:
|
||||
for primary_start in range(selected_window.start, selected_window.stop, chunk_samples):
|
||||
primary_stop = min(selected_window.stop, primary_start + chunk_samples)
|
||||
read_start = max(selected_window.start, primary_start - overlap)
|
||||
read_stop = min(selected_window.stop, primary_stop + overlap)
|
||||
iq = read_cs8_window(raw, read_start, read_stop)
|
||||
maximum_loaded = max(maximum_loaded, int(iq.size))
|
||||
received = receive_iq_stream(
|
||||
iq,
|
||||
carrier_hz=NOMINAL_CARRIER_HZ,
|
||||
decoder_backend=decoder_backend,
|
||||
)
|
||||
candidate_count += received.candidate_count
|
||||
rejected_candidates += received.rejected_candidates
|
||||
for name, duration in received.stage_time_s.items():
|
||||
stage_time[name] = stage_time.get(name, 0.0) + duration
|
||||
for packet in received.packets:
|
||||
global_packet = replace(
|
||||
packet,
|
||||
start_sample=packet.start_sample + read_start,
|
||||
)
|
||||
if not primary_start <= global_packet.start_sample < primary_stop:
|
||||
continue
|
||||
if not _is_overlap_duplicate(packets, global_packet):
|
||||
packets.append(global_packet)
|
||||
del iq
|
||||
packets.sort(key=lambda packet: packet.start_sample)
|
||||
return StreamingReception(
|
||||
IqStreamResult(
|
||||
packets=tuple(packets),
|
||||
candidate_count=candidate_count,
|
||||
rejected_candidates=rejected_candidates,
|
||||
stage_time_s=stage_time,
|
||||
),
|
||||
selected,
|
||||
"EXPLICIT",
|
||||
overlap,
|
||||
maximum_loaded,
|
||||
count_cs8_clipping(path),
|
||||
)
|
||||
|
||||
|
||||
def decode_cs8_streaming(
|
||||
path: Path,
|
||||
*,
|
||||
capture_started_at: str | None = None,
|
||||
tx_started_at: str | None = None,
|
||||
tx_finished_at: str | None = None,
|
||||
decoder_backend: str = "numba",
|
||||
chunk_samples: int = DEFAULT_CHUNK_SAMPLES,
|
||||
margin_seconds: float = TIMESTAMP_MARGIN_SECONDS,
|
||||
) -> StreamingReception:
|
||||
"""Выбирает timestamp-окно либо потоковый power locator и запускает приём."""
|
||||
|
||||
total_samples = cs8_sample_count(path)
|
||||
timestamp_values = (capture_started_at, tx_started_at, tx_finished_at)
|
||||
if all(value is not None for value in timestamp_values):
|
||||
windows = (
|
||||
timestamp_sample_window(
|
||||
capture_started_at=str(capture_started_at),
|
||||
tx_started_at=str(tx_started_at),
|
||||
tx_finished_at=str(tx_finished_at),
|
||||
total_samples=total_samples,
|
||||
margin_seconds=margin_seconds,
|
||||
),
|
||||
)
|
||||
source = "TX_TIMESTAMPS"
|
||||
elif any(value is not None for value in timestamp_values):
|
||||
raise ValueError("для timestamp-окна нужны все три времени RX/TX")
|
||||
else:
|
||||
windows = locate_power_windows(path)
|
||||
source = "STREAMING_POWER_LOCATOR"
|
||||
if not windows:
|
||||
return StreamingReception(
|
||||
IqStreamResult((), 0, 0, {}),
|
||||
(),
|
||||
source,
|
||||
required_overlap_samples(),
|
||||
0,
|
||||
count_cs8_clipping(path),
|
||||
)
|
||||
reception = receive_cs8_windows(
|
||||
path,
|
||||
windows,
|
||||
decoder_backend=decoder_backend,
|
||||
chunk_samples=chunk_samples,
|
||||
)
|
||||
return replace(reception, window_source=source)
|
||||
Reference in New Issue
Block a user