364 lines
14 KiB
Python
364 lines
14 KiB
Python
"""Потоковое чтение больших cs8-захватов и оконный приём RF-001A."""
|
||
|
||
from __future__ import annotations
|
||
|
||
from dataclasses import dataclass, replace
|
||
from datetime import datetime, timezone
|
||
import ctypes
|
||
from pathlib import Path
|
||
import threading
|
||
from typing import Iterable
|
||
|
||
import numpy as np
|
||
|
||
from protocol.video003_modem import SAMPLE_RATE_HZ, build_frame, samples_per_symbol
|
||
from protocol.video005_iq_waveform import NOMINAL_CARRIER_HZ, SYMBOL_RATE, shape_packet
|
||
from protocol.video005_receiver import IqStreamResult, ReceivedIqPacket, receive_iq_stream
|
||
from tools.qpsk_rf_common import PAYLOAD_BYTES, SEED, payload_for_sequence
|
||
|
||
|
||
DEFAULT_CHUNK_SAMPLES = 4_000_000
|
||
LOCATOR_BLOCK_SAMPLES = 400_000
|
||
TIMESTAMP_MARGIN_SECONDS = 2.0
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class SampleWindow:
|
||
start: int
|
||
stop: int
|
||
|
||
@property
|
||
def sample_count(self) -> int:
|
||
return self.stop - self.start
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class StreamingReception:
|
||
reception: IqStreamResult
|
||
windows: tuple[SampleWindow, ...]
|
||
window_source: str
|
||
overlap_samples: int
|
||
maximum_loaded_samples: int
|
||
clipping_codes: int
|
||
|
||
|
||
class MemoryPeakSampler:
|
||
"""Измеряет пиковый working set процесса во время анализа."""
|
||
|
||
def __init__(self, poll_seconds: float = 0.02) -> None:
|
||
self.poll_seconds = poll_seconds
|
||
self.baseline_bytes = current_process_rss_bytes()
|
||
self.peak_bytes = self.baseline_bytes
|
||
self._stop = threading.Event()
|
||
self._thread = threading.Thread(target=self._sample, daemon=True)
|
||
|
||
def _sample(self) -> None:
|
||
while not self._stop.wait(self.poll_seconds):
|
||
self.peak_bytes = max(self.peak_bytes, current_process_rss_bytes())
|
||
|
||
def __enter__(self) -> MemoryPeakSampler:
|
||
self._thread.start()
|
||
return self
|
||
|
||
def __exit__(self, *_exc: object) -> None:
|
||
self.peak_bytes = max(self.peak_bytes, current_process_rss_bytes())
|
||
self._stop.set()
|
||
self._thread.join(timeout=1.0)
|
||
|
||
@property
|
||
def additional_peak_bytes(self) -> int:
|
||
return max(0, self.peak_bytes - self.baseline_bytes)
|
||
|
||
|
||
def current_process_rss_bytes() -> int:
|
||
"""Возвращает текущий working set процесса без сторонних библиотек."""
|
||
|
||
if not hasattr(ctypes, "windll"):
|
||
return 0
|
||
|
||
class ProcessMemoryCounters(ctypes.Structure):
|
||
_fields_ = [
|
||
("cb", ctypes.c_ulong),
|
||
("page_fault_count", ctypes.c_ulong),
|
||
("peak_working_set_size", ctypes.c_size_t),
|
||
("working_set_size", ctypes.c_size_t),
|
||
("quota_peak_paged_pool_usage", ctypes.c_size_t),
|
||
("quota_paged_pool_usage", ctypes.c_size_t),
|
||
("quota_peak_non_paged_pool_usage", ctypes.c_size_t),
|
||
("quota_non_paged_pool_usage", ctypes.c_size_t),
|
||
("pagefile_usage", ctypes.c_size_t),
|
||
("peak_pagefile_usage", ctypes.c_size_t),
|
||
]
|
||
|
||
counters = ProcessMemoryCounters()
|
||
counters.cb = ctypes.sizeof(counters)
|
||
get_current_process = ctypes.windll.kernel32.GetCurrentProcess # type: ignore[attr-defined]
|
||
get_current_process.restype = ctypes.c_void_p
|
||
get_process_memory_info = ctypes.windll.psapi.GetProcessMemoryInfo # type: ignore[attr-defined]
|
||
get_process_memory_info.argtypes = [
|
||
ctypes.c_void_p,
|
||
ctypes.POINTER(ProcessMemoryCounters),
|
||
ctypes.c_ulong,
|
||
]
|
||
get_process_memory_info.restype = ctypes.c_int
|
||
succeeded = get_process_memory_info(
|
||
get_current_process(),
|
||
ctypes.byref(counters),
|
||
counters.cb,
|
||
)
|
||
return int(counters.working_set_size) if succeeded else 0
|
||
|
||
|
||
def cs8_sample_count(path: Path) -> int:
|
||
size = path.stat().st_size
|
||
if size == 0 or size % 2:
|
||
raise ValueError("cs8 должен содержать целое ненулевое число пар I/Q")
|
||
return size // 2
|
||
|
||
|
||
def read_cs8_window(raw: np.memmap, start: int, stop: int) -> np.ndarray:
|
||
"""Преобразует в complex64 только заданное окно cs8."""
|
||
|
||
total_samples = raw.size // 2
|
||
if start < 0 or stop <= start or stop > total_samples:
|
||
raise ValueError("границы окна cs8 выходят за пределы файла")
|
||
values = np.asarray(raw[start * 2:stop * 2], dtype=np.int8)
|
||
real = values[0::2].astype(np.float32)
|
||
imaginary = values[1::2].astype(np.float32)
|
||
return ((real + 1j * imaginary) / 128).astype(np.complex64)
|
||
|
||
|
||
def rf001_packet_waveform_samples() -> int:
|
||
"""Вычисляет длину волны из настоящего 256-байтного пакета RF-001."""
|
||
|
||
if PAYLOAD_BYTES != 256:
|
||
raise AssertionError("канонический RF-001 payload неожиданно изменился")
|
||
payload = payload_for_sequence(SEED, 0)
|
||
frame = build_frame(payload, "QPSK", "2/3", sequence_number=0)
|
||
return int(shape_packet(frame).size)
|
||
|
||
|
||
def required_overlap_samples() -> int:
|
||
"""Перекрывает целую максимальную волну одного пакета RF-001."""
|
||
|
||
return rf001_packet_waveform_samples()
|
||
|
||
|
||
def _parse_utc(value: str) -> datetime:
|
||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||
if parsed.tzinfo is None:
|
||
raise ValueError("timestamp должен содержать часовой пояс")
|
||
return parsed.astimezone(timezone.utc)
|
||
|
||
|
||
def timestamp_sample_window(
|
||
*,
|
||
capture_started_at: str,
|
||
tx_started_at: str,
|
||
tx_finished_at: str,
|
||
total_samples: int,
|
||
sample_rate: int = SAMPLE_RATE_HZ,
|
||
margin_seconds: float = TIMESTAMP_MARGIN_SECONDS,
|
||
) -> SampleWindow:
|
||
"""Переводит UTC-границы TX в безопасное окно отсчётов RX."""
|
||
|
||
if total_samples <= 0 or sample_rate <= 0 or margin_seconds < 0:
|
||
raise ValueError("параметры timestamp-окна должны быть положительными")
|
||
capture = _parse_utc(capture_started_at)
|
||
started = _parse_utc(tx_started_at)
|
||
finished = _parse_utc(tx_finished_at)
|
||
if finished < started:
|
||
raise ValueError("TX_FINISHED_AT не может предшествовать TX_STARTED_AT")
|
||
margin_samples = round(margin_seconds * sample_rate)
|
||
start_offset = round((started - capture).total_seconds() * sample_rate)
|
||
stop_offset = round((finished - capture).total_seconds() * sample_rate)
|
||
start = max(0, start_offset - margin_samples)
|
||
stop = min(total_samples, stop_offset + margin_samples)
|
||
if stop <= start:
|
||
raise ValueError("TX timestamps не пересекаются с RX capture")
|
||
return SampleWindow(start, stop)
|
||
|
||
|
||
def _merge_windows(windows: Iterable[SampleWindow]) -> tuple[SampleWindow, ...]:
|
||
ordered = sorted(windows, key=lambda item: item.start)
|
||
merged: list[SampleWindow] = []
|
||
for window in ordered:
|
||
if not merged or window.start > merged[-1].stop:
|
||
merged.append(window)
|
||
else:
|
||
merged[-1] = SampleWindow(
|
||
merged[-1].start,
|
||
max(merged[-1].stop, window.stop),
|
||
)
|
||
return tuple(merged)
|
||
|
||
|
||
def locate_power_windows(
|
||
path: Path,
|
||
*,
|
||
block_samples: int = LOCATOR_BLOCK_SAMPLES,
|
||
padding_seconds: float = 1.5,
|
||
) -> tuple[SampleWindow, ...]:
|
||
"""Ищет повышенную мощность по всему cs8-файлу ограниченными блоками."""
|
||
|
||
total_samples = cs8_sample_count(path)
|
||
if block_samples <= 0 or padding_seconds < 0:
|
||
raise ValueError("параметры locator должны быть неотрицательными")
|
||
powers: list[float] = []
|
||
with path.open("rb") as stream:
|
||
while raw_bytes := stream.read(block_samples * 2):
|
||
values = np.frombuffer(raw_bytes, dtype=np.int8).astype(np.float32)
|
||
powers.append(float(np.mean(values * values)))
|
||
if not powers:
|
||
return ()
|
||
power_array = np.asarray(powers, dtype=np.float64)
|
||
median = float(np.median(power_array))
|
||
mad = float(np.median(np.abs(power_array - median)))
|
||
threshold = median + max(6.0 * mad, median * 0.15, 1.0)
|
||
active = np.flatnonzero(power_array > threshold)
|
||
if active.size == 0:
|
||
return ()
|
||
padding = round(padding_seconds * SAMPLE_RATE_HZ)
|
||
candidates = (
|
||
SampleWindow(
|
||
max(0, int(index) * block_samples - padding),
|
||
min(total_samples, (int(index) + 1) * block_samples + padding),
|
||
)
|
||
for index in active
|
||
)
|
||
return _merge_windows(candidates)
|
||
|
||
|
||
def count_cs8_clipping(path: Path, *, block_samples: int = DEFAULT_CHUNK_SAMPLES) -> int:
|
||
"""Считает rail-коды во всём файле, сохраняя ограниченный объём памяти."""
|
||
|
||
cs8_sample_count(path)
|
||
clipping = 0
|
||
with path.open("rb") as stream:
|
||
while raw_bytes := stream.read(block_samples * 2):
|
||
values = np.frombuffer(raw_bytes, dtype=np.int8)
|
||
clipping += int(np.count_nonzero((values == -128) | (values == 127)))
|
||
return clipping
|
||
|
||
|
||
def _is_overlap_duplicate(
|
||
packets: list[ReceivedIqPacket],
|
||
candidate: ReceivedIqPacket,
|
||
) -> bool:
|
||
tolerance = samples_per_symbol(SYMBOL_RATE)
|
||
return any(
|
||
packet.packet == candidate.packet
|
||
and abs(packet.start_sample - candidate.start_sample) <= tolerance
|
||
for packet in packets
|
||
)
|
||
|
||
|
||
def receive_cs8_windows(
|
||
path: Path,
|
||
windows: Iterable[SampleWindow],
|
||
*,
|
||
decoder_backend: str = "numba",
|
||
chunk_samples: int = DEFAULT_CHUNK_SAMPLES,
|
||
overlap_samples: int | None = None,
|
||
) -> StreamingReception:
|
||
"""Принимает заданные окна частями с перекрытием и удаляет повторы."""
|
||
|
||
selected = _merge_windows(windows)
|
||
overlap = required_overlap_samples() if overlap_samples is None else overlap_samples
|
||
if chunk_samples <= overlap or overlap < required_overlap_samples():
|
||
raise ValueError("chunk должен быть длиннее полного overlap RF-001")
|
||
raw = np.memmap(path, dtype=np.int8, mode="r")
|
||
packets: list[ReceivedIqPacket] = []
|
||
candidate_count = 0
|
||
rejected_candidates = 0
|
||
stage_time: dict[str, float] = {}
|
||
maximum_loaded = 0
|
||
for selected_window in selected:
|
||
for primary_start in range(selected_window.start, selected_window.stop, chunk_samples):
|
||
primary_stop = min(selected_window.stop, primary_start + chunk_samples)
|
||
read_start = max(selected_window.start, primary_start - overlap)
|
||
read_stop = min(selected_window.stop, primary_stop + overlap)
|
||
iq = read_cs8_window(raw, read_start, read_stop)
|
||
maximum_loaded = max(maximum_loaded, int(iq.size))
|
||
received = receive_iq_stream(
|
||
iq,
|
||
carrier_hz=NOMINAL_CARRIER_HZ,
|
||
decoder_backend=decoder_backend,
|
||
)
|
||
candidate_count += received.candidate_count
|
||
rejected_candidates += received.rejected_candidates
|
||
for name, duration in received.stage_time_s.items():
|
||
stage_time[name] = stage_time.get(name, 0.0) + duration
|
||
for packet in received.packets:
|
||
global_packet = replace(
|
||
packet,
|
||
start_sample=packet.start_sample + read_start,
|
||
)
|
||
if not primary_start <= global_packet.start_sample < primary_stop:
|
||
continue
|
||
if not _is_overlap_duplicate(packets, global_packet):
|
||
packets.append(global_packet)
|
||
del iq
|
||
packets.sort(key=lambda packet: packet.start_sample)
|
||
return StreamingReception(
|
||
IqStreamResult(
|
||
packets=tuple(packets),
|
||
candidate_count=candidate_count,
|
||
rejected_candidates=rejected_candidates,
|
||
stage_time_s=stage_time,
|
||
),
|
||
selected,
|
||
"EXPLICIT",
|
||
overlap,
|
||
maximum_loaded,
|
||
count_cs8_clipping(path),
|
||
)
|
||
|
||
|
||
def decode_cs8_streaming(
|
||
path: Path,
|
||
*,
|
||
capture_started_at: str | None = None,
|
||
tx_started_at: str | None = None,
|
||
tx_finished_at: str | None = None,
|
||
decoder_backend: str = "numba",
|
||
chunk_samples: int = DEFAULT_CHUNK_SAMPLES,
|
||
margin_seconds: float = TIMESTAMP_MARGIN_SECONDS,
|
||
) -> StreamingReception:
|
||
"""Выбирает timestamp-окно либо потоковый power locator и запускает приём."""
|
||
|
||
total_samples = cs8_sample_count(path)
|
||
timestamp_values = (capture_started_at, tx_started_at, tx_finished_at)
|
||
if all(value is not None for value in timestamp_values):
|
||
windows = (
|
||
timestamp_sample_window(
|
||
capture_started_at=str(capture_started_at),
|
||
tx_started_at=str(tx_started_at),
|
||
tx_finished_at=str(tx_finished_at),
|
||
total_samples=total_samples,
|
||
margin_seconds=margin_seconds,
|
||
),
|
||
)
|
||
source = "TX_TIMESTAMPS"
|
||
elif any(value is not None for value in timestamp_values):
|
||
raise ValueError("для timestamp-окна нужны все три времени RX/TX")
|
||
else:
|
||
windows = locate_power_windows(path)
|
||
source = "STREAMING_POWER_LOCATOR"
|
||
if not windows:
|
||
return StreamingReception(
|
||
IqStreamResult((), 0, 0, {}),
|
||
(),
|
||
source,
|
||
required_overlap_samples(),
|
||
0,
|
||
count_cs8_clipping(path),
|
||
)
|
||
reception = receive_cs8_windows(
|
||
path,
|
||
windows,
|
||
decoder_backend=decoder_backend,
|
||
chunk_samples=chunk_samples,
|
||
)
|
||
return replace(reception, window_source=source)
|