Harden RF node control and agent updates
This commit is contained in:
LittleSam129
937
tools/rf001_staircase_search.py
Normal file
937
tools/rf001_staircase_search.py
Normal file
@@ -0,0 +1,937 @@
|
||||
"""Потоковый поиск эталонной лестницы Lab064/Lab065 в большом cs8-файле.
|
||||
|
||||
Утилита не обращается к SDR, не использует пакетный приёмник и не меняет
|
||||
исходные IQ-файлы. Полная запись просматривается блоками с перекрытием.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
from dataclasses import asdict, dataclass
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import threading
|
||||
import time
|
||||
from typing import Callable, Iterable
|
||||
|
||||
import numpy as np
|
||||
from scipy import signal
|
||||
|
||||
EXPECTED_REFERENCE_SHA256 = (
|
||||
"fa260719659b2984fe29588238415622cd75c6daefec6cc9130ba1bf851bd3f9"
|
||||
)
|
||||
ANALYZER_VERSION = "2026.09.28.1"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReferenceLayout:
|
||||
sample_rate_hz: int = 8_000_000
|
||||
symbol_rate: int = 20_000
|
||||
carrier_hz: float = 50_000.0
|
||||
leading_samples: int = 800_000
|
||||
segment_samples: int = 800_000
|
||||
gap_samples: int = 400_000
|
||||
trailing_samples: int = 800_000
|
||||
levels_db: tuple[float, ...] = (-18.0, -12.0, -6.0, 0.0)
|
||||
|
||||
@property
|
||||
def total_samples(self) -> int:
|
||||
return (
|
||||
self.leading_samples
|
||||
+ len(self.levels_db) * self.segment_samples
|
||||
+ (len(self.levels_db) - 1) * self.gap_samples
|
||||
+ self.trailing_samples
|
||||
)
|
||||
|
||||
def active_windows(self) -> tuple[tuple[int, int], ...]:
|
||||
cursor = self.leading_samples
|
||||
windows = []
|
||||
for index in range(len(self.levels_db)):
|
||||
windows.append((cursor, cursor + self.segment_samples))
|
||||
cursor += self.segment_samples
|
||||
if index + 1 < len(self.levels_db):
|
||||
cursor += self.gap_samples
|
||||
return tuple(windows)
|
||||
|
||||
def background_windows(self) -> tuple[tuple[int, int], ...]:
|
||||
width = min(self.gap_samples, self.leading_samples, self.trailing_samples) // 2
|
||||
cursor = (self.leading_samples - width) // 2
|
||||
windows = [(cursor, cursor + width)]
|
||||
cursor = self.leading_samples + self.segment_samples
|
||||
for index in range(len(self.levels_db) - 1):
|
||||
start = cursor + (self.gap_samples - width) // 2
|
||||
windows.append((start, start + width))
|
||||
cursor += self.gap_samples + self.segment_samples
|
||||
final_start = self.total_samples - self.trailing_samples
|
||||
start = final_start + (self.trailing_samples - width) // 2
|
||||
windows.append((start, start + width))
|
||||
return tuple(windows)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SearchCriteria:
|
||||
decimation: int = 32
|
||||
input_block_samples: int = 4_000_000
|
||||
anti_alias_passband_hz: float = 75_000.0
|
||||
anti_alias_stopband_hz: float = 120_000.0
|
||||
anti_alias_attenuation_db: float = 80.0
|
||||
feature_period_seconds: float = 0.001
|
||||
refinement_radius_seconds: float = 0.003
|
||||
cfo_min_hz: float = -3.0
|
||||
cfo_max_hz: float = 3.0
|
||||
cfo_step_hz: float = 0.25
|
||||
normalized_correlation_minimum: float = 0.05
|
||||
peak_to_background_minimum_db: float = 6.0
|
||||
maximum_step_error_db: float = 3.0
|
||||
l3_matched_to_residual_minimum_db: float = -18.0
|
||||
l3_spectral_contrast_minimum_db: float = 3.0
|
||||
histogram_stride: int = 1024
|
||||
maximum_reported_candidates: int = 5
|
||||
|
||||
|
||||
DEFAULT_LAYOUT = ReferenceLayout()
|
||||
DEFAULT_CRITERIA = SearchCriteria()
|
||||
|
||||
|
||||
def current_process_rss_bytes() -> int:
|
||||
if not hasattr(ctypes, "windll"):
|
||||
return 0
|
||||
|
||||
class ProcessMemoryCounters(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("cb", ctypes.c_ulong),
|
||||
("page_fault_count", ctypes.c_ulong),
|
||||
("peak_working_set_size", ctypes.c_size_t),
|
||||
("working_set_size", ctypes.c_size_t),
|
||||
("quota_peak_paged_pool_usage", ctypes.c_size_t),
|
||||
("quota_paged_pool_usage", ctypes.c_size_t),
|
||||
("quota_peak_non_paged_pool_usage", ctypes.c_size_t),
|
||||
("quota_non_paged_pool_usage", ctypes.c_size_t),
|
||||
("pagefile_usage", ctypes.c_size_t),
|
||||
("peak_pagefile_usage", ctypes.c_size_t),
|
||||
]
|
||||
|
||||
counters = ProcessMemoryCounters()
|
||||
counters.cb = ctypes.sizeof(counters)
|
||||
get_current_process = ctypes.windll.kernel32.GetCurrentProcess # type: ignore[attr-defined]
|
||||
get_current_process.restype = ctypes.c_void_p
|
||||
get_process_memory_info = ctypes.windll.psapi.GetProcessMemoryInfo # type: ignore[attr-defined]
|
||||
get_process_memory_info.argtypes = [
|
||||
ctypes.c_void_p,
|
||||
ctypes.POINTER(ProcessMemoryCounters),
|
||||
ctypes.c_ulong,
|
||||
]
|
||||
get_process_memory_info.restype = ctypes.c_int
|
||||
succeeded = get_process_memory_info(
|
||||
get_current_process(),
|
||||
ctypes.byref(counters),
|
||||
counters.cb,
|
||||
)
|
||||
return int(counters.working_set_size) if succeeded else 0
|
||||
|
||||
|
||||
class MemoryPeakSampler:
|
||||
def __init__(self, poll_seconds: float = 0.02) -> None:
|
||||
self.poll_seconds = poll_seconds
|
||||
self.baseline_bytes = current_process_rss_bytes()
|
||||
self.peak_bytes = self.baseline_bytes
|
||||
self._stop = threading.Event()
|
||||
self._thread = threading.Thread(target=self._sample, daemon=True)
|
||||
|
||||
def _sample(self) -> None:
|
||||
while not self._stop.wait(self.poll_seconds):
|
||||
self.peak_bytes = max(self.peak_bytes, current_process_rss_bytes())
|
||||
|
||||
def __enter__(self) -> "MemoryPeakSampler":
|
||||
self._thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *_exc: object) -> None:
|
||||
self.peak_bytes = max(self.peak_bytes, current_process_rss_bytes())
|
||||
self._stop.set()
|
||||
self._thread.join(timeout=1.0)
|
||||
|
||||
@property
|
||||
def additional_peak_bytes(self) -> int:
|
||||
return max(0, self.peak_bytes - self.baseline_bytes)
|
||||
|
||||
|
||||
def _write_progress(
|
||||
path: Path,
|
||||
*,
|
||||
phase: str,
|
||||
processed_samples: int,
|
||||
total_samples: int,
|
||||
) -> None:
|
||||
value = {
|
||||
"schema_version": 1,
|
||||
"phase": phase,
|
||||
"processed_samples": processed_samples,
|
||||
"total_samples": total_samples,
|
||||
"coverage_complete": (
|
||||
phase == "COMPLETE" and processed_samples == total_samples
|
||||
),
|
||||
"updated_at_epoch": time.time(),
|
||||
"hardware_accessed": False,
|
||||
"rf_tx": 0,
|
||||
}
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||||
temporary.write_text(
|
||||
json.dumps(value, separators=(",", ":"), sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.replace(temporary, path)
|
||||
|
||||
|
||||
def sha256_file(
|
||||
path: Path,
|
||||
progress: Callable[[int, int], None] | None = None,
|
||||
) -> str:
|
||||
digest = hashlib.sha256()
|
||||
total = path.stat().st_size
|
||||
processed = 0
|
||||
with path.open("rb") as stream:
|
||||
while block := stream.read(4 * 1024 * 1024):
|
||||
digest.update(block)
|
||||
processed += len(block)
|
||||
if progress is not None:
|
||||
progress(processed, total)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def cs8_sample_count(path: Path) -> int:
|
||||
size = path.stat().st_size
|
||||
if size <= 0 or size % 2:
|
||||
raise ValueError("cs8-файл должен содержать целое ненулевое число пар I/Q")
|
||||
return size // 2
|
||||
|
||||
|
||||
def design_antialias_filter(
|
||||
layout: ReferenceLayout,
|
||||
criteria: SearchCriteria,
|
||||
) -> np.ndarray:
|
||||
"""Build the fixed linear-phase FIR used before every decimation."""
|
||||
|
||||
if criteria.decimation <= 0:
|
||||
raise ValueError("decimation must be positive")
|
||||
if criteria.decimation == 1:
|
||||
return np.ones(1, dtype=np.float64)
|
||||
nyquist = layout.sample_rate_hz / 2
|
||||
decimated_nyquist = layout.sample_rate_hz / (2 * criteria.decimation)
|
||||
if not (
|
||||
0 < criteria.anti_alias_passband_hz
|
||||
< criteria.anti_alias_stopband_hz
|
||||
< decimated_nyquist
|
||||
):
|
||||
raise ValueError("anti-alias passband and stopband are invalid")
|
||||
transition_width = (
|
||||
criteria.anti_alias_stopband_hz - criteria.anti_alias_passband_hz
|
||||
) / nyquist
|
||||
taps_count, beta = signal.kaiserord(
|
||||
criteria.anti_alias_attenuation_db,
|
||||
transition_width,
|
||||
)
|
||||
taps_count = max(3, taps_count | 1)
|
||||
cutoff_hz = (
|
||||
criteria.anti_alias_passband_hz + criteria.anti_alias_stopband_hz
|
||||
) / 2
|
||||
return signal.firwin(
|
||||
taps_count,
|
||||
cutoff_hz,
|
||||
window=("kaiser", beta),
|
||||
fs=layout.sample_rate_hz,
|
||||
).astype(np.float64)
|
||||
|
||||
|
||||
def _read_cs8_complex(path: Path, start: int, stop: int) -> np.ndarray:
|
||||
"""Read one bounded source interval; out-of-file samples are zero."""
|
||||
|
||||
if start >= stop:
|
||||
raise ValueError("invalid source interval")
|
||||
total = cs8_sample_count(path)
|
||||
values = np.zeros(stop - start, dtype=np.complex64)
|
||||
clipped_start = max(0, start)
|
||||
clipped_stop = min(total, stop)
|
||||
if clipped_start >= clipped_stop:
|
||||
return values
|
||||
raw = np.memmap(path, dtype=np.int8, mode="r")
|
||||
interleaved = np.asarray(
|
||||
raw[2 * clipped_start:2 * clipped_stop],
|
||||
dtype=np.float32,
|
||||
)
|
||||
destination = slice(clipped_start - start, clipped_stop - start)
|
||||
values[destination] = interleaved[0::2] + 1j * interleaved[1::2]
|
||||
return values
|
||||
|
||||
|
||||
def read_filtered_decimated_window(
|
||||
path: Path,
|
||||
start: int,
|
||||
stop: int,
|
||||
layout: ReferenceLayout,
|
||||
criteria: SearchCriteria,
|
||||
taps: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
"""Read a decimated window with FIR overlap and delay correction.
|
||||
|
||||
Output index ``k`` is centred on input sample ``k * decimation``. The
|
||||
function reads half the FIR on both sides, so adjacent blocks have exactly
|
||||
the overlap required to match one full-file convolution. Zeros are used
|
||||
only outside the physical file.
|
||||
"""
|
||||
|
||||
total = cs8_sample_count(path)
|
||||
total_decimated = math.ceil(total / criteria.decimation)
|
||||
if not 0 <= start < stop <= total_decimated:
|
||||
raise ValueError("invalid filtered-decimated window")
|
||||
delay = (taps.size - 1) // 2
|
||||
first_center = start * criteria.decimation
|
||||
last_center = (stop - 1) * criteria.decimation
|
||||
source_start = first_center - delay
|
||||
source_stop = last_center + delay + 1
|
||||
values = _read_cs8_complex(path, source_start, source_stop)
|
||||
if taps.size == 1:
|
||||
return values[::criteria.decimation].astype(np.complex64, copy=False)
|
||||
filtered = signal.fftconvolve(values, taps, mode="valid")
|
||||
result = filtered[::criteria.decimation]
|
||||
if result.size != stop - start:
|
||||
raise RuntimeError("filtered decimator produced an unexpected length")
|
||||
return result.astype(np.complex64, copy=False)
|
||||
|
||||
|
||||
def iter_filtered_decimated(
|
||||
path: Path,
|
||||
layout: ReferenceLayout,
|
||||
criteria: SearchCriteria,
|
||||
taps: np.ndarray,
|
||||
) -> Iterable[tuple[int, np.ndarray]]:
|
||||
"""Yield the full file in bounded, boundary-correct decimated blocks."""
|
||||
|
||||
if criteria.input_block_samples <= 0:
|
||||
raise ValueError("input block size must be positive")
|
||||
total = cs8_sample_count(path)
|
||||
total_decimated = math.ceil(total / criteria.decimation)
|
||||
block = max(1, criteria.input_block_samples // criteria.decimation)
|
||||
for start in range(0, total_decimated, block):
|
||||
stop = min(total_decimated, start + block)
|
||||
yield start, read_filtered_decimated_window(
|
||||
path,
|
||||
start,
|
||||
stop,
|
||||
layout,
|
||||
criteria,
|
||||
taps,
|
||||
)
|
||||
|
||||
|
||||
def load_reference(
|
||||
path: Path,
|
||||
layout: ReferenceLayout,
|
||||
criteria: SearchCriteria,
|
||||
*,
|
||||
expected_sha256: str | None,
|
||||
) -> tuple[np.ndarray, str]:
|
||||
if cs8_sample_count(path) != layout.total_samples:
|
||||
raise ValueError("размер эталона не соответствует фиксированной структуре")
|
||||
digest = sha256_file(path)
|
||||
if expected_sha256 is not None and digest != expected_sha256:
|
||||
raise ValueError("SHA256 эталона не совпадает с разрешённым значением")
|
||||
stop = math.ceil(layout.total_samples / criteria.decimation)
|
||||
taps = design_antialias_filter(layout, criteria)
|
||||
reference = read_filtered_decimated_window(
|
||||
path,
|
||||
0,
|
||||
stop,
|
||||
layout,
|
||||
criteria,
|
||||
taps,
|
||||
)
|
||||
reference -= np.mean(reference, dtype=np.complex128)
|
||||
return reference, digest
|
||||
|
||||
|
||||
def _frequency_grid(criteria: SearchCriteria) -> np.ndarray:
|
||||
count = round(
|
||||
(criteria.cfo_max_hz - criteria.cfo_min_hz) / criteria.cfo_step_hz
|
||||
)
|
||||
return criteria.cfo_min_hz + criteria.cfo_step_hz * np.arange(count + 1)
|
||||
|
||||
|
||||
def _histogram_quantile(histogram: np.ndarray, quantile: float) -> float:
|
||||
total = int(histogram.sum())
|
||||
if total == 0:
|
||||
return 0.0
|
||||
target = math.ceil(total * quantile)
|
||||
index = int(np.searchsorted(np.cumsum(histogram), target, side="left"))
|
||||
return min(1.0, (index + 0.5) / histogram.size)
|
||||
|
||||
|
||||
def _deduplicate_candidates(
|
||||
candidates: Iterable[dict[str, float | int]],
|
||||
minimum_separation: int,
|
||||
limit: int,
|
||||
) -> list[dict[str, float | int]]:
|
||||
selected: list[dict[str, float | int]] = []
|
||||
for candidate in sorted(candidates, key=lambda row: float(row["score"]), reverse=True):
|
||||
start = int(candidate["start_decimated_sample"])
|
||||
if any(
|
||||
abs(start - int(existing["start_decimated_sample"])) < minimum_separation
|
||||
for existing in selected
|
||||
):
|
||||
continue
|
||||
selected.append(candidate)
|
||||
if len(selected) == limit:
|
||||
break
|
||||
return selected
|
||||
|
||||
|
||||
def _power_features(values: np.ndarray, stride: int) -> np.ndarray:
|
||||
complete = values.size // stride
|
||||
if complete == 0:
|
||||
return np.empty(0, dtype=np.float64)
|
||||
reshaped = values[:complete * stride].reshape(complete, stride)
|
||||
return np.mean(np.abs(reshaped) ** 2, axis=1, dtype=np.float64)
|
||||
|
||||
|
||||
def _normalized_sliding_scores(
|
||||
values: np.ndarray,
|
||||
template: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
centred = template.astype(np.float64) - float(np.mean(template))
|
||||
template_energy = float(np.vdot(centred, centred).real)
|
||||
count = centred.size
|
||||
prefix = np.concatenate(([0.0], np.cumsum(values, dtype=np.float64)))
|
||||
prefix_power = np.concatenate(
|
||||
([0.0], np.cumsum(values * values, dtype=np.float64))
|
||||
)
|
||||
window_sum = prefix[count:] - prefix[:-count]
|
||||
window_power = prefix_power[count:] - prefix_power[:-count]
|
||||
ac_power = window_power - (window_sum * window_sum) / count
|
||||
denominator = np.sqrt(np.maximum(template_energy * ac_power, 1e-30))
|
||||
correlation = signal.fftconvolve(values, centred[::-1], mode="valid")
|
||||
return np.abs(correlation) / denominator
|
||||
|
||||
|
||||
def _refine_candidate(
|
||||
capture_path: Path,
|
||||
reference: np.ndarray,
|
||||
approximate_start: int,
|
||||
layout: ReferenceLayout,
|
||||
criteria: SearchCriteria,
|
||||
taps: np.ndarray,
|
||||
) -> dict[str, float | int]:
|
||||
decimated_rate = layout.sample_rate_hz / criteria.decimation
|
||||
radius = max(1, round(criteria.refinement_radius_seconds * decimated_rate))
|
||||
total_decimated = math.ceil(
|
||||
cs8_sample_count(capture_path) / criteria.decimation
|
||||
)
|
||||
first = max(0, approximate_start - radius)
|
||||
last = min(total_decimated - reference.size, approximate_start + radius)
|
||||
received = read_filtered_decimated_window(
|
||||
capture_path,
|
||||
first,
|
||||
last + reference.size,
|
||||
layout,
|
||||
criteria,
|
||||
taps,
|
||||
)
|
||||
count = reference.size
|
||||
prefix_power = np.concatenate(
|
||||
([0.0], np.cumsum(np.abs(received) ** 2, dtype=np.float64))
|
||||
)
|
||||
prefix_sum = np.concatenate(
|
||||
([0.0 + 0.0j], np.cumsum(received, dtype=np.complex128))
|
||||
)
|
||||
window_power = prefix_power[count:] - prefix_power[:-count]
|
||||
window_sum = prefix_sum[count:] - prefix_sum[:-count]
|
||||
window_ac_power = window_power - (np.abs(window_sum) ** 2) / count
|
||||
reference_energy = float(np.vdot(reference, reference).real)
|
||||
denominator = np.sqrt(np.maximum(reference_energy * window_ac_power, 1e-30))
|
||||
time_axis = np.arange(reference.size, dtype=np.float64) / decimated_rate
|
||||
best_score = -1.0
|
||||
best_index = 0
|
||||
best_cfo = 0.0
|
||||
for cfo_hz in _frequency_grid(criteria):
|
||||
rotated = reference * np.exp(2j * np.pi * cfo_hz * time_axis)
|
||||
correlation = signal.fftconvolve(
|
||||
received,
|
||||
np.conj(rotated[::-1]),
|
||||
mode="valid",
|
||||
)
|
||||
scores = np.abs(correlation) / denominator
|
||||
index = int(np.argmax(scores))
|
||||
score = float(scores[index])
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_index = index
|
||||
best_cfo = float(cfo_hz)
|
||||
start_decimated = first + best_index
|
||||
return {
|
||||
"score": best_score,
|
||||
"start_decimated_sample": start_decimated,
|
||||
"start_sample": start_decimated * criteria.decimation,
|
||||
"start_seconds": (
|
||||
start_decimated * criteria.decimation / layout.sample_rate_hz
|
||||
),
|
||||
"cfo_hz": best_cfo,
|
||||
}
|
||||
|
||||
|
||||
def locate_reference_streaming(
|
||||
capture_path: Path,
|
||||
reference: np.ndarray,
|
||||
layout: ReferenceLayout,
|
||||
criteria: SearchCriteria,
|
||||
progress: Callable[[int, int], None] | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Scan every source sample through a bounded anti-aliased feature stream."""
|
||||
|
||||
total_samples = cs8_sample_count(capture_path)
|
||||
total_decimated = math.ceil(total_samples / criteria.decimation)
|
||||
if total_decimated < reference.size:
|
||||
raise ValueError("RX короче эталонной лестницы")
|
||||
decimated_rate = layout.sample_rate_hz / criteria.decimation
|
||||
feature_stride = round(criteria.feature_period_seconds * decimated_rate)
|
||||
if feature_stride <= 0 or criteria.histogram_stride <= 0:
|
||||
raise ValueError("feature period and histogram stride must be positive")
|
||||
taps = design_antialias_filter(layout, criteria)
|
||||
feature_parts: list[np.ndarray] = []
|
||||
pending = np.empty(0, dtype=np.complex64)
|
||||
maximum_loaded = 0
|
||||
streamed_decimated = 0
|
||||
for block_start, block in iter_filtered_decimated(
|
||||
capture_path,
|
||||
layout,
|
||||
criteria,
|
||||
taps,
|
||||
):
|
||||
if block_start != streamed_decimated:
|
||||
raise RuntimeError("decimated stream is not contiguous")
|
||||
streamed_decimated += block.size
|
||||
if progress is not None:
|
||||
progress(
|
||||
min(total_samples, streamed_decimated * criteria.decimation),
|
||||
total_samples,
|
||||
)
|
||||
combined = np.concatenate((pending, block)) if pending.size else block
|
||||
complete = combined.size // feature_stride
|
||||
used = complete * feature_stride
|
||||
if used:
|
||||
feature_parts.append(_power_features(combined[:used], feature_stride))
|
||||
pending = combined[used:].copy()
|
||||
maximum_loaded = max(maximum_loaded, int(combined.size))
|
||||
if pending.size:
|
||||
feature_parts.append(
|
||||
np.array([np.mean(np.abs(pending) ** 2, dtype=np.float64)])
|
||||
)
|
||||
features = np.concatenate(feature_parts)
|
||||
reference_features = _power_features(reference, feature_stride)
|
||||
if features.size < reference_features.size:
|
||||
raise ValueError("RX feature stream is shorter than the reference")
|
||||
scores = _normalized_sliding_scores(features, reference_features)
|
||||
sampled = np.clip(scores[::criteria.histogram_stride], 0.0, 1.0)
|
||||
histogram = np.histogram(sampled, bins=2000, range=(0.0, 1.0))[0]
|
||||
background_p99 = _histogram_quantile(histogram, 0.99)
|
||||
candidate_count = min(max(20, criteria.maximum_reported_candidates * 8), scores.size)
|
||||
indexes = np.argpartition(scores, -candidate_count)[-candidate_count:]
|
||||
coarse = [
|
||||
{
|
||||
"score": float(scores[index]),
|
||||
"start_decimated_sample": int(index * feature_stride),
|
||||
"start_feature": int(index),
|
||||
}
|
||||
for index in indexes
|
||||
]
|
||||
coarse = _deduplicate_candidates(
|
||||
coarse,
|
||||
max(1, reference.size // 3),
|
||||
criteria.maximum_reported_candidates,
|
||||
)
|
||||
refined = []
|
||||
for row in coarse:
|
||||
candidate = _refine_candidate(
|
||||
capture_path,
|
||||
reference,
|
||||
int(row["start_decimated_sample"]),
|
||||
layout,
|
||||
criteria,
|
||||
taps,
|
||||
)
|
||||
temporal_score = float(row["score"])
|
||||
candidate["temporal_power_score"] = temporal_score
|
||||
candidate["score_to_background_p99_db"] = (
|
||||
20.0 * math.log10(temporal_score / background_p99)
|
||||
if temporal_score > 0 and background_p99 > 0
|
||||
else None
|
||||
)
|
||||
refined.append(candidate)
|
||||
independent = _deduplicate_candidates(
|
||||
refined,
|
||||
max(1, reference.size // 3),
|
||||
criteria.maximum_reported_candidates,
|
||||
)
|
||||
return {
|
||||
"coverage_start_sample": 0,
|
||||
"coverage_stop_sample": total_samples,
|
||||
"coverage_seconds": total_samples / layout.sample_rate_hz,
|
||||
"coverage_complete": streamed_decimated == total_decimated,
|
||||
"source_samples_processed": total_samples,
|
||||
"tested_feature_start_positions": int(scores.size),
|
||||
"feature_stride_decimated_samples": feature_stride,
|
||||
"feature_stride_source_samples": feature_stride * criteria.decimation,
|
||||
"frequency_grid_hz": [
|
||||
float(value) for value in _frequency_grid(criteria)
|
||||
],
|
||||
"background_score_p99": background_p99,
|
||||
"candidates": independent,
|
||||
"maximum_loaded_decimated_samples": maximum_loaded,
|
||||
"maximum_loaded_iq_bytes": maximum_loaded * np.dtype(np.complex64).itemsize,
|
||||
"anti_alias_filter": {
|
||||
"kind": "linear_phase_fir_kaiser",
|
||||
"tap_count": int(taps.size),
|
||||
"passband_hz": criteria.anti_alias_passband_hz,
|
||||
"stopband_hz": criteria.anti_alias_stopband_hz,
|
||||
"design_attenuation_db": criteria.anti_alias_attenuation_db,
|
||||
"group_delay_source_samples": (int(taps.size) - 1) // 2,
|
||||
"position_delay_correction_source_samples": (
|
||||
-((int(taps.size) - 1) // 2)
|
||||
),
|
||||
"boundary_handling": "symmetric_overlap_with_zero_extension_at_file_edges",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _read_candidate(
|
||||
capture_path: Path,
|
||||
start_decimated: int,
|
||||
count: int,
|
||||
layout: ReferenceLayout,
|
||||
criteria: SearchCriteria,
|
||||
) -> np.ndarray:
|
||||
taps = design_antialias_filter(layout, criteria)
|
||||
return read_filtered_decimated_window(
|
||||
capture_path,
|
||||
start_decimated,
|
||||
start_decimated + count,
|
||||
layout,
|
||||
criteria,
|
||||
taps,
|
||||
)
|
||||
|
||||
|
||||
def _decimated_window(window: tuple[int, int], decimation: int) -> tuple[int, int]:
|
||||
return round(window[0] / decimation), round(window[1] / decimation)
|
||||
|
||||
|
||||
def _band_power(values: np.ndarray, sample_rate_hz: float, low: float, high: float) -> float:
|
||||
nperseg = min(4096, values.size)
|
||||
frequencies, density = signal.welch(
|
||||
values,
|
||||
fs=sample_rate_hz,
|
||||
nperseg=nperseg,
|
||||
noverlap=nperseg // 2,
|
||||
return_onesided=False,
|
||||
scaling="density",
|
||||
)
|
||||
mask = (frequencies >= low) & (frequencies <= high)
|
||||
if not np.any(mask):
|
||||
return 0.0
|
||||
return float(np.trapezoid(density[mask], frequencies[mask]))
|
||||
|
||||
|
||||
def verify_candidate(
|
||||
capture_path: Path,
|
||||
reference: np.ndarray,
|
||||
candidate: dict[str, float | int],
|
||||
layout: ReferenceLayout,
|
||||
criteria: SearchCriteria,
|
||||
) -> dict[str, object]:
|
||||
start_decimated = int(candidate["start_decimated_sample"])
|
||||
received = _read_candidate(
|
||||
capture_path,
|
||||
start_decimated,
|
||||
reference.size,
|
||||
layout,
|
||||
criteria,
|
||||
)
|
||||
decimated_rate = layout.sample_rate_hz / criteria.decimation
|
||||
cfo_hz = float(candidate["cfo_hz"])
|
||||
time_axis = np.arange(reference.size, dtype=np.float64) / decimated_rate
|
||||
rotated = reference * np.exp(2j * np.pi * cfo_hz * time_axis)
|
||||
segment_decimated = round(layout.segment_samples / criteria.decimation)
|
||||
trim = min(
|
||||
max(1, round(21 * layout.sample_rate_hz / layout.symbol_rate / criteria.decimation)),
|
||||
max(1, segment_decimated // 10),
|
||||
)
|
||||
rows = []
|
||||
for index, window in enumerate(layout.active_windows()):
|
||||
start, stop = _decimated_window(window, criteria.decimation)
|
||||
start += trim
|
||||
stop -= trim
|
||||
expected = rotated[start:stop]
|
||||
observed = received[start:stop]
|
||||
expected = expected - np.mean(expected, dtype=np.complex128)
|
||||
observed = observed - np.mean(observed, dtype=np.complex128)
|
||||
expected_energy = float(np.vdot(expected, expected).real)
|
||||
observed_energy = float(np.vdot(observed, observed).real)
|
||||
gain = np.vdot(expected, observed) / max(expected_energy, 1e-30)
|
||||
matched = gain * expected
|
||||
residual = observed - matched
|
||||
matched_power = float(np.mean(np.abs(matched) ** 2))
|
||||
residual_power = float(np.mean(np.abs(residual) ** 2))
|
||||
coherence = float(
|
||||
abs(np.vdot(expected, observed))
|
||||
/ math.sqrt(max(expected_energy * observed_energy, 1e-30))
|
||||
)
|
||||
rows.append(
|
||||
{
|
||||
"label": f"L{index}",
|
||||
"expected_level_db": layout.levels_db[index],
|
||||
"matched_power": matched_power,
|
||||
"residual_power": residual_power,
|
||||
"matched_to_residual_db": (
|
||||
10.0 * math.log10(matched_power / residual_power)
|
||||
if matched_power > 0 and residual_power > 0
|
||||
else None
|
||||
),
|
||||
"normalized_coherence": coherence,
|
||||
"gain_abs": float(abs(gain)),
|
||||
}
|
||||
)
|
||||
|
||||
step_rows = []
|
||||
for lower, upper in zip(rows, rows[1:]):
|
||||
expected_step = float(upper["expected_level_db"]) - float(lower["expected_level_db"])
|
||||
lower_power = float(lower["matched_power"])
|
||||
upper_power = float(upper["matched_power"])
|
||||
observed_step = (
|
||||
10.0 * math.log10(upper_power / lower_power)
|
||||
if lower_power > 0 and upper_power > 0
|
||||
else None
|
||||
)
|
||||
step_rows.append(
|
||||
{
|
||||
"comparison": f"{upper['label']}/{lower['label']}",
|
||||
"expected_step_db": expected_step,
|
||||
"observed_step_db": observed_step,
|
||||
"error_db": observed_step - expected_step if observed_step is not None else None,
|
||||
}
|
||||
)
|
||||
finite_errors = [
|
||||
abs(float(row["error_db"]))
|
||||
for row in step_rows
|
||||
if row["error_db"] is not None
|
||||
]
|
||||
maximum_step_error = max(finite_errors) if len(finite_errors) == len(step_rows) else None
|
||||
|
||||
l3_start, l3_stop = _decimated_window(layout.active_windows()[-1], criteria.decimation)
|
||||
background_start, background_stop = _decimated_window(
|
||||
layout.background_windows()[-2], criteria.decimation
|
||||
)
|
||||
band_half_width = 0.675 * layout.symbol_rate
|
||||
low = layout.carrier_hz - band_half_width
|
||||
high = layout.carrier_hz + band_half_width
|
||||
l3_band_power = _band_power(received[l3_start:l3_stop], decimated_rate, low, high)
|
||||
background_band_power = _band_power(
|
||||
received[background_start:background_stop], decimated_rate, low, high
|
||||
)
|
||||
spectral_contrast = (
|
||||
10.0 * math.log10(l3_band_power / background_band_power)
|
||||
if l3_band_power > 0 and background_band_power > 0
|
||||
else None
|
||||
)
|
||||
l3_ratio = rows[-1]["matched_to_residual_db"]
|
||||
matched_powers = [float(row["matched_power"]) for row in rows]
|
||||
monotonic = all(upper > lower for lower, upper in zip(matched_powers, matched_powers[1:]))
|
||||
step_pass = (
|
||||
maximum_step_error is not None
|
||||
and maximum_step_error <= criteria.maximum_step_error_db
|
||||
and monotonic
|
||||
)
|
||||
coherence_pass = (
|
||||
l3_ratio is not None
|
||||
and float(l3_ratio) >= criteria.l3_matched_to_residual_minimum_db
|
||||
)
|
||||
spectral_pass = (
|
||||
spectral_contrast is not None
|
||||
and spectral_contrast >= criteria.l3_spectral_contrast_minimum_db
|
||||
)
|
||||
return {
|
||||
"segments": rows,
|
||||
"steps": step_rows,
|
||||
"maximum_absolute_step_error_db": maximum_step_error,
|
||||
"monotonic_matched_power": monotonic,
|
||||
"step_pattern_pass": step_pass,
|
||||
"l3_matched_to_residual_pass": coherence_pass,
|
||||
"l3_band_power": l3_band_power,
|
||||
"background_band_power": background_band_power,
|
||||
"l3_spectral_contrast_db": spectral_contrast,
|
||||
"spectral_contrast_pass": spectral_pass,
|
||||
"qualified": step_pass and coherence_pass and spectral_pass,
|
||||
}
|
||||
|
||||
|
||||
def classify_search(
|
||||
search: dict[str, object],
|
||||
verifications: list[dict[str, object]],
|
||||
criteria: SearchCriteria,
|
||||
) -> tuple[str, list[str]]:
|
||||
if search.get("coverage_complete") is not True:
|
||||
return "ANALYSIS_INCOMPLETE", ["FULL_CAPTURE_COVERAGE_NOT_CONFIRMED"]
|
||||
candidates = list(search["candidates"])
|
||||
if not candidates:
|
||||
return "NOT_DETECTED", ["NO_CANDIDATES"]
|
||||
peak = candidates[0]
|
||||
score = float(peak["score"])
|
||||
contrast = peak.get("score_to_background_p99_db")
|
||||
correlation_pass = score >= criteria.normalized_correlation_minimum
|
||||
contrast_pass = (
|
||||
contrast is not None
|
||||
and float(contrast) >= criteria.peak_to_background_minimum_db
|
||||
)
|
||||
qualified = [row for row in verifications if row.get("qualified") is True]
|
||||
reasons = []
|
||||
if not correlation_pass:
|
||||
reasons.append("NORMALIZED_CORRELATION_BELOW_THRESHOLD")
|
||||
if not contrast_pass:
|
||||
reasons.append("PEAK_TO_BACKGROUND_CONTRAST_BELOW_THRESHOLD")
|
||||
if not qualified:
|
||||
reasons.append("STEP_OR_SPECTRAL_CONFIRMATION_FAILED")
|
||||
if correlation_pass and contrast_pass and len(qualified) == 1:
|
||||
return "DETECTED", []
|
||||
if (correlation_pass and contrast_pass) or qualified:
|
||||
if len(qualified) > 1:
|
||||
reasons.append("MULTIPLE_QUALIFIED_CANDIDATES")
|
||||
return "AMBIGUOUS", reasons
|
||||
return "NOT_DETECTED", reasons
|
||||
|
||||
|
||||
def analyze(
|
||||
reference_path: Path,
|
||||
capture_path: Path,
|
||||
report_path: Path,
|
||||
*,
|
||||
expected_capture_sha256: str,
|
||||
layout: ReferenceLayout = DEFAULT_LAYOUT,
|
||||
criteria: SearchCriteria = DEFAULT_CRITERIA,
|
||||
expected_reference_sha256: str = EXPECTED_REFERENCE_SHA256,
|
||||
progress_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
if report_path.exists():
|
||||
raise FileExistsError(f"отчёт уже существует: {report_path}")
|
||||
total_samples = cs8_sample_count(capture_path)
|
||||
if progress_path is not None:
|
||||
_write_progress(
|
||||
progress_path,
|
||||
phase="VERIFYING_REFERENCE",
|
||||
processed_samples=0,
|
||||
total_samples=total_samples,
|
||||
)
|
||||
reference, reference_sha256 = load_reference(
|
||||
reference_path,
|
||||
layout,
|
||||
criteria,
|
||||
expected_sha256=expected_reference_sha256,
|
||||
)
|
||||
capture_progress: Callable[[int, int], None] | None = None
|
||||
if progress_path is not None:
|
||||
capture_progress = lambda processed, total: _write_progress(
|
||||
progress_path,
|
||||
phase="VERIFYING_RX_SHA256",
|
||||
processed_samples=processed // 2,
|
||||
total_samples=total // 2,
|
||||
)
|
||||
capture_sha256 = sha256_file(capture_path, progress=capture_progress)
|
||||
if capture_sha256 != expected_capture_sha256:
|
||||
raise ValueError("SHA256 RX-файла не совпадает с ожидаемым значением")
|
||||
analysis_started = time.perf_counter()
|
||||
with MemoryPeakSampler() as memory:
|
||||
search_progress: Callable[[int, int], None] | None = None
|
||||
if progress_path is not None:
|
||||
search_progress = lambda processed, total: _write_progress(
|
||||
progress_path,
|
||||
phase="SEARCHING",
|
||||
processed_samples=processed,
|
||||
total_samples=total,
|
||||
)
|
||||
search = locate_reference_streaming(
|
||||
capture_path,
|
||||
reference,
|
||||
layout,
|
||||
criteria,
|
||||
progress=search_progress,
|
||||
)
|
||||
verifications = []
|
||||
for candidate in search["candidates"]:
|
||||
verification = verify_candidate(
|
||||
capture_path,
|
||||
reference,
|
||||
candidate,
|
||||
layout,
|
||||
criteria,
|
||||
)
|
||||
verifications.append({"candidate": candidate, **verification})
|
||||
verdict, reasons = classify_search(search, verifications, criteria)
|
||||
analysis_seconds = time.perf_counter() - analysis_started
|
||||
result = {
|
||||
"schema_version": 1,
|
||||
"analyzer_version": ANALYZER_VERSION,
|
||||
"mode": "OFFLINE_REFERENCE_STAIRCASE_SEARCH",
|
||||
"reference_profile": "LAB065_REFERENCE_STAIRCASE",
|
||||
"reference_path": str(reference_path),
|
||||
"reference_sha256": reference_sha256,
|
||||
"capture_path": str(capture_path),
|
||||
"capture_sha256": capture_sha256,
|
||||
"capture_bytes": capture_path.stat().st_size,
|
||||
"capture_complex_samples": cs8_sample_count(capture_path),
|
||||
"layout": asdict(layout),
|
||||
"criteria_fixed_before_capture_analysis": asdict(criteria),
|
||||
"search": search,
|
||||
"verifications": verifications,
|
||||
"verdict": verdict,
|
||||
"verdict_reasons": reasons,
|
||||
"peak_process_memory_bytes": memory.peak_bytes,
|
||||
"peak_additional_memory_bytes": memory.additional_peak_bytes,
|
||||
"analysis_seconds": analysis_seconds,
|
||||
"hardware_accessed": False,
|
||||
"rf_tx": 0,
|
||||
}
|
||||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
serialized = json.dumps(result, ensure_ascii=False, indent=2, allow_nan=False) + "\n"
|
||||
with report_path.open("x", encoding="utf-8") as stream:
|
||||
stream.write(serialized)
|
||||
if progress_path is not None:
|
||||
_write_progress(
|
||||
progress_path,
|
||||
phase="COMPLETE",
|
||||
processed_samples=total_samples,
|
||||
total_samples=total_samples,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--reference", required=True, type=Path)
|
||||
parser.add_argument("--capture", required=True, type=Path)
|
||||
parser.add_argument("--capture-sha256", required=True)
|
||||
parser.add_argument("--report", required=True, type=Path)
|
||||
parser.add_argument("--progress", type=Path)
|
||||
arguments = parser.parse_args()
|
||||
result = analyze(
|
||||
arguments.reference,
|
||||
arguments.capture,
|
||||
arguments.report,
|
||||
expected_capture_sha256=arguments.capture_sha256,
|
||||
progress_path=arguments.progress,
|
||||
)
|
||||
print(f"REFERENCE STAIRCASE = {result['verdict']}; RF TX = 0")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user