turov_av/SDR-Rover
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lab053: achieve real-time 5 fps software video receiver

Browse Source
This commit is contained in:
LittleSam129
2026-09-09 10:26:55 +03:00
parent cc38900cae
commit 0368c6d3e9
44 changed files with 472121 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

213
experiments/lab053_packet_processing.py Normal file
View File

@@ -0,0 +1,213 @@
"""Lab053: локальные вычисления настоящего пакета с неизменным PHY.
Старые функции получают частный словарь зависимостей: их Python-код не
переписывается и globals старых лабораторий не изменяются. Это позволяет
одновременно декодировать пакеты с одним набором неизменяемых констант.
"""
from __future__ import annotations
from collections import Counter
from contextvars import ContextVar
from dataclasses import dataclass, field
from functools import lru_cache
import math
from pathlib import Path
import sys
import time
from types import FunctionType, SimpleNamespace
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import numpy as np
from scipy.signal import oaconvolve
from experiments import lab052_candidate_detection as detection
from experiments.lab053_viterbi import viterbi_decode_soft, warmup
previous = detection.previous
physical = previous.lab045
lab043 = previous.lab043
SAMPLE_RATE = 8000000
SYMBOL_RATE = 500000
SAMPLES_PER_SYMBOL = 16
def bind(function, **dependencies):
"""Связать тот же code object с частными константами и зависимостями.
Без exec/eval и setattr модулей. Неподменённые protocol/parser helpers
остаются штатными. Копируется словарь ссылок, а не массивы или IQ.
"""
function = getattr(function, "__wrapped__", function)
namespace = dict(function.__globals__, **dependencies)
result = FunctionType(function.__code__, namespace, function.__name__, function.__defaults__, function.__closure__)
result.__kwdefaults__ = function.__kwdefaults__
return result
@dataclass
class PacketMeasure:
seconds: Counter = field(default_factory=Counter)
exclusive_seconds: Counter = field(default_factory=Counter)
calls: Counter = field(default_factory=Counter)
stack: list = field(default_factory=list)
cache: dict = field(default_factory=dict)
CURRENT: ContextVar[PacketMeasure | None] = ContextVar("lab053_packet_measure", default=None)
def measured(name):
def decorate(function):
def call(*args, **kwargs):
context = CURRENT.get()
if context is None:
return function(*args, **kwargs)
entry = [0.0]
context.stack.append(entry)
started = time.perf_counter()
try:
return function(*args, **kwargs)
finally:
elapsed = time.perf_counter() - started
context.stack.pop()
if context.stack:
context.stack[-1][0] += elapsed
context.seconds[name] += elapsed
context.exclusive_seconds[name] += elapsed - entry[0]
context.calls[name] += 1
return call
return decorate
@lru_cache(maxsize=16)
def rotation(length: int) -> np.ndarray:
indexes = np.arange(length)
values = np.exp(-2j * np.pi * lab043.CARRIER_OFFSET_HZ * indexes / SAMPLE_RATE).astype(np.complex64)
values.flags.writeable = False
return values
class PacketProcessor:
"""Параметры фиксированы; состояние кандидата принадлежит ContextVar.
Уточнение использует прежние ±2000 отсчётов, timing — все 1601 смещение.
Полное тело по-прежнему запрашивается только после валидного PHY header.
"""
def __init__(self, *, viterbi=viterbi_decode_soft):
self.startup = warmup() if viterbi is viterbi_decode_soft else {"backend": "CUSTOM", "warmup_seconds": 0}
constants = {"SAMPLES_PER_SYMBOL": SAMPLES_PER_SYMBOL, "SYMBOL_RATE": SYMBOL_RATE}
scale = bind(physical.lab044._scale_and_carrier, **constants)
self.bits = physical.lab044.preamble_bits().copy()
waveform = scale(previous.iq.radio.bpsk_modulate(self.bits))
self.reference = physical.lab044.signed_iq_bytes_to_complex64(lab043.to_cs8_bytes(waveform))
self.taps = bind(lab043._matched_filter_taps, **constants)()
self.expected = previous.iq.radio.bpsk_modulate(self.bits)
self.kernel = np.zeros((len(self.bits) - 1) * SAMPLES_PER_SYMBOL + 1, np.complex64)
self.kernel[::SAMPLES_PER_SYMBOL] = np.conj(self.expected)
self.kernel = self.kernel[::-1].copy()
for values in (self.bits, self.reference, self.taps, self.expected, self.kernel):
values.flags.writeable = False
slow_offsets = bind(lab043._timing_offsets, **constants)
slow_timing = bind(previous.ORIGINAL_TIMING, _timing_offsets=slow_offsets, **constants)
self.timing = measured("timing")(bind(previous.fast_preamble_timing,
lab045=SimpleNamespace(SAMPLES_PER_SYMBOL=SAMPLES_PER_SYMBOL),
timing_kernel=lambda unused: self.kernel, ORIGINAL_TIMING=slow_timing))
estimate = measured("cfo_estimation")(bind(lab043._estimate_cfo_gain, **constants))
self.carrier = measured("phase_polarity")(bind(lab043._evaluate_symbols,
_estimate_cfo_gain=estimate, **constants))
self.refine = measured("candidate_refinement")(bind(previous.ORIGINAL_REFINE, **constants))
header = measured("phy_header")(physical.decode_phy_header_soft)
body = bind(physical.analyze_post_preamble_soft, decode_phy_header_soft=header,
conv=SimpleNamespace(encoded_bit_count=physical.conv.encoded_bit_count,
viterbi_decode_soft=measured("viterbi")(viterbi)),
parse_packet=measured("protocol_crc")(physical.parse_packet))
physical_namespace = SimpleNamespace(**dict(vars(physical), **constants))
physical_namespace.decode_phy_header_soft = header
physical_namespace.analyze_post_preamble_soft = measured("body_processing")(body)
preamble_namespace = SimpleNamespace(_preamble_reference_samples=lambda: self.reference,
preamble_bits=lambda: self.bits,
lab043=SimpleNamespace(_refine_frame_start=self.refine))
self.strict = bind(previous.offline.decode_candidate_from_memmap,
lab046c=SimpleNamespace(lab044=preamble_namespace, lab045=physical_namespace),
lab046b=SimpleNamespace(_extract_soft_window=self.extract,
IqFrameReception=previous.iq.IqFrameReception))
@measured("iq_window_preparation")
def read_window(self, raw, low, high):
return raw[2 * low:2 * high].view(np.complex64)
@measured("frequency_shift")
def shift(self, segment):
return (segment - np.mean(segment)) * rotation(len(segment))
@measured("matched_rrc_filtering")
def filter(self, values):
return oaconvolve(values, self.taps, mode="full")
@measured("soft_symbol_extraction")
def normalize(self, matched, nominal, timing, count, carrier):
symbols = matched[nominal + timing:nominal + timing + count * SAMPLES_PER_SYMBOL:SAMPLES_PER_SYMBOL].astype(np.complex128)
if symbols.size != count:
return None
symbol_time = (np.arange(count, dtype=np.float64) + .5) / SYMBOL_RATE
normalized = symbols * np.exp(-2j * math.pi * carrier["cfo_hz"] * symbol_time)
normalized /= carrier["abs_h"] * np.exp(1j * carrier["phase_rad"])
return normalized.real
@measured("extract_total")
def extract(self, samples, frame_start, symbol_count):
length = (symbol_count + 42) * SAMPLES_PER_SYMBOL
if frame_start < 0 or frame_start + length > samples.size:
return previous.iq.SoftWindow(False, np.empty(0), 0, 0, 0, {}, "TRUNCATED_IQ_WINDOW")
segment = samples[frame_start:frame_start + length]
matched = self.filter(self.shift(segment))
nominal = 16 * SAMPLES_PER_SYMBOL + self.taps.size - 1
timing = self.timing(matched, nominal, self.expected)
carrier = self.carrier(matched, nominal, timing, self.bits, len(self.bits))
if carrier["abs_h"] == 0:
return previous.iq.SoftWindow(False, np.empty(0), timing, 0, nominal, carrier, "ZERO_GAIN")
values = self.normalize(matched, nominal, timing, symbol_count, carrier)
if values is None:
return previous.iq.SoftWindow(False, np.empty(0), timing, 0, nominal, carrier, "TRUNCATED_SYMBOLS")
return previous.iq.SoftWindow(True, values, int(timing),
int((frame_start + nominal + timing) % SAMPLES_PER_SYMBOL), nominal, carrier, None)
def decode(self, samples, candidate, number=1):
context = PacketMeasure()
token = CURRENT.set(context)
started = time.perf_counter()
cpu_started = time.thread_time()
try:
outcome = self.strict(samples.view(np.float32), samples.size, candidate, number,
read_iq_window=self.read_window)
elapsed = time.perf_counter() - started
row = {"accepted": outcome.accepted, "reason": outcome.reason, "seconds": elapsed,
"thread_cpu_seconds": time.thread_time() - cpu_started,
"stages": dict(context.seconds), "exclusive_stages": dict(context.exclusive_seconds),
"calls": dict(context.calls), "coarse": candidate.sample_offset,
"refined": outcome.refined_start_sample}
if outcome.frame is not None:
row.update(timing=outcome.frame.timing_offset_samples, cfo_hz=outcome.frame.estimated_cfo_hz,
phase_rad=outcome.frame.estimated_phase_rad, polarity=outcome.frame.bpsk_polarity)
return outcome, row
finally:
CURRENT.reset(token)
def run_functional_tests() -> None:
processor = PacketProcessor()
assert len(processor.bits) == 1280 and len(processor.taps) == 161
assert not processor.reference.flags.writeable
def main() -> None:
run_functional_tests()
print("Lab053 packet processing functional checks PASS; RF TX=0")
if __name__ == "__main__":
main()

312
experiments/lab053_pipeline.py Normal file
View File

@@ -0,0 +1,312 @@
"""Lab053: ограниченный CPU-конвейер с фактическим временем выдачи JPEG.
Непрерывный IQ приходит по часовому расписанию либо с обратным давлением
для измерения пропускной способности. Переполнение не склеивает разрыв IQ:
сессия останавливается с явной причиной. Оборудование не используется.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
from pathlib import Path
from queue import Empty, Full, Queue
import sys
from threading import Event, Thread
import time
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import numpy as np
from experiments import lab052_detector_study as study
from experiments.lab051_receiver_performance import ImageSink
from experiments.lab051_receiver_profile import process_memory
from experiments.lab053_receiver_profile import SOURCE, OUTPUT, distribution, write_result
from experiments.lab053_streaming_receiver import StreamingReceiver
from experiments.lab053_packet_processing import PacketProcessor, previous
class BoundedQueue(Queue):
"""Пик фиксируется под тем же lock, что помещение в очередь."""
def __init__(self, capacity):
if type(capacity) is not int or capacity <= 0:
raise ValueError("Очередь должна иметь положительную целую вместимость")
super().__init__(maxsize=capacity)
self.peak = self.overflows = 0
def _put(self, item):
super()._put(item)
self.peak = max(self.peak, self._qsize())
def offer(self, item):
try:
self.put_nowait(item)
except Full:
self.overflows += 1
return False
return True
def run_pipeline(*, workers=1, paced=True, source=SOURCE, frame_count=30,
iq_capacity=8, candidate_capacity=4, output_capacity=16,
processing_delay=0.0, collect_packets=True, numpy_viterbi=False) -> dict:
"""Эталон доступен только оценке после сборки/decode; RX получает только IQ."""
if type(frame_count) is not int or frame_count <= 0:
raise ValueError("Число входных изображений должно быть положительным целым")
if not math.isfinite(processing_delay) or processing_delay < 0:
raise ValueError("Задержка должна быть конечной и неотрицательной")
try:
source_bytes = Path(source).stat().st_size
except OSError:
source_bytes = None # produce запишет точное исключение открытия.
sequence = study.sequence_fixture()
encoded = {number + 1: sequence[number % len(sequence)].jpeg for number in range(frame_count)}
capture_times = {number + 1: number / 5 for number in range(frame_count)}
processor = PacketProcessor(viterbi=previous.fast_viterbi) if numpy_viterbi else None
receiver = StreamingReceiver(workers=workers, candidate_capacity=candidate_capacity, processor=processor)
startup = receiver.processor.startup
if numpy_viterbi:
startup = {**startup, "backend": "NUMPY_LAB051"}
iq_queue, output_queue = BoundedQueue(iq_capacity), BoundedQueue(output_capacity)
sink = ImageSink()
source_done, receiver_done, stop = Event(), Event(), Event()
completed, block_rows, packet_rows, fragment_rows, rss_rows, errors = [], [], [], [], [], []
input_statistics = {"read_seconds": 0.0, "read_blocks": 0, "read_samples": 0,
"maximum_release_lateness_seconds": 0.0, "backpressure_seconds": 0.0}
output_statistics = {"freshness_seconds": 0.0, "service_seconds": 0.0}
receiver_service = receiver_wait = 0.0
process_started = time.process_time()
epoch = time.perf_counter()
def fail(reason, error=None):
errors.append({"reason": reason, "error": None if error is None else repr(error)})
stop.set()
def produce():
try:
with Path(source).open("rb") as capture:
while not stop.is_set():
started = time.perf_counter()
raw = capture.read(receiver.block_samples * 8)
if len(raw) % 8:
raise ValueError("Усечённый complex64 в источнике")
samples = np.frombuffer(raw, dtype=np.complex64)
input_statistics["read_seconds"] += time.perf_counter() - started
if not samples.size:
if not input_statistics["read_samples"]:
fail("EMPTY_IQ_SOURCE")
break
input_statistics["read_samples"] += samples.size
input_statistics["read_blocks"] += 1
arrival = input_statistics["read_samples"] / 8000000
if paced:
stop.wait(max(0.0, epoch + arrival - time.perf_counter()))
if stop.is_set():
break
lateness = max(0.0, time.perf_counter() - epoch - arrival)
input_statistics["maximum_release_lateness_seconds"] = max(
lateness, input_statistics["maximum_release_lateness_seconds"])
if not iq_queue.offer((samples, arrival)):
fail("IQ_QUEUE_OVERFLOW")
break
else:
started = time.perf_counter()
while not stop.is_set():
try:
iq_queue.put((samples, arrival), timeout=.02)
break
except Full:
continue
input_statistics["backpressure_seconds"] += time.perf_counter() - started
except BaseException as error:
fail("SOURCE_EXCEPTION", error)
finally:
source_done.set()
def consume_output():
try:
while not receiver_done.is_set() or not output_queue.empty():
try:
packet, receiver_timestamp = output_queue.get(timeout=.02)
except Empty:
continue
started = time.perf_counter()
assembly_before = sink.assembly_seconds
item = sink.offer(packet, time.perf_counter() - epoch)
fragment_rows.append({"start_sample": packet.start_sample,
"seconds": sink.assembly_seconds - assembly_before})
output_statistics["service_seconds"] += time.perf_counter() - started
if item is not None:
ready = time.perf_counter() - epoch
jpeg = item.pop("jpeg")
number = item["image_id"]
# Эти сведения не передаются назад в детектор/PHY/assembler.
started = time.perf_counter()
capture_time = capture_times.get(number)
age = ready - capture_time if capture_time is not None else None
fresh = paced and age is not None and 0 <= age <= .2
output_statistics["freshness_seconds"] += time.perf_counter() - started
completed.append({**item, "decode_complete_timestamp": ready,
"receiver_packet_timestamp": receiver_timestamp,
"capture_timestamp": capture_time, "image_age_seconds": age if paced else None,
"fresh": fresh, "sha256": hashlib.sha256(jpeg).hexdigest(),
"sha_match": number in encoded and jpeg == encoded[number]})
except BaseException as error:
fail("OUTPUT_EXCEPTION", error)
producer = Thread(target=produce, name="lab053_iq_source")
consumer = Thread(target=consume_output, name="lab053_jpeg_output")
producer.start()
consumer.start()
def emit(packets):
for packet in packets:
if not output_queue.offer((packet, time.perf_counter() - epoch)):
fail("OUTPUT_QUEUE_OVERFLOW")
return
try:
while not stop.is_set() and (not source_done.is_set() or not iq_queue.empty()):
waiting = time.perf_counter()
try:
samples, arrival = iq_queue.get(timeout=.02)
except Empty:
receiver_wait += time.perf_counter() - waiting
continue
receiver_wait += time.perf_counter() - waiting
started = time.perf_counter()
block_start = started - epoch
if processing_delay:
time.sleep(processing_delay)
packets = receiver.push_iq_block(samples)
cost = time.perf_counter() - started
receiver_service += cost
emit(packets)
if collect_packets:
packet_rows.extend(receiver.last_packet_rows)
block_rows.append({"samples_received": receiver.total_samples, "arrival_time": arrival,
"worker_start": block_start, "worker_end": time.perf_counter() - epoch,
"processing_seconds": cost, "iq_queue_size": iq_queue.qsize(),
"candidate_queue_peak": receiver.candidate_queue_peak,
"lag_seconds": time.perf_counter() - epoch - arrival if paced else None,
"packets": len(packets)})
if len(block_rows) % 128 == 0:
rss_rows.append({"iq_seconds": receiver.total_samples / 8000000,
"iq_queue": iq_queue.qsize(), "candidate_jobs": receiver.pending_jobs,
"retained_iq_bytes": receiver.buffer.nbytes, **process_memory()})
if not stop.is_set():
started = time.perf_counter()
emit(receiver.finish())
receiver_service += time.perf_counter() - started
if collect_packets:
packet_rows.extend(receiver.last_packet_rows)
except BaseException as error:
fail("RECEIVER_EXCEPTION", error)
finally:
stop.set()
producer.join()
receiver.close()
receiver_done.set()
consumer.join()
for queue in (iq_queue, output_queue):
while True:
try:
queue.get_nowait()
except Empty:
break
wall = time.perf_counter() - epoch
process_cpu = time.process_time() - process_started
iq_seconds = source_bytes / 8 / 8000000 if source_bytes is not None else None
horizon = frame_count / 5
ages = [row["image_age_seconds"] for row in completed if row["image_age_seconds"] is not None]
fresh = [row for row in completed if row["fresh"] and row["decode_complete_timestamp"] <= horizon]
valid = [row for row in packet_rows if row["accepted"]]
# В paced-опыте wall включает ожидание прихода IQ и не является service time.
service_sum = receiver_service + input_statistics["read_seconds"] + output_statistics["service_seconds"] + output_statistics["freshness_seconds"]
result = {"mode": "PACED_WALL_CLOCK" if paced else "SATURATED_BOUNDED_REPLAY",
"source": str(source), "workers": workers, "startup": startup,
"input_fps": 5, "input_frames": frame_count, "observation_seconds": horizon,
"iq_seconds": iq_seconds, "consumed_iq_seconds": receiver.total_samples / 8000000,
"source_consumed_completely": source_bytes is not None and receiver.total_samples * 8 == source_bytes and not errors,
"full_wall_seconds": wall, "process_cpu_seconds": process_cpu,
"processing_seconds": service_sum if paced else wall,
"processing_per_rf_second": (service_sum if paced else wall) / iq_seconds if iq_seconds else None,
"service_sum_seconds": service_sum, "receiver_service_seconds": receiver_service,
"receiver_wait_seconds": receiver_wait, "input_statistics": input_statistics,
"output_statistics": output_statistics, "assembly_seconds": sink.assembly_seconds,
"jpeg_decode_seconds": sink.jpeg_decode_seconds,
"coarse_seconds": receiver.coarse_seconds, "cheap_seconds": receiver.cheap_seconds,
"shared_front_end_seconds": receiver.decimator.seconds,
"coarse_detection_exclusive_seconds": receiver.coarse_seconds - receiver.decimator.seconds,
"iq_input_preparation_seconds": receiver.input_preparation_seconds,
"cluster_seconds": receiver.cluster_seconds, "true_strict_seconds": receiver.true_strict_seconds,
"false_strict_seconds": receiver.false_strict_seconds,
"rx_physical_packets": receiver.packet_count, "candidate_count": receiver.candidate_count,
"received_images": len(completed), "object_jpeg_integrity": len(completed) == frame_count and all(row["sha_match"] for row in completed),
"fresh_displayed": len(fresh) if paced else None, "displayed": len(fresh) if paced else None,
"fresh_display_fps": len(fresh) / horizon if paced else None,
"image_age_min_seconds": min(ages) if ages else None,
"image_age_median_seconds": float(np.median(ages)) if ages else None,
"image_age_max_seconds": max(ages) if ages else None, "max_video_age_seconds": .2,
"iq_queue_capacity": iq_capacity, "iq_queue_peak": iq_queue.peak, "iq_overflow": iq_queue.overflows,
"candidate_queue_capacity": candidate_capacity, "candidate_queue_peak": receiver.candidate_queue_peak,
"candidate_overflow": receiver.candidate_overflow,
"output_queue_capacity": output_capacity, "output_queue_peak": output_queue.peak,
"output_overflow": output_queue.overflows,
"speculative_checks": receiver.speculative_checks,
"discarded_speculative_checks": receiver.discarded_speculative_checks,
"discarded_speculative_seconds": receiver.discarded_speculative_seconds,
"retained_iq_peak_bytes": receiver.peak_buffer_samples * 8,
"allocated_iq_ring_bytes": 2 * receiver.ring_capacity * 8,
"iq_queue_limit_bytes": iq_capacity * receiver.block_samples * 8,
"working_arrays_engineering_estimate_bytes": 2 * receiver.ring_capacity * 8 + (iq_capacity + 2) * receiver.block_samples * 8 + workers * 10 * receiver.lookahead * 8,
"process_memory": process_memory(), "rss_samples": rss_rows,
"detector": dict(receiver.detection_statistics), "rejected_reasons": dict(receiver.rejected_reasons),
"retained_after_close_bytes": receiver.buffer.nbytes + receiver.ring.nbytes + receiver.decimator.values.nbytes,
"pending_jobs_after_close": receiver.pending_jobs, "errors": errors,
"valid_packet_total": distribution([row["seconds"] for row in valid]),
"valid_packet_thread_cpu": distribution([row["thread_cpu_seconds"] for row in valid]),
"fragment_processing": distribution([row["seconds"] for row in fragment_rows]),
"valid_packet_stages": {name: distribution([row["exclusive_stages"].get(name, 0) for row in valid])
for name in sorted({name for row in valid for name in row["stages"]})},
"images": completed, "blocks": block_rows, "packet_rows": packet_rows,
"camera_display_latency": "UNKNOWN", "independent_live_capture_timestamp": False,
"freshness_evaluation": "post-JPEG source timestamp; does not feed RX verdict", "rf_tx": 0}
return result
def run_functional_tests() -> None:
queue = BoundedQueue(1)
assert queue.offer(1) and not queue.offer(2)
assert queue.overflows == 1 and queue.peak == 1 and queue.get_nowait() == 1
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--workers", type=int, default=1)
parser.add_argument("--unpaced", action="store_true")
parser.add_argument("--name")
parser.add_argument("--source", type=Path, default=SOURCE)
parser.add_argument("--frames", type=int, default=30)
parser.add_argument("--processing-delay", type=float, default=0)
parser.add_argument("--numpy-viterbi", action="store_true")
arguments = parser.parse_args()
result = run_pipeline(workers=arguments.workers, paced=not arguments.unpaced,
source=arguments.source, frame_count=arguments.frames,
processing_delay=arguments.processing_delay, collect_packets=arguments.frames <= 30,
numpy_viterbi=arguments.numpy_viterbi)
name = arguments.name or f"{'throughput' if arguments.unpaced else 'paced'}_workers_{arguments.workers}.json"
write_result(name, result)
print(json.dumps({key: result[key] for key in ("mode", "workers", "processing_seconds", "processing_per_rf_second",
"full_wall_seconds", "rx_physical_packets", "received_images", "object_jpeg_integrity", "fresh_display_fps",
"image_age_min_seconds", "image_age_median_seconds", "image_age_max_seconds", "iq_queue_peak",
"candidate_queue_peak", "iq_overflow", "candidate_overflow", "errors")}), flush=True)
if __name__ == "__main__":
main()

333
experiments/lab053_receiver_profile.py Normal file
View File

@@ -0,0 +1,333 @@
"""Lab053: новый профиль Lab052 и точный файловый источник для конвейера."""
from __future__ import annotations
import argparse
from contextlib import contextmanager
from dataclasses import replace
import hashlib
import json
from pathlib import Path
import sys
import subprocess
import time
from types import SimpleNamespace
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import numpy as np
from experiments import lab052_candidate_detection as detection
from experiments import lab052_detector_study as study
from experiments.lab051_receiver_profile import StageProfiler
ROOT = Path(__file__).resolve().parents[1]
OUTPUT = ROOT / "data/processed/lab053"
SOURCE = ROOT / "data/raw/lab053/source.c64"
def write_result(name: str, value: dict) -> None:
OUTPUT.mkdir(parents=True, exist_ok=True)
(OUTPUT / name).write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def distribution(values) -> dict:
values = np.asarray(values, dtype=np.float64)
if not values.size:
return {"count": 0, "sum": 0.0, "mean": None, "median": None, "p95": None, "max": None}
return {"count": len(values), "sum": float(values.sum()), "mean": float(values.mean()),
"median": float(np.median(values)), "p95": float(np.quantile(values, .95)),
"max": float(values.max())}
class DetailedProfiler(StageProfiler):
"""Вложенные времена не складываются повторно; только отдельный замер."""
@contextmanager
def measure(self, *, optimized=True):
with super().measure(optimized=optimized):
patches = []
def wrap(function, name):
def measured(*args, **kwargs):
scope = self.shared if self.current is None else self.current
stage = scope[name]
started = time.perf_counter()
entry = [0.0]
self.stack.append(entry)
try:
return function(*args, **kwargs)
finally:
elapsed = time.perf_counter() - started
self.stack.pop()
if self.stack:
self.stack[-1][0] += elapsed
stage.calls += 1
stage.seconds += elapsed
stage.exclusive_seconds += elapsed - entry[0]
return measured
targets = ((detection.CandidateFrontEnd, "coarse_scores", "coarse_detection"),
(detection.CandidateFrontEnd, "split_scores", "cheap_lab052"),
(detection.previous.DecimationCache, "get", "shared_decimation"),
(detection.previous.BlockFrontEnd, "extract", "soft_extraction"))
try:
for owner, name, label in targets:
original = getattr(owner, name)
patches.append((owner, name, original))
setattr(owner, name, wrap(original, label))
yield self
finally:
for owner, name, original in reversed(patches):
setattr(owner, name, original)
def detailed_summary(self):
output = self.summary()
for kind in ("valid", "false"):
rows = [row for row in self.rows if row["kind"] == kind]
names = sorted({name for row in rows for name in row["stages"]})
output[kind]["packet_total"] = distribution([row["seconds"] for row in rows])
output[kind]["per_packet_stages"] = {
name: {field: distribution([row["stages"].get(name, {}).get(field, 0.0) for row in rows])
for field in ("seconds", "exclusive_seconds")} for name in names}
return output
def baseline_profile() -> None:
sequence = study.sequence_fixture()
profiler = DetailedProfiler()
started = time.perf_counter()
with profiler.measure():
stream, receiver = study.measured_stream(sequence, detection.StreamingReceiver)
full_experiment_seconds = time.perf_counter() - started
result = {"stream": stream, "profile": profiler.detailed_summary(), "packets": profiler.rows,
"experiment_wall_seconds": full_experiment_seconds,
"source_and_diagnostics_seconds": full_experiment_seconds - stream["processing_seconds"],
"timing_includes_instrumentation": True, "rf_tx": 0}
write_result("baseline_profile.json", result)
print(json.dumps({"profile": result["profile"], "processing": stream["processing_seconds"],
"rx_packets": receiver.packet_count}), flush=True)
def prepare_source(frame_count=30) -> None:
"""Точные блоки прежнего генератора; конечный файл принадлежит источнику.
RX далее читает ограниченные блоки обычным read, без mmap всего файла.
Стоимость синтеза передаваемой формы не объявляется стоимостью live RX.
"""
source_path = SOURCE if frame_count == 30 else SOURCE.with_name(f"source_{frame_count}.c64")
source_path.parent.mkdir(parents=True, exist_ok=True)
sequence = study.sequence_fixture()
sequence = [replace(sequence[number % len(sequence)], image_id=number + 1) for number in range(frame_count)]
checksum = hashlib.sha256()
counts = []
started = time.perf_counter()
with source_path.open("wb") as output:
class Recorder(detection.StreamingReceiver):
def push_iq_block(self, samples):
data = samples.tobytes()
output.write(data)
checksum.update(data)
counts.append(samples.size)
self.total_samples += samples.size
return []
def finish(self):
self.closed = True
return []
original = study.performance.fast
try:
study.performance.fast = SimpleNamespace(StreamingReceiver=Recorder)
study.performance.run_stream(sequence)
finally:
study.performance.fast = original
result = {"path": str(source_path.relative_to(ROOT)), "dtype": "complex64", "blocks": len(counts),
"input_frames": frame_count, "sample_count": sum(counts), "bytes": source_path.stat().st_size, "sha256": checksum.hexdigest(),
"block_samples": 65536, "last_block_samples": counts[-1],
"iq_seconds": sum(counts) / 8000000, "preparation_seconds": time.perf_counter() - started,
"source_fixture": "data/processed/lab052/source_fixture.json", "rf_tx": 0}
write_result("source.json" if frame_count == 30 else f"source_{frame_count}.json", result)
print(json.dumps(result), flush=True)
def optimized_stream(workers=1) -> dict:
from experiments.lab053_streaming_receiver import StreamingReceiver
rows = []
result, receiver = study.measured_stream(study.sequence_fixture(), lambda: StreamingReceiver(
workers=workers, packet_observer=lambda row, outcome: rows.append(row)))
result.update(workers=workers, packet_rows=rows, candidate_queue_peak=receiver.candidate_queue_peak,
candidate_overflow=receiver.candidate_overflow,
speculative_checks=receiver.speculative_checks,
discarded_speculative_checks=receiver.discarded_speculative_checks,
discarded_speculative_seconds=receiver.discarded_speculative_seconds)
valid = [row for row in rows if row["accepted"]]
result["valid_packet_total"] = distribution([row["seconds"] for row in valid])
result["valid_packet_stages"] = {name: distribution([row["exclusive_stages"].get(name, 0) for row in valid])
for name in sorted({name for row in valid for name in row["stages"]})}
write_result(f"initial_workers_{workers}.json", result)
print(json.dumps({key: result[key] for key in ("workers", "processing_seconds", "processing_per_rf_second",
"rx_physical_packets", "received_images", "object_jpeg_integrity", "fresh_display_fps",
"image_age_min_seconds", "image_age_median_seconds", "image_age_max_seconds", "candidate_count",
"valid_packet_total", "candidate_queue_peak", "discarded_speculative_checks")}), flush=True)
return result
def summarize_results() -> None:
"""Выбрать целые медианные проходы и пересчитать критерии из их строк."""
def read(name):
return json.loads((OUTPUT / name).read_text(encoding="utf-8"))
selected = {}
repeats = {}
for mode in ("paced", "throughput"):
runs = [read(f"{mode}_1_repeat_{number}.json") for number in (1, 2, 3)]
chosen = sorted(range(3), key=lambda number: runs[number]["processing_seconds"])[1]
result = dict(runs[chosen])
result["selection"] = {"method": "whole median processing-time run", "repeat": chosen + 1,
"source": f"{mode}_1_repeat_{chosen + 1}.json"}
selected[mode], repeats[mode] = result, runs
write_result("stream.json" if mode == "paced" else "throughput.json", result)
write_result(f"{mode}_workers_1.json", result)
main, throughput = selected["paced"], selected["throughput"]
comparison = [{"workers": number,
"throughput_processing_per_rf_second": read(f"throughput_workers_{number}.json")["processing_per_rf_second"],
**{key: read(f"paced_workers_{number}.json")[key] for key in (
"processing_per_rf_second", "fresh_display_fps", "image_age_max_seconds", "candidate_queue_peak",
"iq_queue_peak", "speculative_checks", "discarded_speculative_checks")}} for number in (1, 2, 4)]
write_result("worker_comparison.json", {"selected_workers": 1, "comparisons": comparison,
"reason": "single worker is sufficient; additional workers give no measured throughput or latency benefit"})
long = read("long_stream.json")
equivalence, sensitivity = read("equivalence.json"), read("sensitivity.json")
regression = read("regression20k/result.json")
source = read("source.json")
overflow_probe = read("overflow_probe.json")
with (ROOT / source["path"]).open("rb") as source_handle:
source_sha256 = hashlib.file_digest(source_handle, "sha256").hexdigest()
cases = repeats["paced"] + repeats["throughput"] + [read(f"{mode}_workers_{worker}.json")
for mode in ("paced", "throughput") for worker in (2, 4)]
checks = {
"all_main_runs_240_packets_30_exact_jpegs": all(row["rx_physical_packets"] == 240 and row["received_images"] == 30
and row["object_jpeg_integrity"] for row in cases),
"all_main_runs_255_strict_15_false": all(row["candidate_count"] == 255 for row in cases),
"all_paced_runs_5_fresh_fps": all(row["fresh_display_fps"] == 5 for row in cases if row["mode"] == "PACED_WALL_CLOCK"),
"all_main_age_at_most_200ms": all(row["image_age_max_seconds"] <= .2 for row in cases if row["mode"] == "PACED_WALL_CLOCK"),
"all_main_processing_ratio_below_08": all(row["processing_per_rf_second"] <= .8 for row in cases),
"all_main_sources_complete_and_no_overflow": all(row["source_consumed_completely"] and not row["errors"] and
row["iq_overflow"] == row["candidate_overflow"] == row["output_overflow"] == 0 for row in cases),
"all_main_queues_within_caps": all(row["iq_queue_peak"] <= row["iq_queue_capacity"] and
row["candidate_queue_peak"] <= row["candidate_queue_capacity"] and row["output_queue_peak"] <= row["output_queue_capacity"] for row in cases),
"same_240_phy_crc_timing_all_worker_counts": equivalence["passed"],
"sensitivity_37_of_45_identical": sensitivity["same_verdicts"] and sensitivity["optimized_correct"] == 37 and sensitivity["lost_baseline_packets"] == 0,
"long_960_packets_120_jpeg_5_fps": long["rx_physical_packets"] == 960 and long["received_images"] == 120 and
long["object_jpeg_integrity"] and long["fresh_display_fps"] == 5 and long["image_age_max_seconds"] <= .2,
"long_queues_within_caps_no_overflow": long["iq_queue_peak"] <= long["iq_queue_capacity"] and
long["candidate_queue_peak"] <= long["candidate_queue_capacity"] and not long["errors"] and
long["iq_overflow"] == long["candidate_overflow"] == long["output_overflow"] == 0,
"long_complete_source_and_cleared_state": long["consumed_iq_seconds"] == long["iq_seconds"] and
long["retained_after_close_bytes"] == long["pending_jobs_after_close"] == 0,
"old20k_36_crc_sha": regression["strict"]["crc_pass"] == 36 and regression["object"]["valid_fragments"] == 36 and
regression["after_independent_verdict"]["sha_match"],
"source_exact_duration": source["sample_count"] == 48355328 and source["iq_seconds"] == 6.044416,
"source_file_size": (ROOT / source["path"]).stat().st_size == source["bytes"],
"source_file_sha256": source_sha256 == source["sha256"],
"forced_overflow_is_reported_and_state_cleared": overflow_probe["iq_overflow"] == 1 and
overflow_probe["iq_queue_peak"] == overflow_probe["iq_queue_capacity"] and
not overflow_probe["source_consumed_completely"] and
any(row["reason"] == "IQ_QUEUE_OVERFLOW" for row in overflow_probe["errors"]) and
overflow_probe["retained_after_close_bytes"] == overflow_probe["pending_jobs_after_close"] == 0,
"protocol_unchanged": not subprocess.check_output(["git", "status", "--porcelain", "--untracked-files=all", "--", "protocol"], cwd=ROOT).strip(),
"lab052_detector_unchanged": not subprocess.check_output(["git", "diff", "--", "experiments/lab052_candidate_detection.py"], cwd=ROOT).strip(),
"lab053_index_empty": not subprocess.check_output(["git", "diff", "--cached", "--name-only"], cwd=ROOT).strip(),
}
fec_crc = sum(main["valid_packet_stages"][name]["sum"] for name in ("viterbi", "protocol_crc"))
partition = {"A_iq_read_and_input_preparation": main["input_statistics"]["read_seconds"] + main["iq_input_preparation_seconds"],
"B_common_decimation": main["shared_front_end_seconds"], "C_coarse_detection": main["coarse_detection_exclusive_seconds"],
"D_cheap_lab052": main["cheap_seconds"], "E_true_packet_without_G": main["true_strict_seconds"] - fec_crc,
"F_false_strict": main["false_strict_seconds"], "G_viterbi_crc": fec_crc,
"H_fragment_assembly": main["assembly_seconds"], "I_jpeg_decode": main["jpeg_decode_seconds"],
"J_freshness": main["output_statistics"]["freshness_seconds"]}
partition["other_orchestration_and_instrumentation"] = main["processing_seconds"] - sum(partition.values())
rss = long["rss_samples"]
late = rss[len(rss)//2:]
# К старой оценке временных массивов добавлены ЯВНЫЕ резервы постоянных кэшей.
ring_samples = main["allocated_iq_ring_bytes"] // 16
maximum_waveform = ring_samples - 2*65536 - 8000
rotation_cache = 16 * maximum_waveform * 8
detection_rotation_cache = 8 * ((ring_samples + 7)//8) * 8
constants_reserve = 1024*1024
memory = {"retained_iq_peak_bytes": main["retained_iq_peak_bytes"], "allocated_ring_bytes": main["allocated_iq_ring_bytes"],
"iq_queue_peak_bytes": main["iq_queue_peak"] * 65536*8, "iq_queue_limit_bytes": main["iq_queue_limit_bytes"],
"temporary_working_buffers_estimate_bytes": main["working_arrays_engineering_estimate_bytes"],
"maximum_frame_waveform_samples": maximum_waveform, "rotation_cache_maxsize": 16,
"maximum_rotation_cache_bytes": rotation_cache, "maximum_detection_rotation_cache_bytes": detection_rotation_cache,
"constant_templates_and_trellis_reserve_bytes": constants_reserve,
"working_arrays_and_constant_caches_estimate_bytes": main["working_arrays_engineering_estimate_bytes"] + rotation_cache + detection_rotation_cache + constants_reserve,
"estimate_is_not_a_universal_c_allocation_bound": True,
"process_memory": main["process_memory"], "long_process_memory": long["process_memory"],
"long_late_rss_min_bytes": min(row["rss_bytes"] for row in late), "long_late_rss_max_bytes": max(row["rss_bytes"] for row in late),
"long_late_rss_slope_bytes_per_second": float(np.polyfit([row["iq_seconds"] for row in late], [row["rss_bytes"] for row in late], 1)[0]),
"receiver_state_bounded": checks["all_main_queues_within_caps"] and checks["long_queues_within_caps_no_overflow"]
and checks["long_complete_source_and_cleared_state"], "experiment_diagnostic_history_is_external": True}
write_result("memory.json", memory)
write_result("acceptance.json", {"checks": checks, "required_checks_pass": all(checks.values()),
"real_time_capable": all(checks.values()), "software_5fps_target_pass": all(checks.values()),
"hardware_validated": False, "head": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip(),
"source_code_sha256": {str(path.relative_to(ROOT)): hashlib.sha256(path.read_bytes()).hexdigest()
for path in sorted((ROOT/"experiments").glob("lab053_*.py"))},
"paced_service_partition_seconds": partition, "paced_service_processing_per_rf_second": main["processing_per_rf_second"],
"saturated_wall_processing_per_rf_second": throughput["processing_per_rf_second"],
"estimated_single_worker_service_for_40_packets_seconds": 40*main["valid_packet_total"]["mean"],
"budget_note": "perf_counter service durations; Windows thread_time has 15.625ms accounting quantization", "rf_tx": 0})
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
figure, axes = plt.subplots(2, 2, figsize=(12, 8), constrained_layout=True)
for number, row in enumerate(repeats["paced"], 1):
axes[0, 0].plot([image["image_id"] for image in row["images"]],
[1000*image["image_age_seconds"] for image in row["images"]], label=f"Проход {number}")
axes[0, 0].axhline(200, color="red", linestyle="--", label="Предел 200 мс")
axes[0, 0].set(xlabel="Номер изображения", ylabel="Возраст JPEG, мс", title="Подача IQ по часам: 5 свежих кадров/с")
axes[0, 0].legend(fontsize=8)
axes[0, 1].bar(["Lab052", "Lab053\nработа стадий", "Lab053\nбез ожидания"], [1.2192732766011487, main["processing_per_rf_second"], throughput["processing_per_rf_second"]])
axes[0, 1].axhline(1, color="red", linestyle="--")
axes[0, 1].axhline(.8, color="green", linestyle=":")
axes[0, 1].set(ylabel="Время обработки / длительность IQ", title="Пропускная способность")
axes[1, 0].plot([row["capture_timestamp"] for row in long["images"]], [1000*row["image_age_seconds"] for row in long["images"]])
axes[1, 0].axhline(200, color="red", linestyle="--")
axes[1, 0].set(xlabel="Время входной сессии, с", ylabel="Возраст JPEG, мс", title="Длинная сессия: 120 точных JPEG")
axes[1, 1].plot([row["iq_seconds"] for row in rss], [row["rss_bytes"]/2**20 for row in rss], marker=".")
axes[1, 1].set(xlabel="Принятая длительность IQ, с", ylabel="RSS процесса, МиБ", title="Память в длинной сессии")
for axis in axes.flat:
axis.grid(alpha=.2)
figure.savefig(OUTPUT / "performance_and_age.png", dpi=150)
figure.savefig(OUTPUT / "performance_and_age.svg")
plt.close(figure)
svg = OUTPUT / "performance_and_age.svg"
svg.write_text("\n".join(line.rstrip() for line in svg.read_text(encoding="utf-8").splitlines()) + "\n", encoding="utf-8")
print(json.dumps({"checks": checks, "memory": memory, "partition": partition}), flush=True)
def run_functional_tests() -> None:
assert distribution([1, 2, 3])["median"] == 2
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--stage", choices=("profile", "source", "optimized", "report"), default="profile")
parser.add_argument("--workers", type=int, default=1)
parser.add_argument("--frames", type=int, default=30)
arguments = parser.parse_args()
if arguments.stage == "optimized":
optimized_stream(arguments.workers)
elif arguments.stage == "source":
prepare_source(arguments.frames)
elif arguments.stage == "report":
summarize_results()
else:
baseline_profile()
if __name__ == "__main__":
main()

145
experiments/lab053_regression.py Normal file
View File

@@ -0,0 +1,145 @@
"""Lab053: сравнение независимых решений Lab052/053 после полного приёма."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
import sys
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import numpy as np
from experiments import lab052_candidate_detection as baseline
from experiments.lab053_receiver_profile import SOURCE, write_result
from experiments.lab053_streaming_receiver import StreamingReceiver
class VerdictObserver:
def __init__(self):
self.decisions = []
self.frames = {}
def scores(self, *args):
pass
def outcome(self, base, candidate, outcome):
self.decisions.append((base + candidate.sample_offset, outcome.reason,
base + outcome.refined_start_sample, outcome.accepted))
if outcome.accepted:
frame = outcome.frame
self.frames[base + outcome.refined_start_sample] = {
"end": base + outcome.next_sample, "timing": frame.timing_offset_samples,
"cfo_hz": frame.estimated_cfo_hz, "phase_rad": frame.estimated_phase_rad,
"polarity": frame.bpsk_polarity, "crc": outcome.crc,
"packet_bytes": frame.reception.packet_bytes,
"soft": frame.post_preamble_soft.copy()}
def read_all(receiver, source=SOURCE):
packets = []
try:
with source.open("rb") as capture:
while raw := capture.read(receiver.block_samples * 8):
packets.extend(receiver.push_iq_block(np.frombuffer(raw, np.complex64)))
packets.extend(receiver.finish())
return packets
finally:
if hasattr(receiver, "close"):
receiver.close()
def equivalence():
old = VerdictObserver()
read_all(baseline.StreamingReceiver(observer=old))
results = []
for workers in (1, 2, 4):
new = VerdictObserver()
receiver = StreamingReceiver(workers=workers, observer=new)
read_all(receiver)
same_starts = set(old.frames) == set(new.frames)
rows = []
for start in sorted(set(old.frames) & set(new.frames)):
first, second = old.frames[start], new.frames[start]
rows.append({"start_sample": start,
**{f"same_{key}": first[key] == second[key] for key in ("end", "timing", "polarity", "crc", "packet_bytes")},
"cfo_difference_hz": abs(first["cfo_hz"] - second["cfo_hz"]),
"phase_difference_rad": abs(first["phase_rad"] - second["phase_rad"]),
"maximum_soft_difference": float(np.max(np.abs(first["soft"] - second["soft"]))),
"packet_sha256": hashlib.sha256(second["packet_bytes"]).hexdigest()})
checks = {"same_240_starts": same_starts and len(rows) == 240,
"same_candidate_policy_sequence": old.decisions == new.decisions,
"same_phy_timing_crc_packet_bits": all(all(row[f"same_{key}"] for key in
("end", "timing", "polarity", "crc", "packet_bytes")) for row in rows),
"cfo_within_002_hz": all(row["cfo_difference_hz"] < .002 for row in rows),
"phase_within_2e5_rad": all(row["phase_difference_rad"] < 2e-5 for row in rows),
"soft_within_1e4": all(row["maximum_soft_difference"] < 1e-4 for row in rows)}
result = {"workers": workers, "checks": checks, "passed": all(checks.values()),
"strict_candidates": receiver.candidate_count, "accepted": receiver.packet_count,
"maximum_cfo_difference_hz": max(row["cfo_difference_hz"] for row in rows),
"maximum_phase_difference_rad": max(row["phase_difference_rad"] for row in rows),
"maximum_soft_difference": max(row["maximum_soft_difference"] for row in rows), "packets": rows}
results.append(result)
print(json.dumps({key: value for key, value in result.items() if key != "packets"}), flush=True)
write_result("equivalence.json", {"baseline": "committed Lab052", "workers": results,
"passed": all(row["passed"] for row in results), "oracle_used_in_receiver": False, "rf_tx": 0})
def sensitivity():
physical = baseline.phy.lab045
payload = bytes(range(256))
plan = physical.build_transmission_plan(payload)
raw = baseline.phy.build_parameterized_tx_cs8(plan, baseline.phy.PhyMode(500000))
components = np.frombuffer(raw, np.int8).astype(np.float32)
clean = components[0::2] + 1j * components[1::2]
energy = float(np.sum(np.abs(clean) ** 2)) / plan.frame_symbol_count
rows = []
for ebn0 in (0, 1, 2, 3, 7):
for cfo in (-100, 10, 100):
for seed, scale in ((52031, .002), (52032, .2), (52033, 1.)):
random_generator = np.random.default_rng(seed)
sigma = np.sqrt(energy / (2 * 10 ** (ebn0 / 10)))
phase = .73 + (seed % 2) * np.pi
values = clean * np.exp(1j * (phase + 2*np.pi*cfo*np.arange(len(clean))/8000000))
values += sigma * (random_generator.normal(size=len(clean)) + 1j*random_generator.normal(size=len(clean)))
samples = np.concatenate((np.zeros(63217, np.complex64), (values * scale).astype(np.complex64),
np.zeros(350000, np.complex64)))
verdicts = []
for constructor in (baseline.StreamingReceiver, StreamingReceiver):
receiver = constructor()
packets = []
try:
for start in range(0, len(samples), receiver.block_samples):
packets.extend(receiver.push_iq_block(samples[start:start + receiver.block_samples]))
packets.extend(receiver.finish())
finally:
if hasattr(receiver, "close"):
receiver.close()
verdicts.append(len(packets) == 1 and packets[0].payload == payload)
rows.append({"ebn0_db": ebn0, "cfo_hz": cfo, "seed": seed, "scale": scale,
"baseline_correct": verdicts[0], "optimized_correct": verdicts[1]})
result = {"count": len(rows), "baseline_correct": sum(row["baseline_correct"] for row in rows),
"optimized_correct": sum(row["optimized_correct"] for row in rows),
"lost_baseline_packets": sum(row["baseline_correct"] and not row["optimized_correct"] for row in rows),
"same_verdicts": all(row["baseline_correct"] == row["optimized_correct"] for row in rows),
"cases": rows, "rf_tx": 0}
write_result("sensitivity.json", result)
print(json.dumps({key: value for key, value in result.items() if key != "cases"}), flush=True)
def run_functional_tests() -> None:
assert VerdictObserver().decisions == []
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--stage", choices=("equivalence", "sensitivity"), default="equivalence")
arguments = parser.parse_args()
{"equivalence": equivalence, "sensitivity": sensitivity}[arguments.stage]()
if __name__ == "__main__":
main()

287
experiments/lab053_streaming_receiver.py Normal file
View File

@@ -0,0 +1,287 @@
"""Lab053: прежняя политика Lab052 с независимыми строгими вычислениями."""
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import sys
import time
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import numpy as np
from experiments import lab052_candidate_detection as detection
from experiments.lab053_packet_processing import PacketProcessor
class CandidateQueueOverflow(RuntimeError):
pass
class DetectionWindow(detection.CandidateFrontEnd):
"""Те же coarse/split функции Lab052; строгие FFT здесь не создаются."""
def __init__(self, samples, owned_end, decimator, base):
self.samples, self.owned_end = samples, owned_end
self.decimator, self.base = decimator, base
class MeasuredDecimation(detection.previous.DecimationCache):
def __init__(self, decimation):
super().__init__(decimation)
self.seconds = 0.0
def get(self, samples, base):
started = time.perf_counter()
try:
return super().get(samples, base)
finally:
self.seconds += time.perf_counter() - started
class StreamingReceiver(detection.StreamingReceiver):
"""Порядок принятия прежний; опережающие результаты применяются по очереди.
Только представители групп могут вычисляться одновременно. Альтернативы
остаются после FAIL, пропуск внутри кадра разрешает только фактический CRC.
Неиспользованные опережающие проверки отдельно учтены, их verdict игнорируется.
"""
def __init__(self, *, workers=1, candidate_capacity=4, observer=None, packet_observer=None,
processor=None, block_samples=65536):
if type(workers) is not int or workers not in (1, 2, 4):
raise ValueError("Допустимы 1, 2 или 4 работника")
if type(candidate_capacity) is not int or candidate_capacity < workers:
raise ValueError("Вместимость кандидатов должна быть не меньше числа работников")
super().__init__(block_samples=block_samples, observer=observer)
self.processor = processor or PacketProcessor()
self.packet_observer = packet_observer
self.workers, self.candidate_capacity = workers, candidate_capacity
self.executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="lab053_strict") if workers > 1 else None
self.candidate_queue_peak = self.candidate_overflow = self.pending_jobs = 0
self.speculative_checks = self.discarded_speculative_checks = 0
self.speculative_seconds = self.discarded_speculative_seconds = 0.0
self.last_packet_rows = []
self.ring_capacity = self.buffer_limit_samples
self.ring = np.empty(2 * self.ring_capacity, np.complex64)
self.decimator = MeasuredDecimation(self.decimation)
self.input_preparation_seconds = 0.0
def submit(self, candidate):
if self.closed:
raise RuntimeError("Поток уже закрыт")
if self.pending_jobs >= self.candidate_capacity:
self.candidate_overflow += 1
raise CandidateQueueOverflow("Очередь строгих кандидатов заполнена")
self.pending_jobs += 1
self.candidate_queue_peak = max(self.candidate_queue_peak, self.pending_jobs)
try:
return self.executor.submit(self.processor.decode, self.buffer, candidate)
except BaseException:
self.pending_jobs -= 1
raise
def _scan(self, owned_end):
self.buffer.flags.writeable = False
front = DetectionWindow(self.buffer, owned_end - self.base_sample, self.decimator, self.base_sample)
started = time.perf_counter()
scores = front.coarse_scores(self.buffer, self.reference)
self.coarse_seconds += time.perf_counter() - started
peaks = detection.offline._all_local_peak_indexes(scores)
offsets = self.base_sample + peaks * self.decimation
owned_peaks = peaks[(offsets >= self.scan_sample) & (offsets < owned_end)]
threshold = np.float32(detection.phy.lab047.FIXED_HARDWARE_CANDIDATE_THRESHOLD)
first_stage = owned_peaks[scores[owned_peaks] >= threshold]
self.detection_statistics["total_local_peaks"] += len(owned_peaks)
self.detection_statistics["first_stage_candidates"] += len(first_stage)
self.peak_candidate_count = max(self.peak_candidate_count, len(first_stage))
if self.observer is not None:
self.observer.scores(scores, self.base_sample, self.scan_sample, owned_end, self.decimation)
started = time.perf_counter()
parts = front.split_scores(self.reference, self.rule.parts, len(scores)) if len(scores) else np.empty((0, 2))
passed = np.all(parts[first_stage] >= detection.part_threshold(self.rule.parts), axis=1)
self.cheap_seconds += time.perf_counter() - started
self.detection_statistics["cheap_stage_checked"] += len(first_stage)
self.detection_statistics["cheap_stage_rejected"] += int(np.sum(~passed))
indexes = first_stage[passed]
if self.observer is not None and hasattr(self.observer, "cheap"):
for index, keep in zip(first_stage, passed):
self.observer.cheap(self.base_sample + int(index) * self.decimation,
float(scores[index]), parts[index], bool(keep))
started = time.perf_counter()
groups = list(detection.peak_groups(indexes, self.rule.cluster_width_samples // self.decimation))
self.cluster_seconds += time.perf_counter() - started
self.detection_statistics["clusters"] += len(groups)
self.peak_cluster_count = max(self.peak_cluster_count, len(groups))
self.candidate_state_count = len(indexes)
self.peak_eligible_count = max(self.peak_eligible_count, len(indexes))
self.working_detection_arrays_peak_bytes = max(self.working_detection_arrays_peak_bytes,
front.detection_values.nbytes + scores.nbytes * 4 + parts.nbytes + (len(peaks) + len(indexes)) * 24)
ordered = [group[np.argsort(scores[group], kind="stable")[::-1]] for group in groups]
futures = {}
emitted = []
def candidate_at(index):
return detection.offline.CoarseCandidate(int(index) * self.decimation, float(scores[index]))
try:
if self.executor:
for group in ordered:
if len(futures) == self.candidate_capacity:
break
index = int(group[0])
if self.base_sample + index * self.decimation >= self.accepted_until - self.decimation:
futures[index] = self.submit(candidate_at(index))
self.speculative_checks += 1
attempted_groups = set()
for group_number, index in ((number, int(item)) for number, group in enumerate(ordered) for item in group):
absolute = self.base_sample + index * self.decimation
if absolute < self.accepted_until - self.decimation:
self.detection_statistics["inside_accepted_frame"] += 1
continue
self.detection_statistics["alternative_checks" if group_number in attempted_groups else "representative_checks"] += 1
attempted_groups.add(group_number)
self.candidate_count += 1
candidate = candidate_at(index)
if index in futures:
future = futures.pop(index)
try:
outcome, row = future.result()
finally:
self.pending_jobs -= 1
self.speculative_seconds += row["seconds"]
else:
if self.pending_jobs >= self.candidate_capacity:
# Освободить один уже вычисленный, но пропущенный результат.
skipped = next((key for key in futures
if self.base_sample + key*self.decimation < self.accepted_until-self.decimation), None)
if skipped is None:
self.candidate_overflow += 1
raise CandidateQueueOverflow("Нет места для альтернативного кандидата")
unused, discarded = futures.pop(skipped).result()
self.pending_jobs -= 1
self.discarded_speculative_checks += 1
self.discarded_speculative_seconds += discarded["seconds"]
self.candidate_queue_peak = max(self.candidate_queue_peak, self.pending_jobs + 1)
if self.executor:
future = self.submit(candidate)
try:
outcome, row = future.result()
finally:
self.pending_jobs -= 1
else:
outcome, row = self.processor.decode(self.buffer, candidate, self.candidate_count)
self.strict_seconds += row["seconds"]
row.update(base_sample=self.base_sample, absolute_refined=self.base_sample + outcome.refined_start_sample)
self.last_packet_rows.append(row)
if self.packet_observer is not None:
self.packet_observer(row, outcome)
if self.observer is not None:
self.observer.outcome(self.base_sample, candidate, outcome)
if outcome.accepted and outcome.crc == detection.phy.lab045.CRC_PASS and outcome.frame is not None:
self.true_strict_seconds += row["seconds"]
self.accepted_until = self.base_sample + outcome.next_sample
self.packet_count += 1
emitted.append(detection.previous.previous.ReceivedPacket(
self.base_sample + outcome.refined_start_sample, self.accepted_until,
self.total_samples, bytes(outcome.frame.reception.payload_bytes)))
else:
self.false_strict_seconds += row["seconds"]
self.rejected_reasons[outcome.reason] += 1
finally:
active_error = sys.exc_info()[0] is not None
cleanup_error = None
try:
for future in futures.values():
try:
unused, row = future.result()
self.discarded_speculative_checks += 1
self.discarded_speculative_seconds += row["seconds"]
except BaseException as error:
if cleanup_error is None:
cleanup_error = error
finally:
self.pending_jobs = self.candidate_state_count = 0
if active_error or cleanup_error is not None:
self.close()
if not active_error and cleanup_error is not None:
raise cleanup_error
self.blocks_processed += 1
self.scan_sample = owned_end
retain_start = max(0, self.scan_sample - self.margin)
trim = retain_start - self.base_sample
if trim > 0:
self.base_sample = retain_start
start = self.base_sample % self.ring_capacity
self.buffer = self.ring[start:start + self.total_samples - self.base_sample]
return emitted
def push_iq_block(self, samples):
preparation_started = time.perf_counter()
if self.closed:
raise RuntimeError("Поток уже закрыт")
samples = np.asarray(samples, dtype=np.complex64)
if samples.ndim != 1 or samples.size > self.block_samples or not np.all(np.isfinite(samples)):
raise ValueError("Нужен конечный одномерный IQ-блок не больше block_samples")
self.last_packet_rows = []
started = time.perf_counter()
if self.total_samples + samples.size - self.base_sample > self.buffer_limit_samples:
raise RuntimeError("Превышен предел буфера IQ")
# Две копии только НОВОГО блока дают непрерывный view через стык кольца.
# Старое окно не переносится. Все строгие jobs предыдущего push завершены.
position = self.total_samples % self.ring_capacity
first = min(samples.size, self.ring_capacity - position)
self.ring[position:position + first] = samples[:first]
self.ring[position + self.ring_capacity:position + self.ring_capacity + first] = samples[:first]
remaining = samples.size - first
if remaining:
self.ring[:remaining] = samples[first:]
self.ring[self.ring_capacity:self.ring_capacity + remaining] = samples[first:]
self.total_samples += samples.size
start = self.base_sample % self.ring_capacity
self.buffer = self.ring[start:start + self.total_samples - self.base_sample]
self.peak_buffer_samples = max(self.peak_buffer_samples, self.buffer.size)
if self.buffer.size > self.buffer_limit_samples:
raise RuntimeError("Превышен предел буфера IQ")
emitted = []
self.input_preparation_seconds += time.perf_counter() - preparation_started
while self.total_samples >= self.scan_sample + self.core_samples + self.lookahead:
emitted.extend(self._scan(self.scan_sample + self.core_samples))
self.processing_seconds += time.perf_counter() - started
return emitted
def finish(self):
if self.closed:
raise RuntimeError("Поток уже закрыт")
self.last_packet_rows = []
started = time.perf_counter()
try:
return self._scan(self.total_samples) if self.scan_sample < self.total_samples else []
finally:
self.close()
self.processing_seconds += time.perf_counter() - started
def close(self):
"""Дождаться всех выполняющихся jobs до освобождения IQ и счётчиков."""
if self.executor:
self.executor.shutdown(wait=True, cancel_futures=True)
self.pending_jobs = self.candidate_state_count = 0
self.buffer = np.empty(0, np.complex64)
self.ring = np.empty(0, np.complex64)
self.decimator.values = np.empty(0, np.complex64)
self.closed = True
def run_functional_tests() -> None:
assert StreamingReceiver().finish() == []
def main() -> None:
run_functional_tests()
print("Lab053 receiver functional checks PASS; RF TX=0")
if __name__ == "__main__":
main()

98
experiments/lab053_viterbi.py Normal file
View File

@@ -0,0 +1,98 @@
"""Lab053: те же переходы soft Viterbi, циклы компилируются только для CPU."""
from __future__ import annotations
from pathlib import Path
import sys
import time
import numpy as np
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from experiments import lab051_streaming_receiver as previous
runtime = Path(__file__).resolve().parents[1] / ".lab053_cpu_runtime/packages"
if runtime.is_dir():
sys.path.insert(0, str(runtime))
try:
from numba import njit
except ImportError:
njit = None
conv = previous.lab045.conv
symbols = conv._BRANCH_SYMBOLS
CODES = (2 * (symbols[:, :, 0] > 0) + (symbols[:, :, 1] > 0)).astype(np.intp)
PREDECESSORS = conv._PREDECESSORS.copy()
CODES.flags.writeable = PREDECESSORS.flags.writeable = False
def trellis_decode(metrics, predecessors, codes):
"""Порядок сложений исходный; строгое < сохраняет tie в ветвь 0.
Не объединяем несколько шагов: это могло бы изменить решение при
равных округлённых суммах. Старт/финиш 0, 64 состояния, хвост 6 бит.
"""
step_count = metrics.shape[0]
path = np.full(64, 1e18, dtype=np.float64)
path[0] = 0.0
next_path = np.empty(64, np.float64)
choices = np.empty((step_count, 64), np.uint8)
for step in range(step_count):
for state in range(64):
zero = path[predecessors[state, 0]] + metrics[step, codes[state, 0]]
one = path[predecessors[state, 1]] + metrics[step, codes[state, 1]]
take_one = one < zero
choices[step, state] = take_one
next_path[state] = one if take_one else zero
path, next_path = next_path, path
decoded = np.empty(step_count, np.uint8)
state = 0
for step in range(step_count - 1, -1, -1):
decoded[step] = state >> 5
state = predecessors[state, choices[step, state]]
assert state == 0
return decoded[:-6]
compiled_trellis = njit(cache=True, nogil=True, fastmath=False)(trellis_decode) if njit is not None else None
def viterbi_decode_soft(soft_values, information_bit_count):
if compiled_trellis is None:
return previous.fast_viterbi(soft_values, information_bit_count)
if not isinstance(information_bit_count, (int, np.integer)) or information_bit_count <= 0:
return previous.ORIGINAL_VITERBI(soft_values, information_bit_count)
values = np.asarray(soft_values, dtype=np.float64)
if (values.ndim != 1 or values.size != conv.encoded_bit_count(information_bit_count)
or not np.all(np.isfinite(values))):
return previous.ORIGINAL_VITERBI(soft_values, information_bit_count)
received = values.reshape(int(information_bit_count) + 6, 2)
negative = (received + 1.0) ** 2
positive = (received - 1.0) ** 2
metrics = np.column_stack((negative[:, 0] + negative[:, 1], negative[:, 0] + positive[:, 1],
positive[:, 0] + negative[:, 1], positive[:, 0] + positive[:, 1]))
return compiled_trellis(metrics, PREDECESSORS, CODES)
def warmup() -> dict:
started = time.perf_counter()
values = np.ones(conv.encoded_bit_count(8), np.float64)
assert np.array_equal(viterbi_decode_soft(values, 8), previous.fast_viterbi(values, 8))
return {"backend": "NUMBA_CPU" if compiled_trellis is not None else "NUMPY_LAB051",
"warmup_seconds": time.perf_counter() - started, "fastmath": False, "gpu_used": False}
def run_functional_tests() -> None:
values = np.zeros(conv.encoded_bit_count(8), np.float64)
assert np.array_equal(viterbi_decode_soft(values, 8), previous.fast_viterbi(values, 8))
def main() -> None:
print(warmup())
run_functional_tests()
if __name__ == "__main__":
main()