313 lines
17 KiB
Python
313 lines
17 KiB
Python
"""Lab053: ограниченный CPU-конвейер с фактическим временем выдачи JPEG.
|
||
|
||
Непрерывный IQ приходит по часовому расписанию либо с обратным давлением
|
||
для измерения пропускной способности. Переполнение не склеивает разрыв IQ:
|
||
сессия останавливается с явной причиной. Оборудование не используется.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import math
|
||
from pathlib import Path
|
||
from queue import Empty, Full, Queue
|
||
import sys
|
||
from threading import Event, Thread
|
||
import time
|
||
|
||
if __package__ in (None, ""):
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||
|
||
import numpy as np
|
||
|
||
from experiments import lab052_detector_study as study
|
||
from experiments.lab051_receiver_performance import ImageSink
|
||
from experiments.lab051_receiver_profile import process_memory
|
||
from experiments.lab053_receiver_profile import SOURCE, OUTPUT, distribution, write_result
|
||
from experiments.lab053_streaming_receiver import StreamingReceiver
|
||
from experiments.lab053_packet_processing import PacketProcessor, previous
|
||
|
||
|
||
class BoundedQueue(Queue):
|
||
"""Пик фиксируется под тем же lock, что помещение в очередь."""
|
||
|
||
def __init__(self, capacity):
|
||
if type(capacity) is not int or capacity <= 0:
|
||
raise ValueError("Очередь должна иметь положительную целую вместимость")
|
||
super().__init__(maxsize=capacity)
|
||
self.peak = self.overflows = 0
|
||
|
||
def _put(self, item):
|
||
super()._put(item)
|
||
self.peak = max(self.peak, self._qsize())
|
||
|
||
def offer(self, item):
|
||
try:
|
||
self.put_nowait(item)
|
||
except Full:
|
||
self.overflows += 1
|
||
return False
|
||
return True
|
||
|
||
|
||
def run_pipeline(*, workers=1, paced=True, source=SOURCE, frame_count=30,
|
||
iq_capacity=8, candidate_capacity=4, output_capacity=16,
|
||
processing_delay=0.0, collect_packets=True, numpy_viterbi=False) -> dict:
|
||
"""Эталон доступен только оценке после сборки/decode; RX получает только IQ."""
|
||
if type(frame_count) is not int or frame_count <= 0:
|
||
raise ValueError("Число входных изображений должно быть положительным целым")
|
||
if not math.isfinite(processing_delay) or processing_delay < 0:
|
||
raise ValueError("Задержка должна быть конечной и неотрицательной")
|
||
try:
|
||
source_bytes = Path(source).stat().st_size
|
||
except OSError:
|
||
source_bytes = None # produce запишет точное исключение открытия.
|
||
sequence = study.sequence_fixture()
|
||
encoded = {number + 1: sequence[number % len(sequence)].jpeg for number in range(frame_count)}
|
||
capture_times = {number + 1: number / 5 for number in range(frame_count)}
|
||
processor = PacketProcessor(viterbi=previous.fast_viterbi) if numpy_viterbi else None
|
||
receiver = StreamingReceiver(workers=workers, candidate_capacity=candidate_capacity, processor=processor)
|
||
startup = receiver.processor.startup
|
||
if numpy_viterbi:
|
||
startup = {**startup, "backend": "NUMPY_LAB051"}
|
||
iq_queue, output_queue = BoundedQueue(iq_capacity), BoundedQueue(output_capacity)
|
||
sink = ImageSink()
|
||
source_done, receiver_done, stop = Event(), Event(), Event()
|
||
completed, block_rows, packet_rows, fragment_rows, rss_rows, errors = [], [], [], [], [], []
|
||
input_statistics = {"read_seconds": 0.0, "read_blocks": 0, "read_samples": 0,
|
||
"maximum_release_lateness_seconds": 0.0, "backpressure_seconds": 0.0}
|
||
output_statistics = {"freshness_seconds": 0.0, "service_seconds": 0.0}
|
||
receiver_service = receiver_wait = 0.0
|
||
process_started = time.process_time()
|
||
epoch = time.perf_counter()
|
||
|
||
def fail(reason, error=None):
|
||
errors.append({"reason": reason, "error": None if error is None else repr(error)})
|
||
stop.set()
|
||
|
||
def produce():
|
||
try:
|
||
with Path(source).open("rb") as capture:
|
||
while not stop.is_set():
|
||
started = time.perf_counter()
|
||
raw = capture.read(receiver.block_samples * 8)
|
||
if len(raw) % 8:
|
||
raise ValueError("Усечённый complex64 в источнике")
|
||
samples = np.frombuffer(raw, dtype=np.complex64)
|
||
input_statistics["read_seconds"] += time.perf_counter() - started
|
||
if not samples.size:
|
||
if not input_statistics["read_samples"]:
|
||
fail("EMPTY_IQ_SOURCE")
|
||
break
|
||
input_statistics["read_samples"] += samples.size
|
||
input_statistics["read_blocks"] += 1
|
||
arrival = input_statistics["read_samples"] / 8000000
|
||
if paced:
|
||
stop.wait(max(0.0, epoch + arrival - time.perf_counter()))
|
||
if stop.is_set():
|
||
break
|
||
lateness = max(0.0, time.perf_counter() - epoch - arrival)
|
||
input_statistics["maximum_release_lateness_seconds"] = max(
|
||
lateness, input_statistics["maximum_release_lateness_seconds"])
|
||
if not iq_queue.offer((samples, arrival)):
|
||
fail("IQ_QUEUE_OVERFLOW")
|
||
break
|
||
else:
|
||
started = time.perf_counter()
|
||
while not stop.is_set():
|
||
try:
|
||
iq_queue.put((samples, arrival), timeout=.02)
|
||
break
|
||
except Full:
|
||
continue
|
||
input_statistics["backpressure_seconds"] += time.perf_counter() - started
|
||
except BaseException as error:
|
||
fail("SOURCE_EXCEPTION", error)
|
||
finally:
|
||
source_done.set()
|
||
|
||
def consume_output():
|
||
try:
|
||
while not receiver_done.is_set() or not output_queue.empty():
|
||
try:
|
||
packet, receiver_timestamp = output_queue.get(timeout=.02)
|
||
except Empty:
|
||
continue
|
||
started = time.perf_counter()
|
||
assembly_before = sink.assembly_seconds
|
||
item = sink.offer(packet, time.perf_counter() - epoch)
|
||
fragment_rows.append({"start_sample": packet.start_sample,
|
||
"seconds": sink.assembly_seconds - assembly_before})
|
||
output_statistics["service_seconds"] += time.perf_counter() - started
|
||
if item is not None:
|
||
ready = time.perf_counter() - epoch
|
||
jpeg = item.pop("jpeg")
|
||
number = item["image_id"]
|
||
# Эти сведения не передаются назад в детектор/PHY/assembler.
|
||
started = time.perf_counter()
|
||
capture_time = capture_times.get(number)
|
||
age = ready - capture_time if capture_time is not None else None
|
||
fresh = paced and age is not None and 0 <= age <= .2
|
||
output_statistics["freshness_seconds"] += time.perf_counter() - started
|
||
completed.append({**item, "decode_complete_timestamp": ready,
|
||
"receiver_packet_timestamp": receiver_timestamp,
|
||
"capture_timestamp": capture_time, "image_age_seconds": age if paced else None,
|
||
"fresh": fresh, "sha256": hashlib.sha256(jpeg).hexdigest(),
|
||
"sha_match": number in encoded and jpeg == encoded[number]})
|
||
except BaseException as error:
|
||
fail("OUTPUT_EXCEPTION", error)
|
||
|
||
producer = Thread(target=produce, name="lab053_iq_source")
|
||
consumer = Thread(target=consume_output, name="lab053_jpeg_output")
|
||
producer.start()
|
||
consumer.start()
|
||
|
||
def emit(packets):
|
||
for packet in packets:
|
||
if not output_queue.offer((packet, time.perf_counter() - epoch)):
|
||
fail("OUTPUT_QUEUE_OVERFLOW")
|
||
return
|
||
|
||
try:
|
||
while not stop.is_set() and (not source_done.is_set() or not iq_queue.empty()):
|
||
waiting = time.perf_counter()
|
||
try:
|
||
samples, arrival = iq_queue.get(timeout=.02)
|
||
except Empty:
|
||
receiver_wait += time.perf_counter() - waiting
|
||
continue
|
||
receiver_wait += time.perf_counter() - waiting
|
||
started = time.perf_counter()
|
||
block_start = started - epoch
|
||
if processing_delay:
|
||
time.sleep(processing_delay)
|
||
packets = receiver.push_iq_block(samples)
|
||
cost = time.perf_counter() - started
|
||
receiver_service += cost
|
||
emit(packets)
|
||
if collect_packets:
|
||
packet_rows.extend(receiver.last_packet_rows)
|
||
block_rows.append({"samples_received": receiver.total_samples, "arrival_time": arrival,
|
||
"worker_start": block_start, "worker_end": time.perf_counter() - epoch,
|
||
"processing_seconds": cost, "iq_queue_size": iq_queue.qsize(),
|
||
"candidate_queue_peak": receiver.candidate_queue_peak,
|
||
"lag_seconds": time.perf_counter() - epoch - arrival if paced else None,
|
||
"packets": len(packets)})
|
||
if len(block_rows) % 128 == 0:
|
||
rss_rows.append({"iq_seconds": receiver.total_samples / 8000000,
|
||
"iq_queue": iq_queue.qsize(), "candidate_jobs": receiver.pending_jobs,
|
||
"retained_iq_bytes": receiver.buffer.nbytes, **process_memory()})
|
||
if not stop.is_set():
|
||
started = time.perf_counter()
|
||
emit(receiver.finish())
|
||
receiver_service += time.perf_counter() - started
|
||
if collect_packets:
|
||
packet_rows.extend(receiver.last_packet_rows)
|
||
except BaseException as error:
|
||
fail("RECEIVER_EXCEPTION", error)
|
||
finally:
|
||
stop.set()
|
||
producer.join()
|
||
receiver.close()
|
||
receiver_done.set()
|
||
consumer.join()
|
||
for queue in (iq_queue, output_queue):
|
||
while True:
|
||
try:
|
||
queue.get_nowait()
|
||
except Empty:
|
||
break
|
||
wall = time.perf_counter() - epoch
|
||
process_cpu = time.process_time() - process_started
|
||
iq_seconds = source_bytes / 8 / 8000000 if source_bytes is not None else None
|
||
horizon = frame_count / 5
|
||
ages = [row["image_age_seconds"] for row in completed if row["image_age_seconds"] is not None]
|
||
fresh = [row for row in completed if row["fresh"] and row["decode_complete_timestamp"] <= horizon]
|
||
valid = [row for row in packet_rows if row["accepted"]]
|
||
# В paced-опыте wall включает ожидание прихода IQ и не является service time.
|
||
service_sum = receiver_service + input_statistics["read_seconds"] + output_statistics["service_seconds"] + output_statistics["freshness_seconds"]
|
||
result = {"mode": "PACED_WALL_CLOCK" if paced else "SATURATED_BOUNDED_REPLAY",
|
||
"source": str(source), "workers": workers, "startup": startup,
|
||
"input_fps": 5, "input_frames": frame_count, "observation_seconds": horizon,
|
||
"iq_seconds": iq_seconds, "consumed_iq_seconds": receiver.total_samples / 8000000,
|
||
"source_consumed_completely": source_bytes is not None and receiver.total_samples * 8 == source_bytes and not errors,
|
||
"full_wall_seconds": wall, "process_cpu_seconds": process_cpu,
|
||
"processing_seconds": service_sum if paced else wall,
|
||
"processing_per_rf_second": (service_sum if paced else wall) / iq_seconds if iq_seconds else None,
|
||
"service_sum_seconds": service_sum, "receiver_service_seconds": receiver_service,
|
||
"receiver_wait_seconds": receiver_wait, "input_statistics": input_statistics,
|
||
"output_statistics": output_statistics, "assembly_seconds": sink.assembly_seconds,
|
||
"jpeg_decode_seconds": sink.jpeg_decode_seconds,
|
||
"coarse_seconds": receiver.coarse_seconds, "cheap_seconds": receiver.cheap_seconds,
|
||
"shared_front_end_seconds": receiver.decimator.seconds,
|
||
"coarse_detection_exclusive_seconds": receiver.coarse_seconds - receiver.decimator.seconds,
|
||
"iq_input_preparation_seconds": receiver.input_preparation_seconds,
|
||
"cluster_seconds": receiver.cluster_seconds, "true_strict_seconds": receiver.true_strict_seconds,
|
||
"false_strict_seconds": receiver.false_strict_seconds,
|
||
"rx_physical_packets": receiver.packet_count, "candidate_count": receiver.candidate_count,
|
||
"received_images": len(completed), "object_jpeg_integrity": len(completed) == frame_count and all(row["sha_match"] for row in completed),
|
||
"fresh_displayed": len(fresh) if paced else None, "displayed": len(fresh) if paced else None,
|
||
"fresh_display_fps": len(fresh) / horizon if paced else None,
|
||
"image_age_min_seconds": min(ages) if ages else None,
|
||
"image_age_median_seconds": float(np.median(ages)) if ages else None,
|
||
"image_age_max_seconds": max(ages) if ages else None, "max_video_age_seconds": .2,
|
||
"iq_queue_capacity": iq_capacity, "iq_queue_peak": iq_queue.peak, "iq_overflow": iq_queue.overflows,
|
||
"candidate_queue_capacity": candidate_capacity, "candidate_queue_peak": receiver.candidate_queue_peak,
|
||
"candidate_overflow": receiver.candidate_overflow,
|
||
"output_queue_capacity": output_capacity, "output_queue_peak": output_queue.peak,
|
||
"output_overflow": output_queue.overflows,
|
||
"speculative_checks": receiver.speculative_checks,
|
||
"discarded_speculative_checks": receiver.discarded_speculative_checks,
|
||
"discarded_speculative_seconds": receiver.discarded_speculative_seconds,
|
||
"retained_iq_peak_bytes": receiver.peak_buffer_samples * 8,
|
||
"allocated_iq_ring_bytes": 2 * receiver.ring_capacity * 8,
|
||
"iq_queue_limit_bytes": iq_capacity * receiver.block_samples * 8,
|
||
"working_arrays_engineering_estimate_bytes": 2 * receiver.ring_capacity * 8 + (iq_capacity + 2) * receiver.block_samples * 8 + workers * 10 * receiver.lookahead * 8,
|
||
"process_memory": process_memory(), "rss_samples": rss_rows,
|
||
"detector": dict(receiver.detection_statistics), "rejected_reasons": dict(receiver.rejected_reasons),
|
||
"retained_after_close_bytes": receiver.buffer.nbytes + receiver.ring.nbytes + receiver.decimator.values.nbytes,
|
||
"pending_jobs_after_close": receiver.pending_jobs, "errors": errors,
|
||
"valid_packet_total": distribution([row["seconds"] for row in valid]),
|
||
"valid_packet_thread_cpu": distribution([row["thread_cpu_seconds"] for row in valid]),
|
||
"fragment_processing": distribution([row["seconds"] for row in fragment_rows]),
|
||
"valid_packet_stages": {name: distribution([row["exclusive_stages"].get(name, 0) for row in valid])
|
||
for name in sorted({name for row in valid for name in row["stages"]})},
|
||
"images": completed, "blocks": block_rows, "packet_rows": packet_rows,
|
||
"camera_display_latency": "UNKNOWN", "independent_live_capture_timestamp": False,
|
||
"freshness_evaluation": "post-JPEG source timestamp; does not feed RX verdict", "rf_tx": 0}
|
||
return result
|
||
|
||
|
||
def run_functional_tests() -> None:
|
||
queue = BoundedQueue(1)
|
||
assert queue.offer(1) and not queue.offer(2)
|
||
assert queue.overflows == 1 and queue.peak == 1 and queue.get_nowait() == 1
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument("--workers", type=int, default=1)
|
||
parser.add_argument("--unpaced", action="store_true")
|
||
parser.add_argument("--name")
|
||
parser.add_argument("--source", type=Path, default=SOURCE)
|
||
parser.add_argument("--frames", type=int, default=30)
|
||
parser.add_argument("--processing-delay", type=float, default=0)
|
||
parser.add_argument("--numpy-viterbi", action="store_true")
|
||
arguments = parser.parse_args()
|
||
result = run_pipeline(workers=arguments.workers, paced=not arguments.unpaced,
|
||
source=arguments.source, frame_count=arguments.frames,
|
||
processing_delay=arguments.processing_delay, collect_packets=arguments.frames <= 30,
|
||
numpy_viterbi=arguments.numpy_viterbi)
|
||
name = arguments.name or f"{'throughput' if arguments.unpaced else 'paced'}_workers_{arguments.workers}.json"
|
||
write_result(name, result)
|
||
print(json.dumps({key: result[key] for key in ("mode", "workers", "processing_seconds", "processing_per_rf_second",
|
||
"full_wall_seconds", "rx_physical_packets", "received_images", "object_jpeg_integrity", "fresh_display_fps",
|
||
"image_age_min_seconds", "image_age_median_seconds", "image_age_max_seconds", "iq_queue_peak",
|
||
"candidate_queue_peak", "iq_overflow", "candidate_overflow", "errors")}), flush=True)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|