"""Lab053: ограниченный CPU-конвейер с фактическим временем выдачи JPEG. Непрерывный IQ приходит по часовому расписанию либо с обратным давлением для измерения пропускной способности. Переполнение не склеивает разрыв IQ: сессия останавливается с явной причиной. Оборудование не используется. """ from __future__ import annotations import argparse import hashlib import json import math from pathlib import Path from queue import Empty, Full, Queue import sys from threading import Event, Thread import time if __package__ in (None, ""): sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import numpy as np from experiments import lab052_detector_study as study from experiments.lab051_receiver_performance import ImageSink from experiments.lab051_receiver_profile import process_memory from experiments.lab053_receiver_profile import SOURCE, OUTPUT, distribution, write_result from experiments.lab053_streaming_receiver import StreamingReceiver from experiments.lab053_packet_processing import PacketProcessor, previous class BoundedQueue(Queue): """Пик фиксируется под тем же lock, что помещение в очередь.""" def __init__(self, capacity): if type(capacity) is not int or capacity <= 0: raise ValueError("Очередь должна иметь положительную целую вместимость") super().__init__(maxsize=capacity) self.peak = self.overflows = 0 def _put(self, item): super()._put(item) self.peak = max(self.peak, self._qsize()) def offer(self, item): try: self.put_nowait(item) except Full: self.overflows += 1 return False return True def run_pipeline(*, workers=1, paced=True, source=SOURCE, frame_count=30, iq_capacity=8, candidate_capacity=4, output_capacity=16, processing_delay=0.0, collect_packets=True, numpy_viterbi=False) -> dict: """Эталон доступен только оценке после сборки/decode; RX получает только IQ.""" if type(frame_count) is not int or frame_count <= 0: raise ValueError("Число входных изображений должно быть положительным целым") if not math.isfinite(processing_delay) or processing_delay < 0: raise ValueError("Задержка должна быть конечной и неотрицательной") try: source_bytes = Path(source).stat().st_size except OSError: source_bytes = None # produce запишет точное исключение открытия. sequence = study.sequence_fixture() encoded = {number + 1: sequence[number % len(sequence)].jpeg for number in range(frame_count)} capture_times = {number + 1: number / 5 for number in range(frame_count)} processor = PacketProcessor(viterbi=previous.fast_viterbi) if numpy_viterbi else None receiver = StreamingReceiver(workers=workers, candidate_capacity=candidate_capacity, processor=processor) startup = receiver.processor.startup if numpy_viterbi: startup = {**startup, "backend": "NUMPY_LAB051"} iq_queue, output_queue = BoundedQueue(iq_capacity), BoundedQueue(output_capacity) sink = ImageSink() source_done, receiver_done, stop = Event(), Event(), Event() completed, block_rows, packet_rows, fragment_rows, rss_rows, errors = [], [], [], [], [], [] input_statistics = {"read_seconds": 0.0, "read_blocks": 0, "read_samples": 0, "maximum_release_lateness_seconds": 0.0, "backpressure_seconds": 0.0} output_statistics = {"freshness_seconds": 0.0, "service_seconds": 0.0} receiver_service = receiver_wait = 0.0 process_started = time.process_time() epoch = time.perf_counter() def fail(reason, error=None): errors.append({"reason": reason, "error": None if error is None else repr(error)}) stop.set() def produce(): try: with Path(source).open("rb") as capture: while not stop.is_set(): started = time.perf_counter() raw = capture.read(receiver.block_samples * 8) if len(raw) % 8: raise ValueError("Усечённый complex64 в источнике") samples = np.frombuffer(raw, dtype=np.complex64) input_statistics["read_seconds"] += time.perf_counter() - started if not samples.size: if not input_statistics["read_samples"]: fail("EMPTY_IQ_SOURCE") break input_statistics["read_samples"] += samples.size input_statistics["read_blocks"] += 1 arrival = input_statistics["read_samples"] / 8000000 if paced: stop.wait(max(0.0, epoch + arrival - time.perf_counter())) if stop.is_set(): break lateness = max(0.0, time.perf_counter() - epoch - arrival) input_statistics["maximum_release_lateness_seconds"] = max( lateness, input_statistics["maximum_release_lateness_seconds"]) if not iq_queue.offer((samples, arrival)): fail("IQ_QUEUE_OVERFLOW") break else: started = time.perf_counter() while not stop.is_set(): try: iq_queue.put((samples, arrival), timeout=.02) break except Full: continue input_statistics["backpressure_seconds"] += time.perf_counter() - started except BaseException as error: fail("SOURCE_EXCEPTION", error) finally: source_done.set() def consume_output(): try: while not receiver_done.is_set() or not output_queue.empty(): try: packet, receiver_timestamp = output_queue.get(timeout=.02) except Empty: continue started = time.perf_counter() assembly_before = sink.assembly_seconds item = sink.offer(packet, time.perf_counter() - epoch) fragment_rows.append({"start_sample": packet.start_sample, "seconds": sink.assembly_seconds - assembly_before}) output_statistics["service_seconds"] += time.perf_counter() - started if item is not None: ready = time.perf_counter() - epoch jpeg = item.pop("jpeg") number = item["image_id"] # Эти сведения не передаются назад в детектор/PHY/assembler. started = time.perf_counter() capture_time = capture_times.get(number) age = ready - capture_time if capture_time is not None else None fresh = paced and age is not None and 0 <= age <= .2 output_statistics["freshness_seconds"] += time.perf_counter() - started completed.append({**item, "decode_complete_timestamp": ready, "receiver_packet_timestamp": receiver_timestamp, "capture_timestamp": capture_time, "image_age_seconds": age if paced else None, "fresh": fresh, "sha256": hashlib.sha256(jpeg).hexdigest(), "sha_match": number in encoded and jpeg == encoded[number]}) except BaseException as error: fail("OUTPUT_EXCEPTION", error) producer = Thread(target=produce, name="lab053_iq_source") consumer = Thread(target=consume_output, name="lab053_jpeg_output") producer.start() consumer.start() def emit(packets): for packet in packets: if not output_queue.offer((packet, time.perf_counter() - epoch)): fail("OUTPUT_QUEUE_OVERFLOW") return try: while not stop.is_set() and (not source_done.is_set() or not iq_queue.empty()): waiting = time.perf_counter() try: samples, arrival = iq_queue.get(timeout=.02) except Empty: receiver_wait += time.perf_counter() - waiting continue receiver_wait += time.perf_counter() - waiting started = time.perf_counter() block_start = started - epoch if processing_delay: time.sleep(processing_delay) packets = receiver.push_iq_block(samples) cost = time.perf_counter() - started receiver_service += cost emit(packets) if collect_packets: packet_rows.extend(receiver.last_packet_rows) block_rows.append({"samples_received": receiver.total_samples, "arrival_time": arrival, "worker_start": block_start, "worker_end": time.perf_counter() - epoch, "processing_seconds": cost, "iq_queue_size": iq_queue.qsize(), "candidate_queue_peak": receiver.candidate_queue_peak, "lag_seconds": time.perf_counter() - epoch - arrival if paced else None, "packets": len(packets)}) if len(block_rows) % 128 == 0: rss_rows.append({"iq_seconds": receiver.total_samples / 8000000, "iq_queue": iq_queue.qsize(), "candidate_jobs": receiver.pending_jobs, "retained_iq_bytes": receiver.buffer.nbytes, **process_memory()}) if not stop.is_set(): started = time.perf_counter() emit(receiver.finish()) receiver_service += time.perf_counter() - started if collect_packets: packet_rows.extend(receiver.last_packet_rows) except BaseException as error: fail("RECEIVER_EXCEPTION", error) finally: stop.set() producer.join() receiver.close() receiver_done.set() consumer.join() for queue in (iq_queue, output_queue): while True: try: queue.get_nowait() except Empty: break wall = time.perf_counter() - epoch process_cpu = time.process_time() - process_started iq_seconds = source_bytes / 8 / 8000000 if source_bytes is not None else None horizon = frame_count / 5 ages = [row["image_age_seconds"] for row in completed if row["image_age_seconds"] is not None] fresh = [row for row in completed if row["fresh"] and row["decode_complete_timestamp"] <= horizon] valid = [row for row in packet_rows if row["accepted"]] # В paced-опыте wall включает ожидание прихода IQ и не является service time. service_sum = receiver_service + input_statistics["read_seconds"] + output_statistics["service_seconds"] + output_statistics["freshness_seconds"] result = {"mode": "PACED_WALL_CLOCK" if paced else "SATURATED_BOUNDED_REPLAY", "source": str(source), "workers": workers, "startup": startup, "input_fps": 5, "input_frames": frame_count, "observation_seconds": horizon, "iq_seconds": iq_seconds, "consumed_iq_seconds": receiver.total_samples / 8000000, "source_consumed_completely": source_bytes is not None and receiver.total_samples * 8 == source_bytes and not errors, "full_wall_seconds": wall, "process_cpu_seconds": process_cpu, "processing_seconds": service_sum if paced else wall, "processing_per_rf_second": (service_sum if paced else wall) / iq_seconds if iq_seconds else None, "service_sum_seconds": service_sum, "receiver_service_seconds": receiver_service, "receiver_wait_seconds": receiver_wait, "input_statistics": input_statistics, "output_statistics": output_statistics, "assembly_seconds": sink.assembly_seconds, "jpeg_decode_seconds": sink.jpeg_decode_seconds, "coarse_seconds": receiver.coarse_seconds, "cheap_seconds": receiver.cheap_seconds, "shared_front_end_seconds": receiver.decimator.seconds, "coarse_detection_exclusive_seconds": receiver.coarse_seconds - receiver.decimator.seconds, "iq_input_preparation_seconds": receiver.input_preparation_seconds, "cluster_seconds": receiver.cluster_seconds, "true_strict_seconds": receiver.true_strict_seconds, "false_strict_seconds": receiver.false_strict_seconds, "rx_physical_packets": receiver.packet_count, "candidate_count": receiver.candidate_count, "received_images": len(completed), "object_jpeg_integrity": len(completed) == frame_count and all(row["sha_match"] for row in completed), "fresh_displayed": len(fresh) if paced else None, "displayed": len(fresh) if paced else None, "fresh_display_fps": len(fresh) / horizon if paced else None, "image_age_min_seconds": min(ages) if ages else None, "image_age_median_seconds": float(np.median(ages)) if ages else None, "image_age_max_seconds": max(ages) if ages else None, "max_video_age_seconds": .2, "iq_queue_capacity": iq_capacity, "iq_queue_peak": iq_queue.peak, "iq_overflow": iq_queue.overflows, "candidate_queue_capacity": candidate_capacity, "candidate_queue_peak": receiver.candidate_queue_peak, "candidate_overflow": receiver.candidate_overflow, "output_queue_capacity": output_capacity, "output_queue_peak": output_queue.peak, "output_overflow": output_queue.overflows, "speculative_checks": receiver.speculative_checks, "discarded_speculative_checks": receiver.discarded_speculative_checks, "discarded_speculative_seconds": receiver.discarded_speculative_seconds, "retained_iq_peak_bytes": receiver.peak_buffer_samples * 8, "allocated_iq_ring_bytes": 2 * receiver.ring_capacity * 8, "iq_queue_limit_bytes": iq_capacity * receiver.block_samples * 8, "working_arrays_engineering_estimate_bytes": 2 * receiver.ring_capacity * 8 + (iq_capacity + 2) * receiver.block_samples * 8 + workers * 10 * receiver.lookahead * 8, "process_memory": process_memory(), "rss_samples": rss_rows, "detector": dict(receiver.detection_statistics), "rejected_reasons": dict(receiver.rejected_reasons), "retained_after_close_bytes": receiver.buffer.nbytes + receiver.ring.nbytes + receiver.decimator.values.nbytes, "pending_jobs_after_close": receiver.pending_jobs, "errors": errors, "valid_packet_total": distribution([row["seconds"] for row in valid]), "valid_packet_thread_cpu": distribution([row["thread_cpu_seconds"] for row in valid]), "fragment_processing": distribution([row["seconds"] for row in fragment_rows]), "valid_packet_stages": {name: distribution([row["exclusive_stages"].get(name, 0) for row in valid]) for name in sorted({name for row in valid for name in row["stages"]})}, "images": completed, "blocks": block_rows, "packet_rows": packet_rows, "camera_display_latency": "UNKNOWN", "independent_live_capture_timestamp": False, "freshness_evaluation": "post-JPEG source timestamp; does not feed RX verdict", "rf_tx": 0} return result def run_functional_tests() -> None: queue = BoundedQueue(1) assert queue.offer(1) and not queue.offer(2) assert queue.overflows == 1 and queue.peak == 1 and queue.get_nowait() == 1 def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--workers", type=int, default=1) parser.add_argument("--unpaced", action="store_true") parser.add_argument("--name") parser.add_argument("--source", type=Path, default=SOURCE) parser.add_argument("--frames", type=int, default=30) parser.add_argument("--processing-delay", type=float, default=0) parser.add_argument("--numpy-viterbi", action="store_true") arguments = parser.parse_args() result = run_pipeline(workers=arguments.workers, paced=not arguments.unpaced, source=arguments.source, frame_count=arguments.frames, processing_delay=arguments.processing_delay, collect_packets=arguments.frames <= 30, numpy_viterbi=arguments.numpy_viterbi) name = arguments.name or f"{'throughput' if arguments.unpaced else 'paced'}_workers_{arguments.workers}.json" write_result(name, result) print(json.dumps({key: result[key] for key in ("mode", "workers", "processing_seconds", "processing_per_rf_second", "full_wall_seconds", "rx_physical_packets", "received_images", "object_jpeg_integrity", "fresh_display_fps", "image_age_min_seconds", "image_age_median_seconds", "image_age_max_seconds", "iq_queue_peak", "candidate_queue_peak", "iq_overflow", "candidate_overflow", "errors")}), flush=True) if __name__ == "__main__": main()