Files
SDR-Rover/tools/video_compare.py

449 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""VIDEO-001: JPEG и libx264 на одном сохранённом видеоролике, без SDR."""
from __future__ import annotations
import argparse
from fractions import Fraction
import hashlib
from io import BytesIO
import json
import math
import os
from pathlib import Path
import statistics
import time
import av
import cv2
import numpy as np
from PIL import Image
import psutil
MATRIX = ((320, 240, (5, 10, 15)), (640, 360, (5, 10, 15)),
(640, 480, (5, 10)), (854, 480, (5, 10)), (1280, 720, (5,)))
DEFAULT_WINDOWS = (("white_car", 7.0, 6.0), ("black_car", 33.0, 6.0))
PROCESS = psutil.Process(os.getpid())
FRAGMENT_BYTES = 244
SAMPLE_RATE = 8_000_000
PACKET_SAMPLES = 92_256
GAP_SAMPLES = 40_000
BURST_SECONDS = (16 * PACKET_SAMPLES + 15 * GAP_SAMPLES + 2 * 800_000) / SAMPLE_RATE
OBSERVED_VIDEO_EQUIVALENT_BPS = 10 * FRAGMENT_BYTES * 8 / BURST_SECONDS
def summary(values: list[float]) -> dict:
return {"mean": statistics.mean(values), "median": statistics.median(values),
"max": max(values)}
def peak_second(frame_sizes: list[int], fps: int) -> int:
return 8 * max(sum(frame_sizes[start:start + fps])
for start in range(len(frame_sizes) - fps + 1))
def radio_screen(frame_sizes: list[int], fps: int) -> dict:
packets = [math.ceil(size / FRAGMENT_BYTES) for size in frame_sizes]
peak_packets = max(sum(packets[start:start + fps])
for start in range(len(packets) - fps + 1))
observed_packets_per_second = 10 / BURST_SECONDS
zero_loss_packets_per_second = SAMPLE_RATE / (PACKET_SAMPLES + GAP_SAMPLES)
return {
"peak_packet_slots_per_second": peak_packets,
"mean_packets_per_frame": statistics.mean(packets),
"max_packets_per_frame": max(packets),
"airtime_for_peak_second_s": peak_packets / zero_loss_packets_per_second,
"zero_loss_packet_slots_per_second_with_5ms_gaps": zero_loss_packets_per_second,
"observed_one_burst_accepted_packets_per_second": observed_packets_per_second,
"fits_current_radio_budget": (
"NO_ON_OBSERVED_BURST_SCREEN" if peak_packets > observed_packets_per_second
else "NOT_ESTABLISHED_NO_SUSTAINED_GOODPUT_TEST"),
}
def fit_frame(frame: np.ndarray, width: int, height: int) -> np.ndarray:
"""Уместить исходный кадр целиком, сохранив соотношение сторон."""
source_height, source_width = frame.shape[:2]
scale = min(width / source_width, height / source_height)
scaled_width = min(width, max(1, round(source_width * scale)))
scaled_height = min(height, max(1, round(source_height * scale)))
scaled = cv2.resize(frame, (scaled_width, scaled_height), interpolation=cv2.INTER_AREA)
canvas = np.zeros((height, width, 3), dtype=np.uint8)
left = (width - scaled_width) // 2
top = (height - scaled_height) // 2
canvas[top:top + scaled_height, left:left + scaled_width] = scaled
return cv2.cvtColor(canvas, cv2.COLOR_BGR2RGB)
def source_frames(path: Path, width: int, height: int, fps: int,
start_seconds: float, duration_seconds: float) -> list[np.ndarray]:
capture = cv2.VideoCapture(str(path))
if not capture.isOpened():
raise ValueError(f"Не открывается ролик: {path}")
source_fps = capture.get(cv2.CAP_PROP_FPS)
count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
requested = [round((start_seconds + index / fps) * source_fps)
for index in range(round(duration_seconds * fps))]
if source_fps <= 0 or not requested or len(set(requested)) != len(requested) or requested[-1] >= count:
capture.release()
raise ValueError("Окно вне ролика или частота ролика слишком мала")
selected = set(requested)
found = {}
try:
for index in range(requested[-1] + 1):
success, frame = capture.read()
if not success:
raise ValueError(f"Не прочитан исходный кадр {index}")
if index in selected:
found[index] = fit_frame(frame, width, height)
finally:
capture.release()
return [found[index] for index in requested]
def quality(reference: np.ndarray, decoded: np.ndarray) -> tuple[float, float]:
difference = reference.astype(np.float32) - decoded.astype(np.float32)
error = float(np.mean(difference * difference))
psnr = 10 * math.log10(255 * 255 / error) if error else 99.0
return psnr, float(np.mean(np.abs(difference)))
def save_example(base: Path, reference: np.ndarray, decoded: np.ndarray) -> None:
base.parent.mkdir(parents=True, exist_ok=True)
Image.fromarray(reference).save(base.with_name(base.name + "_source.png"))
Image.fromarray(decoded).save(base.with_name(base.name + "_decoded.png"))
def jpeg_case(frames: list[np.ndarray], fps: int, example: Path) -> dict:
sizes, encodes, decodes, latencies, psnrs, errors, hashes = ([] for _ in range(7))
payloads: list[bytes] = []
peak_rss = PROCESS.memory_info().rss
for index, reference in enumerate(frames):
started = time.perf_counter()
buffer = BytesIO()
Image.fromarray(reference).save(buffer, format="JPEG", quality=35,
subsampling=2, optimize=False, progressive=False)
encoded = buffer.getvalue()
payloads.append(encoded)
encoded_at = time.perf_counter()
with Image.open(BytesIO(encoded)) as image:
decoded = np.asarray(image.convert("RGB")).copy()
decoded_at = time.perf_counter()
sizes.append(len(encoded))
encodes.append(encoded_at - started)
decodes.append(decoded_at - encoded_at)
latencies.append(decoded_at - started)
psnr, error = quality(reference, decoded)
psnrs.append(psnr)
errors.append(error)
hashes.append(hashlib.sha256(decoded.tobytes()).hexdigest())
peak_rss = max(peak_rss, PROCESS.memory_info().rss)
if index == min(fps, len(frames) - 1):
save_example(example, reference, decoded)
with Image.open(BytesIO(payloads[1])) as image:
recovered_after_drop = np.asarray(image.convert("RGB"))
recovery_exact = hashlib.sha256(recovered_after_drop.tobytes()).hexdigest() == hashes[1]
return {"codec": "JPEG_INDEPENDENT",
"settings": {"quality": 35, "subsampling": "4:2:0",
"optimize": False, "progressive": False},
"frame_sizes_bytes": sizes, "reference_frame_size_bytes": max(sizes),
"encode_time_per_frame_s": summary(encodes),
"decode_time_per_frame_s": summary(decodes),
"file_pipeline_latency_s": summary(latencies),
"actual_encoded_fps": len(frames) / sum(encodes),
"unique_decoded_frames": len(set(hashes)), "decoded_frame_count": len(hashes),
"psnr_db": summary(psnrs), "mean_absolute_error": summary(errors),
"peak_process_rss_bytes": peak_rss,
"loss_recovery": {"dropped_frame": 0,
"first_exact_frame": 1 if recovery_exact else None},
"operator_quality": "PENDING_VISUAL_REVIEW"}
def recovery_after_drop(packets: list[bytes], dropped: int, fps: int,
clean_hashes: dict[int, str]) -> dict:
decoder = av.CodecContext.create("h264", "r")
decoder.open()
exact_after_loss = []
decode_errors = 0
for index, payload in enumerate(packets):
if index == dropped:
continue
packet = av.Packet(payload)
packet.pts = packet.dts = index
packet.time_base = Fraction(1, fps)
try:
decoded_frames = decoder.decode(packet)
except av.error.FFmpegError:
decode_errors += 1
continue
for frame in decoded_frames:
if frame.pts is None or frame.pts <= dropped:
continue
digest = hashlib.sha256(frame.to_ndarray(format="rgb24").tobytes()).hexdigest()
if clean_hashes.get(frame.pts) == digest:
exact_after_loss.append(int(frame.pts))
return {"dropped_frame": dropped,
"first_exact_frame": min(exact_after_loss) if exact_after_loss else None,
"decoder_errors": decode_errors}
def new_h264_encoder(width: int, height: int, fps: int,
maxrate_bps: int) -> tuple[av.CodecContext, str]:
encoder = av.CodecContext.create("libx264", "w")
encoder.width, encoder.height = width, height
encoder.pix_fmt = "yuv420p"
encoder.time_base = Fraction(1, fps)
encoder.framerate = Fraction(fps, 1)
encoder.thread_count = 1
parameters = (f"keyint={fps}:min-keyint={fps}:scenecut=0:open-gop=0:"
f"bframes=0:rc-lookahead=0:sync-lookahead=0:repeat-headers=1:"
f"vbv-maxrate={maxrate_bps // 1000}:vbv-bufsize={maxrate_bps // 1000}")
encoder.options = {"preset": "veryfast", "tune": "zerolatency", "crf": "28",
"x264-params": parameters}
encoder.open()
return encoder, parameters
def h264_case(frames: list[np.ndarray], fps: int, maxrate_bps: int, example: Path) -> dict:
height, width, _ = frames[0].shape
encoder, parameters = new_h264_encoder(width, height, fps, maxrate_bps)
decoder = av.CodecContext.create("h264", "r")
decoder.open()
sizes = [0] * len(frames)
encodes, decodes, latencies, psnrs, errors = ([] for _ in range(5))
packets: list[tuple[int, bytes, bool]] = []
hashes: dict[int, str] = {}
submitted: dict[int, float] = {}
peak_rss = PROCESS.memory_info().rss
for index, reference in enumerate(frames):
started = time.perf_counter()
submitted[index] = started
frame = av.VideoFrame.from_ndarray(reference, format="rgb24").reformat(format="yuv420p")
frame.pts = index
encoded_packets = encoder.encode(frame)
encodes.append(time.perf_counter() - started)
decode_cost = 0.0
for encoded in encoded_packets:
packet_index = int(encoded.pts)
payload = bytes(encoded)
packets.append((packet_index, payload, bool(encoded.is_keyframe)))
sizes[packet_index] += len(payload)
packet = av.Packet(payload)
packet.pts = packet.dts = packet_index
packet.time_base = Fraction(1, fps)
decoding_started = time.perf_counter()
decoded_frames = decoder.decode(packet)
decode_cost += time.perf_counter() - decoding_started
for item in decoded_frames:
decoded_index = int(item.pts)
decoded = item.to_ndarray(format="rgb24")
latencies.append(time.perf_counter() - submitted[decoded_index])
hashes[decoded_index] = hashlib.sha256(decoded.tobytes()).hexdigest()
psnr, error = quality(frames[decoded_index], decoded)
psnrs.append(psnr)
errors.append(error)
if decoded_index == min(fps, len(frames) - 1):
save_example(example, frames[decoded_index], decoded)
decodes.append(decode_cost)
peak_rss = max(peak_rss, PROCESS.memory_info().rss)
if encoder.encode(None) or len(packets) != len(frames) or len(hashes) != len(frames):
raise RuntimeError("H.264 задержал или потерял кадры")
if sorted(index for index, _, _ in packets) != list(range(len(frames))):
raise RuntimeError("H.264 нарушил нумерацию кадров")
if any(key != (index % fps == 0) for index, _, key in packets):
raise RuntimeError("Период опорных кадров отличается от заданного")
reference_sizes = [len(payload) for _, payload, key in packets if key]
payloads = [payload for _, payload, _ in packets]
return {"codec": "H264_LIBX264",
"settings": {"container": "Annex B packet stream", "preset": "veryfast",
"tune": "zerolatency", "crf": 28, "pixel_format": "yuv420p",
"maxrate_bps": maxrate_bps, "vbv_buffer_bits": maxrate_bps,
"gop_frames": fps, "b_frames": 0, "encoder_threads": 1,
"x264_parameters": parameters,
"pyav_version": av.__version__,
"ffmpeg_libraries": {name: ".".join(map(str, version))
for name, version in av.library_versions.items()}},
"frame_sizes_bytes": sizes,
"keyframe_sizes_bytes": reference_sizes,
"reference_frame_size_bytes": max(reference_sizes),
"encode_time_per_frame_s": summary(encodes),
"decode_time_per_frame_s": summary(decodes),
"file_pipeline_latency_s": summary(latencies),
"actual_encoded_fps": len(frames) / sum(encodes),
"unique_decoded_frames": len(set(hashes.values())),
"decoded_frame_count": len(hashes),
"psnr_db": summary(psnrs), "mean_absolute_error": summary(errors),
"peak_process_rss_bytes": peak_rss,
"loss_recovery": {
"lost_reference": recovery_after_drop(payloads, 0, fps, hashes),
"lost_delta": recovery_after_drop(payloads, 1, fps, hashes)},
"operator_quality": "PENDING_VISUAL_REVIEW"}
def end_to_end_file_case(source: Path, frames: list[np.ndarray], fps: int,
start_seconds: float, width: int, height: int,
codec: str, maxrate_bps: int,
expected_sizes: list[int]) -> dict:
"""Замерить чтение очередного кадра MP4 до готового декодированного RGB."""
capture = cv2.VideoCapture(str(source))
if not capture.isOpened():
raise ValueError(f"Не открывается ролик: {source}")
source_fps = capture.get(cv2.CAP_PROP_FPS)
requested = [round((start_seconds + index / fps) * source_fps)
for index in range(len(frames))]
if not capture.set(cv2.CAP_PROP_POS_FRAMES, requested[0]):
capture.release()
raise RuntimeError("Не удалось перейти к началу измеряемого участка")
next_source_index = requested[0]
encoder = decoder = None
if codec == "H264_LIBX264":
encoder, _ = new_h264_encoder(width, height, fps, maxrate_bps)
decoder = av.CodecContext.create("h264", "r")
decoder.open()
latencies = []
source_costs = []
peak_rss = PROCESS.memory_info().rss
try:
for index, target in enumerate(requested):
started = time.perf_counter()
while next_source_index <= target:
success, source_frame = capture.read()
if not success:
raise RuntimeError(f"Не прочитан кадр MP4 {next_source_index}")
next_source_index += 1
reference = fit_frame(source_frame, width, height)
source_costs.append(time.perf_counter() - started)
if not np.array_equal(reference, frames[index]):
raise RuntimeError("Кадр прямого файлового прохода отличается от исходного")
if codec == "JPEG_INDEPENDENT":
buffer = BytesIO()
Image.fromarray(reference).save(buffer, format="JPEG", quality=35,
subsampling=2, optimize=False,
progressive=False)
encoded = buffer.getvalue()
with Image.open(BytesIO(encoded)) as image:
decoded = np.asarray(image.convert("RGB"))
else:
frame = av.VideoFrame.from_ndarray(reference, format="rgb24").reformat(
format="yuv420p")
frame.pts = index
packets = encoder.encode(frame)
if len(packets) != 1:
raise RuntimeError("Файловый H.264 проход задержал пакет")
encoded = bytes(packets[0])
packet = av.Packet(encoded)
packet.pts = packet.dts = index
packet.time_base = Fraction(1, fps)
images = decoder.decode(packet)
if len(images) != 1:
raise RuntimeError("Файловый H.264 проход задержал кадр")
decoded = images[0].to_ndarray(format="rgb24")
if len(encoded) != expected_sizes[index] or decoded.shape != reference.shape:
raise RuntimeError("Файловый проход не совпал с основным кодированием")
latencies.append(time.perf_counter() - started)
peak_rss = max(peak_rss, PROCESS.memory_info().rss)
if encoder is not None and encoder.encode(None):
raise RuntimeError("H.264 сохранил отложенные пакеты после файлового прохода")
finally:
capture.release()
return {"end_to_end_file_pipeline_latency_s": summary(latencies),
"source_read_resize_time_per_frame_s": summary(source_costs),
"peak_process_rss_bytes": peak_rss}
def run(source: Path, output: Path, maxrate_bps: int, matrix=MATRIX,
windows=DEFAULT_WINDOWS) -> dict:
source_sha256 = hashlib.sha256(source.read_bytes()).hexdigest()
results = []
for width, height, rates in matrix:
for fps in rates:
for window_name, start_seconds, duration_seconds in windows:
frames = source_frames(source, width, height, fps, start_seconds, duration_seconds)
source_hashes = {hashlib.sha256(frame.tobytes()).hexdigest() for frame in frames}
if len(source_hashes) < len(frames) // 2:
raise RuntimeError("В исходном окне слишком много повторных кадров")
temporal_change = statistics.mean(float(np.mean(np.abs(
later.astype(np.int16) - earlier.astype(np.int16))))
for earlier, later in zip(frames, frames[1:]))
for name, operation in (("jpeg", jpeg_case), ("h264", h264_case)):
suffix = f"_{maxrate_bps}" if name == "h264" else ""
example = (output.parent / "examples" /
f"{window_name}_{width}x{height}_{fps}_{name}{suffix}")
case = (operation(frames, fps, example) if name == "jpeg"
else operation(frames, fps, maxrate_bps, example))
sizes = case["frame_sizes_bytes"]
direct = end_to_end_file_case(
source, frames, fps, start_seconds, width, height,
case["codec"], maxrate_bps, sizes)
case["end_to_end_file_pipeline_latency_s"] = direct[
"end_to_end_file_pipeline_latency_s"]
case["source_read_resize_time_per_frame_s"] = direct[
"source_read_resize_time_per_frame_s"]
case["peak_process_rss_bytes"] = max(
case["peak_process_rss_bytes"], direct["peak_process_rss_bytes"])
case.update({"resolution": [width, height], "fps": fps,
"window": window_name,
"source_unique_frames": len(source_hashes),
"source_temporal_change_mean_absolute": temporal_change,
"unique_decoded_fps": case["unique_decoded_frames"] / duration_seconds,
"identical_decoded_frame_count": (
case["decoded_frame_count"] - case["unique_decoded_frames"]),
"average_bitrate_bps": 8 * sum(sizes) / duration_seconds,
"peak_one_second_bitrate_bps": peak_second(sizes, fps),
"average_frame_size_bytes": statistics.mean(sizes),
"max_frame_size_bytes": max(sizes),
"radio_screen": radio_screen(sizes, fps)})
results.append(case)
print(f"{width}x{height} {fps} {window_name} {case['codec']}: "
f"{case['average_bitrate_bps']:.0f} бит/с", flush=True)
document = {
"status": "MEASURED_SOFTWARE_ONLY", "rf_tx": 0,
"source": str(source), "source_sha256": source_sha256,
"windows": [{"name": name, "start_seconds": start, "duration_seconds": duration}
for name, start, duration in windows],
"radio_budget": {
"status": "MEASURED_ONE_BURST_VIDEO_EQUIVALENT",
"lab066_accepted_packets": 10, "lab066_transmitted_packets": 16,
"burst_seconds_including_guards": BURST_SECONDS,
"video_equivalent_goodput_bps": OBSERVED_VIDEO_EQUIVALENT_BPS,
"zero_loss_goodput_bps_with_5ms_gaps":
FRAGMENT_BYTES * 8 * SAMPLE_RATE / (PACKET_SAMPLES + GAP_SAMPLES),
"sustained_video_goodput": "NOT_ESTABLISHED",
"h264_over_current_fragment_transport": "NOT_IMPLEMENTED"},
"memory_measurement": "RSS всего процесса, пик при обработке случая; случаи в одном процессе",
"resize_method": "Пропорциональное уменьшение с чёрными полями, без растяжения и обрезки",
"latency_measurement": "file_pipeline_latency_s: от подачи готового кадра кодировщику до RGB; end_to_end_file_pipeline_latency_s: от чтения очередного кадра MP4 с пропуском промежуточных до RGB; оба без эфира и экрана",
"operator_quality_scope": "PSNR не заменяет проверку препятствий и читаемости оператором",
"cases": results,
}
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(document, ensure_ascii=False, indent=2, allow_nan=False) + "\n",
encoding="utf-8")
return document
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--source", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--h264-maxrate-bps", type=int, default=160_000)
parser.add_argument("--window", nargs=3, action="append", metavar=("NAME", "START_S", "DURATION_S"),
help="Окно ролика; параметр можно повторить")
arguments = parser.parse_args()
if arguments.h264_maxrate_bps < 10_000 or arguments.h264_maxrate_bps % 1000:
parser.error("maxrate должен быть не меньше 10 кбит/с и кратным 1000")
windows = DEFAULT_WINDOWS
if arguments.window:
try:
windows = tuple((name, float(start), float(duration))
for name, start, duration in arguments.window)
except ValueError:
parser.error("START_S и DURATION_S должны быть числами")
if any(start < 0 or duration <= 0 for _, start, duration in windows):
parser.error("окна должны иметь неотрицательное начало и положительную длительность")
run(arguments.source, arguments.output, arguments.h264_maxrate_bps, windows=windows)
if __name__ == "__main__":
main()