#!/usr/bin/env python3 """Instruments for looking at motion without watching it. See ../motion.md, "The instruments". motion.py chart VIDEO [--regions NAME] [--out DIR] timing chart, events, holds, pops motion.py sheet VIDEO FROM TO [--step N] [--cols N] [--crop X,Y,W,H] [--width PX] [--diff] [--out FILE] motion.py auto VIDEO [--regions NAME] [--out DIR] a sheet for every event the chart finds motion.py track VIDEO FROM TO --color RRGGBB [--crop X,Y,W,H] [--tol N] where a colour is, frame by frame motion.py onion VIDEO FROM TO [--crop X,Y,W,H] [--out FILE] every frame of a move in one picture FROM and TO are seconds (12.4) or frames (f372). Crops and regions are in design units (1920 by 1080) whatever the video's size. Needs ffmpeg, numpy, Pillow. """ import argparse import json import os import subprocess import sys import numpy as np from PIL import Image, ImageDraw, ImageFont W, H = 1920, 1080 AW, AH = 960, 540 # analysis size: half, enough to see one hex digit change THRESH = 24 # a pixel moved if any channel changed by more than this (of 255) SOFT = 6 # ... and is fading if by more than this: a colour cooling, a caption arriving MIN_PX = 12 # a region is moving in a frame if this many of its pixels moved MIN_SOFT = 150 # ... and fading if this many did: under it is the codec HOLD = 0.4 # seconds of nothing that count as a hold def cols(first, n, y, h): return (64 + first * 152, 120 + y, n * 152 - 32, h) HEAD, CAPTION = (0, 0, W, 112), (0, 896, W, 184) # What to measure, by scene (the video's file name): the bands every scene has, and its panels. REGIONS = { "bands": {"head": HEAD, "stage": (0, 112, W, 784), "caption": CAPTION}, "fib": {"head": HEAD, "source": cols(0, 5, 0, 768), "tree": cols(5, 7, 0, 336), "stack": cols(5, 3, 360, 408), "registers": cols(8, 4, 360, 408), "caption": CAPTION}, "list": {"head": HEAD, "source": cols(0, 5, 0, 504), "heap map": cols(0, 5, 528, 240), "nodes": cols(5, 7, 0, 240), "words": cols(5, 7, 264, 504), "caption": CAPTION}, "sort": {"head": HEAD, "array": cols(0, 12, 0, 408), "source": cols(0, 7, 432, 336), "locals": cols(7, 5, 432, 336), "caption": CAPTION}, "syscalls": {"head": HEAD, "source": cols(0, 4, 0, 408), "registers": cols(4, 4, 0, 408), "memory": cols(8, 4, 0, 408), "boundary": (0, 528, W, 96), "kernel": (0, 624, W, 272), "caption": CAPTION}, "zoom": {"head": HEAD, "source": cols(0, 5, 0, 384), "registers": cols(0, 5, 408, 360), "stack": cols(5, 7, 0, 768), "caption": CAPTION}, } PALETTE = ["#9aa6d9", "#e8c46a", "#6fd6e6", "#7fdc9a", "#d59cf5", "#ff6a4a", "#eee6d2", "#17a0b0"] def probe(video): out = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=r_frame_rate,nb_frames", "-of", "json", video], capture_output=True, text=True, check=True) s = json.loads(out.stdout)["streams"][0] num, den = s["r_frame_rate"].split("/") return int(num) / int(den), int(s["nb_frames"]) def frames(video, w, h, first=0, count=None, step=1): """Yield (index, HxWx3 uint8) for frames first, first+step, ...""" vf = f"select='gte(n\\,{first})*not(mod(n-{first}\\,{step}))',scale={w}:{h}:flags=area" cmd = ["ffmpeg", "-v", "error", "-i", video, "-vf", vf, "-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", "rgb24"] if count is not None: cmd += ["-frames:v", str(count)] p = subprocess.Popen(cmd + ["-"], stdout=subprocess.PIPE) i = first while True: buf = p.stdout.read(w * h * 3) if len(buf) < w * h * 3: break yield i, np.frombuffer(buf, np.uint8).reshape(h, w, 3) i += step p.stdout.close() p.wait() def when(text, fps): """'12.4' seconds or 'f372' frames, as a frame index.""" return int(text[1:]) if text.startswith("f") else int(round(float(text) * fps)) def font(size): for path in ("/usr/share/fonts/go/Go-Mono.ttf", "/usr/share/fonts/TTF/DejaVuSansMono.ttf"): if os.path.exists(path): return ImageFont.truetype(path, size) return ImageFont.load_default() def regions_for(video, name): stem = os.path.splitext(os.path.basename(video))[0] return REGIONS.get(name or stem, REGIONS["bands"]) # ---- measure -------------------------------------------------------------- def measure(video, regions): """Per frame and region, against the frame before (at AW by AH): how many pixels moved, and how many changed at all (moved or fading). Returns names, moved, soft, region sizes.""" names = ["all"] + list(regions) boxes = [(0, 0, AW, AH)] + [tuple(int(round(v * AW / W)) for v in r) for r in regions.values()] rows, soft, prev = [], [], None for _, f in frames(video, AW, AH): f = f.astype(np.int16) if prev is None: rows.append([0] * len(names)) soft.append([0] * len(names)) else: d = np.abs(f - prev).max(axis=2) rows.append([int((d[y:y + h, x:x + w] > THRESH).sum()) for x, y, w, h in boxes]) soft.append([int((d[y:y + h, x:x + w] > SOFT).sum()) for x, y, w, h in boxes]) prev = f return names, np.array(rows), np.array(soft), [w * h for _, _, w, h in boxes] def busy(counts, soft): """Per frame and region: anything changing, by moving or by fading.""" return (counts >= MIN_PX) | (soft >= MIN_SOFT) def runs(active): """[(first, last)] of consecutive true frames.""" out, start = [], None for i, a in enumerate(active): if a and start is None: start = i if not a and start is not None: out.append((start, i - 1)) start = None if start is not None: out.append((start, len(active) - 1)) return out def events(names, counts, soft, fps, gap=2): """Stretches of change, joined across gaps of up to `gap` still frames.""" spans = [] on_ = busy(counts, soft) for a, b in runs(on_[:, 0]): if spans and a - spans[-1][1] - 1 <= gap: spans[-1] = (spans[-1][0], b) else: spans.append((a, b)) out = [] for a, b in spans: parts = {} for k, name in enumerate(names[1:], 1): on = np.nonzero(on_[a:b + 1, k])[0] if len(on): hard = np.nonzero(counts[a:b + 1, k] >= MIN_PX)[0] # (first, last, peak, frames that moved): a region that only fades moved in none parts[name] = (a + int(on[0]), a + int(on[-1]), int(counts[a:b + 1, k].max()), len(hard)) hard = int((counts[a:b + 1, 0] >= MIN_PX).sum()) out.append({"from": a, "to": b, "frames": b - a + 1, "seconds": round((b - a + 1) / fps, 3), "peak": int(counts[a:b + 1, 0].max()), "moved": hard, "regions": parts}) return out def report(video, names, counts, soft, sizes, fps): ev = events(names, counts, soft, fps) n = len(counts) lines = [f"{os.path.basename(video)}: {n} frames at {fps:g} fps, {n / fps:.2f} s", ""] moving = sum(e["frames"] for e in ev) lines.append(f"moving {moving / n:.0%} of the time, still {1 - moving / n:.0%}; {len(ev)} events") lines.append("") lines.append(" from to frames peak% what changed (first frame..last, from the event's first; ~ fades only)") last = -1 for e in ev: still = e["from"] - last - 1 if still / fps >= HOLD: lines.append(f" hold {still / fps:5.2f} s") parts = " ".join(f"{'~' if n_ == 0 else ''}{k} {a - e['from']}..{b - e['from']}" for k, (a, b, _, n_) in e["regions"].items()) # A pop: a lot changes in a frame or two with nothing either side of it. pop = " POP" if e["moved"] <= 2 and e["frames"] <= 3 and e["peak"] / sizes[0] > 0.002 else "" many = len([k for k in e["regions"] if k not in ("head", "caption")]) lines.append(f"{e['from'] / fps:6.2f} {e['to'] / fps:6.2f} {e['frames']:6d} {100 * e['peak'] / sizes[0]:5.2f} {parts}" + (f" [{many} panels]" if many > 2 else "") + pop) last = e["to"] if (n - last - 1) / fps >= HOLD: lines.append(f" hold {(n - last - 1) / fps:5.2f} s") lines += ["", "checks (each should come out empty)"] + lint(names, counts, soft, sizes, fps, ev) return ev, "\n".join(lines) WORDS = ("head", "caption") # regions that are not panels def lint(names, counts, soft, sizes, fps, ev): """What a viewer would trip on, found by rule. Each line: what, how often, the first few times.""" def some(times): return ", ".join(f"{t:.2f}" for t in times[:4]) + (" ..." if len(times) > 4 else "") out = [] on = busy(counts, soft) panels = [k for k, n_ in enumerate(names) if k > 0 and n_ not in WORDS] # POP: a lot changes in a frame or two, with stillness either side: nothing eased it. pops = [e["from"] / fps for e in ev if e["moved"] <= 2 and e["frames"] <= 3 and e["peak"] / sizes[0] > 0.002] if pops: out.append(f" POP {len(pops):3d} a change of over 0.2% of the frame in a frame or two: {some(pops)}") # CROWD: three or more panels begin to move within two frames of each other. crowd = [] for e in ev: first = [a - e["from"] for k, (a, b, p, n_) in e["regions"].items() if k not in WORDS and n_ > 0] if sum(1 for f in first if f <= 1) >= 3: crowd.append(e["from"] / fps) if crowd: out.append(f" CROWD {len(crowd):3d} three or more panels start together: {some(crowd)}") # METRONOME: six or more events in a row, evenly spaced. starts = [e["from"] for e in ev if e["peak"] / sizes[0] >= 0.005] # the beats, not their echoes i, runs_ = 0, [] while i < len(starts) - 1: j = i + 1 gap = starts[j] - starts[i] while j + 1 < len(starts) and abs((starts[j + 1] - starts[j]) - gap) <= 1: j += 1 if j - i >= 6: runs_.append((starts[i] / fps, starts[j] / fps, j - i + 1, gap / fps)) i = j for a, b, n_, gap in runs_: out.append(f" METRONOME {n_:3d} events {gap:.2f} s apart, from {a:.2f} to {b:.2f}") # RUSH: a caption arrives or changes, and the picture moves before it can be read. cap = names.index("caption") if "caption" in names else None rush = [] if cap is not None: for a, b in runs(on[:, cap]): if b - a < 3: continue later = np.nonzero(on[b + 1:, panels].any(axis=1))[0] if len(later) and later[0] / fps < 1.0: rush.append(b / fps) if rush: out.append(f" RUSH {len(rush):3d} the picture moves within a second of a caption arriving: {some(rush)}") # AIR: nothing at all changes for three seconds. air = [(a / fps, (b - a + 1) / fps) for a, b in runs(~on[:, 0]) if (b - a + 1) / fps >= 3.0] if air: out.append(f" AIR {len(air):3d} three seconds or more with nothing changing: " + ", ".join(f"{t:.2f} ({d:.1f} s)" for t, d in air[:4])) return out or [" none"] def chart(video, names, counts, soft, sizes, fps, ev, path, per_row=10.0): """Time left to right, a strip a region; bar height is how much of the region changed (log).""" px = max(2, int(round(180 / fps))) # pixels a frame: 6 at 30 fps, 3 at 60 row_frames = int(per_row * fps) n = len(counts) nrows = -(-n // row_frames) left, strip, pad = 130, 34, 34 rh = len(names) * strip + pad img = Image.new("RGB", (left + row_frames * px + 20, nrows * rh + 30), "#0a1030") d = ImageDraw.Draw(img) small, tiny = font(15), font(12) still = ~busy(counts, soft)[:, 0] for r in range(nrows): y0 = 20 + r * rh a = r * row_frames # holds, shaded and timed for s, e in runs(still): if (e - s + 1) / fps < HOLD: continue s2, e2 = max(s, a), min(e, a + row_frames - 1) if s2 > e2: continue d.rectangle([left + (s2 - a) * px, y0, left + (e2 - a + 1) * px - 1, y0 + len(names) * strip - 4], fill="#141c48") if s >= a: d.text((left + (s - a) * px + 4, y0 + 2), f"{(e - s + 1) / fps:.2f}", font=tiny, fill="#6a7ab8") for k, name in enumerate(names): y = y0 + k * strip d.text((8, y + 8), name, font=small, fill="#eee6d2" if k == 0 else PALETTE[(k - 1) % len(PALETTE)]) d.line([left, y + strip - 5, left + row_frames * px, y + strip - 5], fill="#3a4a8a") ink = "#eee6d2" if k == 0 else PALETTE[(k - 1) % len(PALETTE)] for i in range(a, min(a + row_frames, n)): x = left + (i - a) * px for c, ok, fill in ((soft[i, k], soft[i, k] >= MIN_SOFT, None), (counts[i, k], counts[i, k] >= MIN_PX, ink)): if not ok: continue hgt = int(np.clip((np.log10(c / sizes[k]) + 4.3) / 3.6, 0.08, 1) * (strip - 8)) # 0.005% .. 20% # A fade is an outline, a move is solid. d.rectangle([x, y + strip - 5 - hgt, x + px - 2, y + strip - 5], fill=fill, outline=ink) for i in range(a, min(a + row_frames, n) + 1): if i % int(fps) == 0: x = left + (i - a) * px d.line([x, y0 + len(names) * strip - 4, x, y0 + len(names) * strip + 4], fill="#9aa6d9") d.text((x + 3, y0 + len(names) * strip), f"{i / fps:g}s", font=tiny, fill="#9aa6d9") elif i % int(fps / 2) == 0: x = left + (i - a) * px d.line([x, y0 + len(names) * strip - 4, x, y0 + len(names) * strip], fill="#3a4a8a") img.save(path) def cmd_chart(a): fps, _ = probe(a.video) regions = regions_for(a.video, a.regions) names, counts, soft, sizes = measure(a.video, regions) ev, text = report(a.video, names, counts, soft, sizes, fps) os.makedirs(a.out, exist_ok=True) stem = os.path.splitext(os.path.basename(a.video))[0] chart(a.video, names, counts, soft, sizes, fps, ev, os.path.join(a.out, stem + ".png")) open(os.path.join(a.out, stem + ".txt"), "w").write(text + "\n") np.savetxt(os.path.join(a.out, stem + ".csv"), np.hstack([counts, soft]), fmt="%d", delimiter=",", header=",".join(names + [n + "~" for n in names]), comments="") print(text) return fps, ev # ---- look ----------------------------------------------------------------- def crop_of(text): return tuple(int(v) for v in text.split(",")) if text else (0, 0, W, H) def grab(video, first, last, step, crop, width): """Frames first..last as PIL images of `crop` (design units), `width` pixels wide.""" x, y, w, h = crop fps, total = probe(video) last = min(last, total - 1) out = [] for i, f in frames(video, W, H, first, (last - first) // step + 1, step): im = Image.fromarray(f).crop((x, y, x + w, y + h)) out.append((i, im.resize((width, max(1, round(width * h / w))), Image.LANCZOS))) return fps, out def tile(shots, fps, cols_, diff=False): w, h = shots[0][1].size rows = -(-len(shots) // cols_) lab = 20 sheet = Image.new("RGB", (cols_ * (w + 4) + 4, rows * (h + lab + 4) + 4), "#444444") d = ImageDraw.Draw(sheet) f = font(14) prev = None for k, (i, im) in enumerate(shots): x, y = 4 + (k % cols_) * (w + 4), 4 + (k // cols_) * (h + lab + 4) show = im if diff and prev is not None: # What changed since the tile before, white on the frame dimmed. a, b = np.asarray(im, np.int16), np.asarray(prev, np.int16) moved = np.abs(a - b).max(axis=2) > THRESH dim = (np.asarray(im, np.float32) * 0.35).astype(np.uint8) dim[moved] = (255, 255, 255) show = Image.fromarray(dim) sheet.paste(show, (x, y + lab)) d.text((x + 2, y + 1), f"f{i} {i / fps:.3f}s", font=f, fill="#ffffff") prev = im return sheet def cmd_sheet(a): fps, _ = probe(a.video) first, last = when(a.start, fps), when(a.end, fps) crop = crop_of(a.crop) n = (last - first) // a.step + 1 cols_ = a.cols or min(n, 6) width = a.width or min(crop[2], 1900 // cols_ - 4) _, shots = grab(a.video, first, last, a.step, crop, width) out = a.out or f"sheet-{os.path.splitext(os.path.basename(a.video))[0]}-f{first}-f{last}.png" os.makedirs(os.path.dirname(out) or ".", exist_ok=True) tile(shots, fps, cols_, a.diff).save(out) print(out, f"{len(shots)} frames") def cmd_auto(a): """A sheet for each event: every frame of it (every 2nd or 3rd if long), and a frame either side.""" fps, ev = cmd_chart(a) stem = os.path.splitext(os.path.basename(a.video))[0] made = short = 0 for e in ev: if e["frames"] <= 2: # a pop: three of them show what the rest look like short += 1 if short > 3: continue first, last = max(e["from"] - 1, 0), e["to"] + 1 step = max(1, -(-(last - first + 1) // 24)) _, shots = grab(a.video, first, last, step, (0, 0, W, H), 472) path = os.path.join(a.out, f"{stem}-{e['from'] / fps:06.2f}s.png") tile(shots, fps, 4).save(path) made += 1 print(f"{made} sheets in {a.out}") def cmd_track(a): """The centre and area of everything `color` in the crop, a frame a line: an object's spacing.""" fps, _ = probe(a.video) first, last = when(a.start, fps), when(a.end, fps) x, y, w, h = crop_of(a.crop) key = np.array([int(a.color[i:i + 2], 16) for i in (0, 2, 4)], np.int16) print(" frame time x y area dx dy box: left top width height") px = py = None for i, f in frames(a.video, W, H, first, last - first + 1): m = (np.abs(f[y:y + h, x:x + w].astype(np.int16) - key).max(axis=2) <= a.tol) area = int(m.sum()) if area < 4: print(f"{i:6d} {i / fps:6.3f} - - 0") px = py = None continue ys, xs = np.nonzero(m) cx, cy = x + xs.mean(), y + ys.mean() step = f"{cx - px:5.1f} {cy - py:5.1f}" if px is not None else " " box = f"{x + xs.min():5d} {y + ys.min():5d} {xs.max() - xs.min() + 1:5d} {ys.max() - ys.min() + 1:5d}" print(f"{i:6d} {i / fps:6.3f} {cx:6.1f} {cy:6.1f} {area:6d} {step} {box}") px, py = cx, cy def cmd_onion(a): """Every frame of a move in one picture: the first frame, and over it each later frame where it differs.""" fps, _ = probe(a.video) first, last = when(a.start, fps), when(a.end, fps) crop = crop_of(a.crop) _, shots = grab(a.video, first, last, a.step, crop, min(crop[2], 1900)) base = np.asarray(shots[0][1], np.int16) out = base.copy().astype(np.float32) * 0.5 for k, (_, im) in enumerate(shots[1:], 1): f = np.asarray(im, np.int16) moved = np.abs(f - base).max(axis=2) > THRESH out[moved] = np.maximum(out[moved], f[moved] * (0.45 + 0.55 * k / (len(shots) - 1))) path = a.out or f"onion-{os.path.splitext(os.path.basename(a.video))[0]}-f{first}-f{last}.png" os.makedirs(os.path.dirname(path) or ".", exist_ok=True) Image.fromarray(out.clip(0, 255).astype(np.uint8)).save(path) print(path, f"{len(shots)} frames") def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="cmd", required=True) for name, fn in (("chart", cmd_chart), ("auto", cmd_auto)): c = sub.add_parser(name) c.add_argument("video") c.add_argument("--regions") c.add_argument("--out", default="tmp") c.set_defaults(fn=fn) c = sub.add_parser("sheet") c.add_argument("video"), c.add_argument("start"), c.add_argument("end") c.add_argument("--step", type=int, default=1), c.add_argument("--cols", type=int), c.add_argument("--crop") c.add_argument("--width", type=int), c.add_argument("--diff", action="store_true"), c.add_argument("--out") c.set_defaults(fn=cmd_sheet) c = sub.add_parser("track") c.add_argument("video"), c.add_argument("start"), c.add_argument("end") c.add_argument("--color", required=True), c.add_argument("--crop"), c.add_argument("--tol", type=int, default=40) c.set_defaults(fn=cmd_track) c = sub.add_parser("onion") c.add_argument("video"), c.add_argument("start"), c.add_argument("end") c.add_argument("--step", type=int, default=1), c.add_argument("--crop"), c.add_argument("--out") c.set_defaults(fn=cmd_onion) a = p.parse_args() a.fn(a) if __name__ == "__main__": sys.exit(main())