diff options
Diffstat (limited to 'design/motion/motion.py')
| -rw-r--r-- | design/motion/motion.py | 450 |
1 files changed, 450 insertions, 0 deletions
diff --git a/design/motion/motion.py b/design/motion/motion.py new file mode 100644 index 0000000..b0647d7 --- /dev/null +++ b/design/motion/motion.py @@ -0,0 +1,450 @@ +#!/usr/bin/env python3 +"""Instruments for looking at motion without watching it. See ../motion.md, "The instruments". + + motion.py chart VIDEO [--regions NAME] [--out DIR] timing chart, events, holds, pops + motion.py sheet VIDEO FROM TO [--step N] [--cols N] [--crop X,Y,W,H] [--width PX] [--diff] [--out FILE] + motion.py auto VIDEO [--regions NAME] [--out DIR] a sheet for every event the chart finds + motion.py track VIDEO FROM TO --color RRGGBB [--crop X,Y,W,H] [--tol N] where a colour is, frame by frame + motion.py onion VIDEO FROM TO [--crop X,Y,W,H] [--out FILE] every frame of a move in one picture + +FROM and TO are seconds (12.4) or frames (f372). Crops and regions are in +design units (1920 by 1080) whatever the video's size. Needs ffmpeg, numpy, Pillow. +""" +import argparse +import json +import os +import subprocess +import sys + +import numpy as np +from PIL import Image, ImageDraw, ImageFont + +W, H = 1920, 1080 +AW, AH = 960, 540 # analysis size: half, enough to see one hex digit change +THRESH = 24 # a pixel moved if any channel changed by more than this (of 255) +SOFT = 6 # ... and is fading if by more than this: a colour cooling, a caption arriving +MIN_PX = 12 # a region is moving in a frame if this many of its pixels moved +MIN_SOFT = 150 # ... and fading if this many did: under it is the codec +HOLD = 0.4 # seconds of nothing that count as a hold + + +def cols(first, n, y, h): + return (64 + first * 152, 120 + y, n * 152 - 32, h) + + +HEAD, CAPTION = (0, 0, W, 112), (0, 896, W, 184) +# What to measure, by scene (the video's file name): the bands every scene has, and its panels. +REGIONS = { + "bands": {"head": HEAD, "stage": (0, 112, W, 784), "caption": CAPTION}, + "fib": {"head": HEAD, "source": cols(0, 5, 0, 768), "tree": cols(5, 7, 0, 336), "stack": cols(5, 3, 360, 408), + "registers": cols(8, 4, 360, 408), "caption": CAPTION}, + "list": {"head": HEAD, "source": cols(0, 5, 0, 504), "heap map": cols(0, 5, 528, 240), "nodes": cols(5, 7, 0, 240), + "words": cols(5, 7, 264, 504), "caption": CAPTION}, + "sort": {"head": HEAD, "array": cols(0, 12, 0, 408), "source": cols(0, 7, 432, 336), "locals": cols(7, 5, 432, 336), + "caption": CAPTION}, + "syscalls": {"head": HEAD, "source": cols(0, 4, 0, 408), "registers": cols(4, 4, 0, 408), "memory": cols(8, 4, 0, 408), + "boundary": (0, 528, W, 96), "kernel": (0, 624, W, 272), "caption": CAPTION}, + "zoom": {"head": HEAD, "source": cols(0, 5, 0, 384), "registers": cols(0, 5, 408, 360), "stack": cols(5, 7, 0, 768), + "caption": CAPTION}, +} +PALETTE = ["#9aa6d9", "#e8c46a", "#6fd6e6", "#7fdc9a", "#d59cf5", "#ff6a4a", "#eee6d2", "#17a0b0"] + + +def probe(video): + out = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", + "stream=r_frame_rate,nb_frames", "-of", "json", video], capture_output=True, text=True, check=True) + s = json.loads(out.stdout)["streams"][0] + num, den = s["r_frame_rate"].split("/") + return int(num) / int(den), int(s["nb_frames"]) + + +def frames(video, w, h, first=0, count=None, step=1): + """Yield (index, HxWx3 uint8) for frames first, first+step, ...""" + vf = f"select='gte(n\\,{first})*not(mod(n-{first}\\,{step}))',scale={w}:{h}:flags=area" + cmd = ["ffmpeg", "-v", "error", "-i", video, "-vf", vf, "-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", "rgb24"] + if count is not None: + cmd += ["-frames:v", str(count)] + p = subprocess.Popen(cmd + ["-"], stdout=subprocess.PIPE) + i = first + while True: + buf = p.stdout.read(w * h * 3) + if len(buf) < w * h * 3: + break + yield i, np.frombuffer(buf, np.uint8).reshape(h, w, 3) + i += step + p.stdout.close() + p.wait() + + +def when(text, fps): + """'12.4' seconds or 'f372' frames, as a frame index.""" + return int(text[1:]) if text.startswith("f") else int(round(float(text) * fps)) + + +def font(size): + for path in ("/usr/share/fonts/go/Go-Mono.ttf", "/usr/share/fonts/TTF/DejaVuSansMono.ttf"): + if os.path.exists(path): + return ImageFont.truetype(path, size) + return ImageFont.load_default() + + +def regions_for(video, name): + stem = os.path.splitext(os.path.basename(video))[0] + return REGIONS.get(name or stem, REGIONS["bands"]) + + +# ---- measure -------------------------------------------------------------- +def measure(video, regions): + """Per frame and region, against the frame before (at AW by AH): how many pixels moved, and + how many changed at all (moved or fading). Returns names, moved, soft, region sizes.""" + names = ["all"] + list(regions) + boxes = [(0, 0, AW, AH)] + [tuple(int(round(v * AW / W)) for v in r) for r in regions.values()] + rows, soft, prev = [], [], None + for _, f in frames(video, AW, AH): + f = f.astype(np.int16) + if prev is None: + rows.append([0] * len(names)) + soft.append([0] * len(names)) + else: + d = np.abs(f - prev).max(axis=2) + rows.append([int((d[y:y + h, x:x + w] > THRESH).sum()) for x, y, w, h in boxes]) + soft.append([int((d[y:y + h, x:x + w] > SOFT).sum()) for x, y, w, h in boxes]) + prev = f + return names, np.array(rows), np.array(soft), [w * h for _, _, w, h in boxes] + + +def busy(counts, soft): + """Per frame and region: anything changing, by moving or by fading.""" + return (counts >= MIN_PX) | (soft >= MIN_SOFT) + + +def runs(active): + """[(first, last)] of consecutive true frames.""" + out, start = [], None + for i, a in enumerate(active): + if a and start is None: + start = i + if not a and start is not None: + out.append((start, i - 1)) + start = None + if start is not None: + out.append((start, len(active) - 1)) + return out + + +def events(names, counts, soft, fps, gap=2): + """Stretches of change, joined across gaps of up to `gap` still frames.""" + spans = [] + on_ = busy(counts, soft) + for a, b in runs(on_[:, 0]): + if spans and a - spans[-1][1] - 1 <= gap: + spans[-1] = (spans[-1][0], b) + else: + spans.append((a, b)) + out = [] + for a, b in spans: + parts = {} + for k, name in enumerate(names[1:], 1): + on = np.nonzero(on_[a:b + 1, k])[0] + if len(on): + hard = np.nonzero(counts[a:b + 1, k] >= MIN_PX)[0] + # (first, last, peak, frames that moved): a region that only fades moved in none + parts[name] = (a + int(on[0]), a + int(on[-1]), int(counts[a:b + 1, k].max()), len(hard)) + hard = int((counts[a:b + 1, 0] >= MIN_PX).sum()) + out.append({"from": a, "to": b, "frames": b - a + 1, "seconds": round((b - a + 1) / fps, 3), + "peak": int(counts[a:b + 1, 0].max()), "moved": hard, "regions": parts}) + return out + + +def report(video, names, counts, soft, sizes, fps): + ev = events(names, counts, soft, fps) + n = len(counts) + lines = [f"{os.path.basename(video)}: {n} frames at {fps:g} fps, {n / fps:.2f} s", ""] + moving = sum(e["frames"] for e in ev) + lines.append(f"moving {moving / n:.0%} of the time, still {1 - moving / n:.0%}; {len(ev)} events") + lines.append("") + lines.append(" from to frames peak% what changed (first frame..last, from the event's first; ~ fades only)") + last = -1 + for e in ev: + still = e["from"] - last - 1 + if still / fps >= HOLD: + lines.append(f" hold {still / fps:5.2f} s") + parts = " ".join(f"{'~' if n_ == 0 else ''}{k} {a - e['from']}..{b - e['from']}" for k, (a, b, _, n_) in e["regions"].items()) + # A pop: a lot changes in a frame or two with nothing either side of it. + pop = " POP" if e["moved"] <= 2 and e["frames"] <= 3 and e["peak"] / sizes[0] > 0.002 else "" + many = len([k for k in e["regions"] if k not in ("head", "caption")]) + lines.append(f"{e['from'] / fps:6.2f} {e['to'] / fps:6.2f} {e['frames']:6d} {100 * e['peak'] / sizes[0]:5.2f} {parts}" + + (f" [{many} panels]" if many > 2 else "") + pop) + last = e["to"] + if (n - last - 1) / fps >= HOLD: + lines.append(f" hold {(n - last - 1) / fps:5.2f} s") + lines += ["", "checks (each should come out empty)"] + lint(names, counts, soft, sizes, fps, ev) + return ev, "\n".join(lines) + + +WORDS = ("head", "caption") # regions that are not panels + + +def lint(names, counts, soft, sizes, fps, ev): + """What a viewer would trip on, found by rule. Each line: what, how often, the first few times.""" + def some(times): + return ", ".join(f"{t:.2f}" for t in times[:4]) + (" ..." if len(times) > 4 else "") + out = [] + on = busy(counts, soft) + panels = [k for k, n_ in enumerate(names) if k > 0 and n_ not in WORDS] + + # POP: a lot changes in a frame or two, with stillness either side: nothing eased it. + pops = [e["from"] / fps for e in ev if e["moved"] <= 2 and e["frames"] <= 3 and e["peak"] / sizes[0] > 0.002] + if pops: + out.append(f" POP {len(pops):3d} a change of over 0.2% of the frame in a frame or two: {some(pops)}") + + # CROWD: three or more panels begin to move within two frames of each other. + crowd = [] + for e in ev: + first = [a - e["from"] for k, (a, b, p, n_) in e["regions"].items() if k not in WORDS and n_ > 0] + if sum(1 for f in first if f <= 1) >= 3: + crowd.append(e["from"] / fps) + if crowd: + out.append(f" CROWD {len(crowd):3d} three or more panels start together: {some(crowd)}") + + # METRONOME: six or more events in a row, evenly spaced. + starts = [e["from"] for e in ev if e["peak"] / sizes[0] >= 0.005] # the beats, not their echoes + i, runs_ = 0, [] + while i < len(starts) - 1: + j = i + 1 + gap = starts[j] - starts[i] + while j + 1 < len(starts) and abs((starts[j + 1] - starts[j]) - gap) <= 1: + j += 1 + if j - i >= 6: + runs_.append((starts[i] / fps, starts[j] / fps, j - i + 1, gap / fps)) + i = j + for a, b, n_, gap in runs_: + out.append(f" METRONOME {n_:3d} events {gap:.2f} s apart, from {a:.2f} to {b:.2f}") + + # RUSH: a caption arrives or changes, and the picture moves before it can be read. + cap = names.index("caption") if "caption" in names else None + rush = [] + if cap is not None: + for a, b in runs(on[:, cap]): + if b - a < 3: + continue + later = np.nonzero(on[b + 1:, panels].any(axis=1))[0] + if len(later) and later[0] / fps < 1.0: + rush.append(b / fps) + if rush: + out.append(f" RUSH {len(rush):3d} the picture moves within a second of a caption arriving: {some(rush)}") + + # AIR: nothing at all changes for three seconds. + air = [(a / fps, (b - a + 1) / fps) for a, b in runs(~on[:, 0]) if (b - a + 1) / fps >= 3.0] + if air: + out.append(f" AIR {len(air):3d} three seconds or more with nothing changing: " + ", ".join(f"{t:.2f} ({d:.1f} s)" for t, d in air[:4])) + return out or [" none"] + + +def chart(video, names, counts, soft, sizes, fps, ev, path, per_row=10.0): + """Time left to right, a strip a region; bar height is how much of the region changed (log).""" + px = max(2, int(round(180 / fps))) # pixels a frame: 6 at 30 fps, 3 at 60 + row_frames = int(per_row * fps) + n = len(counts) + nrows = -(-n // row_frames) + left, strip, pad = 130, 34, 34 + rh = len(names) * strip + pad + img = Image.new("RGB", (left + row_frames * px + 20, nrows * rh + 30), "#0a1030") + d = ImageDraw.Draw(img) + small, tiny = font(15), font(12) + still = ~busy(counts, soft)[:, 0] + for r in range(nrows): + y0 = 20 + r * rh + a = r * row_frames + # holds, shaded and timed + for s, e in runs(still): + if (e - s + 1) / fps < HOLD: + continue + s2, e2 = max(s, a), min(e, a + row_frames - 1) + if s2 > e2: + continue + d.rectangle([left + (s2 - a) * px, y0, left + (e2 - a + 1) * px - 1, y0 + len(names) * strip - 4], fill="#141c48") + if s >= a: + d.text((left + (s - a) * px + 4, y0 + 2), f"{(e - s + 1) / fps:.2f}", font=tiny, fill="#6a7ab8") + for k, name in enumerate(names): + y = y0 + k * strip + d.text((8, y + 8), name, font=small, fill="#eee6d2" if k == 0 else PALETTE[(k - 1) % len(PALETTE)]) + d.line([left, y + strip - 5, left + row_frames * px, y + strip - 5], fill="#3a4a8a") + ink = "#eee6d2" if k == 0 else PALETTE[(k - 1) % len(PALETTE)] + for i in range(a, min(a + row_frames, n)): + x = left + (i - a) * px + for c, ok, fill in ((soft[i, k], soft[i, k] >= MIN_SOFT, None), (counts[i, k], counts[i, k] >= MIN_PX, ink)): + if not ok: + continue + hgt = int(np.clip((np.log10(c / sizes[k]) + 4.3) / 3.6, 0.08, 1) * (strip - 8)) # 0.005% .. 20% + # A fade is an outline, a move is solid. + d.rectangle([x, y + strip - 5 - hgt, x + px - 2, y + strip - 5], fill=fill, outline=ink) + for i in range(a, min(a + row_frames, n) + 1): + if i % int(fps) == 0: + x = left + (i - a) * px + d.line([x, y0 + len(names) * strip - 4, x, y0 + len(names) * strip + 4], fill="#9aa6d9") + d.text((x + 3, y0 + len(names) * strip), f"{i / fps:g}s", font=tiny, fill="#9aa6d9") + elif i % int(fps / 2) == 0: + x = left + (i - a) * px + d.line([x, y0 + len(names) * strip - 4, x, y0 + len(names) * strip], fill="#3a4a8a") + img.save(path) + + +def cmd_chart(a): + fps, _ = probe(a.video) + regions = regions_for(a.video, a.regions) + names, counts, soft, sizes = measure(a.video, regions) + ev, text = report(a.video, names, counts, soft, sizes, fps) + os.makedirs(a.out, exist_ok=True) + stem = os.path.splitext(os.path.basename(a.video))[0] + chart(a.video, names, counts, soft, sizes, fps, ev, os.path.join(a.out, stem + ".png")) + open(os.path.join(a.out, stem + ".txt"), "w").write(text + "\n") + np.savetxt(os.path.join(a.out, stem + ".csv"), np.hstack([counts, soft]), fmt="%d", delimiter=",", + header=",".join(names + [n + "~" for n in names]), comments="") + print(text) + return fps, ev + + +# ---- look ----------------------------------------------------------------- +def crop_of(text): + return tuple(int(v) for v in text.split(",")) if text else (0, 0, W, H) + + +def grab(video, first, last, step, crop, width): + """Frames first..last as PIL images of `crop` (design units), `width` pixels wide.""" + x, y, w, h = crop + fps, total = probe(video) + last = min(last, total - 1) + out = [] + for i, f in frames(video, W, H, first, (last - first) // step + 1, step): + im = Image.fromarray(f).crop((x, y, x + w, y + h)) + out.append((i, im.resize((width, max(1, round(width * h / w))), Image.LANCZOS))) + return fps, out + + +def tile(shots, fps, cols_, diff=False): + w, h = shots[0][1].size + rows = -(-len(shots) // cols_) + lab = 20 + sheet = Image.new("RGB", (cols_ * (w + 4) + 4, rows * (h + lab + 4) + 4), "#444444") + d = ImageDraw.Draw(sheet) + f = font(14) + prev = None + for k, (i, im) in enumerate(shots): + x, y = 4 + (k % cols_) * (w + 4), 4 + (k // cols_) * (h + lab + 4) + show = im + if diff and prev is not None: + # What changed since the tile before, white on the frame dimmed. + a, b = np.asarray(im, np.int16), np.asarray(prev, np.int16) + moved = np.abs(a - b).max(axis=2) > THRESH + dim = (np.asarray(im, np.float32) * 0.35).astype(np.uint8) + dim[moved] = (255, 255, 255) + show = Image.fromarray(dim) + sheet.paste(show, (x, y + lab)) + d.text((x + 2, y + 1), f"f{i} {i / fps:.3f}s", font=f, fill="#ffffff") + prev = im + return sheet + + +def cmd_sheet(a): + fps, _ = probe(a.video) + first, last = when(a.start, fps), when(a.end, fps) + crop = crop_of(a.crop) + n = (last - first) // a.step + 1 + cols_ = a.cols or min(n, 6) + width = a.width or min(crop[2], 1900 // cols_ - 4) + _, shots = grab(a.video, first, last, a.step, crop, width) + out = a.out or f"sheet-{os.path.splitext(os.path.basename(a.video))[0]}-f{first}-f{last}.png" + os.makedirs(os.path.dirname(out) or ".", exist_ok=True) + tile(shots, fps, cols_, a.diff).save(out) + print(out, f"{len(shots)} frames") + + +def cmd_auto(a): + """A sheet for each event: every frame of it (every 2nd or 3rd if long), and a frame either side.""" + fps, ev = cmd_chart(a) + stem = os.path.splitext(os.path.basename(a.video))[0] + made = short = 0 + for e in ev: + if e["frames"] <= 2: # a pop: three of them show what the rest look like + short += 1 + if short > 3: + continue + first, last = max(e["from"] - 1, 0), e["to"] + 1 + step = max(1, -(-(last - first + 1) // 24)) + _, shots = grab(a.video, first, last, step, (0, 0, W, H), 472) + path = os.path.join(a.out, f"{stem}-{e['from'] / fps:06.2f}s.png") + tile(shots, fps, 4).save(path) + made += 1 + print(f"{made} sheets in {a.out}") + + +def cmd_track(a): + """The centre and area of everything `color` in the crop, a frame a line: an object's spacing.""" + fps, _ = probe(a.video) + first, last = when(a.start, fps), when(a.end, fps) + x, y, w, h = crop_of(a.crop) + key = np.array([int(a.color[i:i + 2], 16) for i in (0, 2, 4)], np.int16) + print(" frame time x y area dx dy box: left top width height") + px = py = None + for i, f in frames(a.video, W, H, first, last - first + 1): + m = (np.abs(f[y:y + h, x:x + w].astype(np.int16) - key).max(axis=2) <= a.tol) + area = int(m.sum()) + if area < 4: + print(f"{i:6d} {i / fps:6.3f} - - 0") + px = py = None + continue + ys, xs = np.nonzero(m) + cx, cy = x + xs.mean(), y + ys.mean() + step = f"{cx - px:5.1f} {cy - py:5.1f}" if px is not None else " " + box = f"{x + xs.min():5d} {y + ys.min():5d} {xs.max() - xs.min() + 1:5d} {ys.max() - ys.min() + 1:5d}" + print(f"{i:6d} {i / fps:6.3f} {cx:6.1f} {cy:6.1f} {area:6d} {step} {box}") + px, py = cx, cy + + +def cmd_onion(a): + """Every frame of a move in one picture: the first frame, and over it each later frame where it differs.""" + fps, _ = probe(a.video) + first, last = when(a.start, fps), when(a.end, fps) + crop = crop_of(a.crop) + _, shots = grab(a.video, first, last, a.step, crop, min(crop[2], 1900)) + base = np.asarray(shots[0][1], np.int16) + out = base.copy().astype(np.float32) * 0.5 + for k, (_, im) in enumerate(shots[1:], 1): + f = np.asarray(im, np.int16) + moved = np.abs(f - base).max(axis=2) > THRESH + out[moved] = np.maximum(out[moved], f[moved] * (0.45 + 0.55 * k / (len(shots) - 1))) + path = a.out or f"onion-{os.path.splitext(os.path.basename(a.video))[0]}-f{first}-f{last}.png" + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + Image.fromarray(out.clip(0, 255).astype(np.uint8)).save(path) + print(path, f"{len(shots)} frames") + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = p.add_subparsers(dest="cmd", required=True) + for name, fn in (("chart", cmd_chart), ("auto", cmd_auto)): + c = sub.add_parser(name) + c.add_argument("video") + c.add_argument("--regions") + c.add_argument("--out", default="tmp") + c.set_defaults(fn=fn) + c = sub.add_parser("sheet") + c.add_argument("video"), c.add_argument("start"), c.add_argument("end") + c.add_argument("--step", type=int, default=1), c.add_argument("--cols", type=int), c.add_argument("--crop") + c.add_argument("--width", type=int), c.add_argument("--diff", action="store_true"), c.add_argument("--out") + c.set_defaults(fn=cmd_sheet) + c = sub.add_parser("track") + c.add_argument("video"), c.add_argument("start"), c.add_argument("end") + c.add_argument("--color", required=True), c.add_argument("--crop"), c.add_argument("--tol", type=int, default=40) + c.set_defaults(fn=cmd_track) + c = sub.add_parser("onion") + c.add_argument("video"), c.add_argument("start"), c.add_argument("end") + c.add_argument("--step", type=int, default=1), c.add_argument("--crop"), c.add_argument("--out") + c.set_defaults(fn=cmd_onion) + a = p.parse_args() + a.fn(a) + + +if __name__ == "__main__": + sys.exit(main()) |
