summaryrefslogtreecommitdiff
path: root/design/motion/motion.py
diff options
context:
space:
mode:
Diffstat (limited to 'design/motion/motion.py')
-rw-r--r--design/motion/motion.py450
1 files changed, 450 insertions, 0 deletions
diff --git a/design/motion/motion.py b/design/motion/motion.py
new file mode 100644
index 0000000..b0647d7
--- /dev/null
+++ b/design/motion/motion.py
@@ -0,0 +1,450 @@
+#!/usr/bin/env python3
+"""Instruments for looking at motion without watching it. See ../motion.md, "The instruments".
+
+ motion.py chart VIDEO [--regions NAME] [--out DIR] timing chart, events, holds, pops
+ motion.py sheet VIDEO FROM TO [--step N] [--cols N] [--crop X,Y,W,H] [--width PX] [--diff] [--out FILE]
+ motion.py auto VIDEO [--regions NAME] [--out DIR] a sheet for every event the chart finds
+ motion.py track VIDEO FROM TO --color RRGGBB [--crop X,Y,W,H] [--tol N] where a colour is, frame by frame
+ motion.py onion VIDEO FROM TO [--crop X,Y,W,H] [--out FILE] every frame of a move in one picture
+
+FROM and TO are seconds (12.4) or frames (f372). Crops and regions are in
+design units (1920 by 1080) whatever the video's size. Needs ffmpeg, numpy, Pillow.
+"""
+import argparse
+import json
+import os
+import subprocess
+import sys
+
+import numpy as np
+from PIL import Image, ImageDraw, ImageFont
+
+W, H = 1920, 1080
+AW, AH = 960, 540 # analysis size: half, enough to see one hex digit change
+THRESH = 24 # a pixel moved if any channel changed by more than this (of 255)
+SOFT = 6 # ... and is fading if by more than this: a colour cooling, a caption arriving
+MIN_PX = 12 # a region is moving in a frame if this many of its pixels moved
+MIN_SOFT = 150 # ... and fading if this many did: under it is the codec
+HOLD = 0.4 # seconds of nothing that count as a hold
+
+
+def cols(first, n, y, h):
+ return (64 + first * 152, 120 + y, n * 152 - 32, h)
+
+
+HEAD, CAPTION = (0, 0, W, 112), (0, 896, W, 184)
+# What to measure, by scene (the video's file name): the bands every scene has, and its panels.
+REGIONS = {
+ "bands": {"head": HEAD, "stage": (0, 112, W, 784), "caption": CAPTION},
+ "fib": {"head": HEAD, "source": cols(0, 5, 0, 768), "tree": cols(5, 7, 0, 336), "stack": cols(5, 3, 360, 408),
+ "registers": cols(8, 4, 360, 408), "caption": CAPTION},
+ "list": {"head": HEAD, "source": cols(0, 5, 0, 504), "heap map": cols(0, 5, 528, 240), "nodes": cols(5, 7, 0, 240),
+ "words": cols(5, 7, 264, 504), "caption": CAPTION},
+ "sort": {"head": HEAD, "array": cols(0, 12, 0, 408), "source": cols(0, 7, 432, 336), "locals": cols(7, 5, 432, 336),
+ "caption": CAPTION},
+ "syscalls": {"head": HEAD, "source": cols(0, 4, 0, 408), "registers": cols(4, 4, 0, 408), "memory": cols(8, 4, 0, 408),
+ "boundary": (0, 528, W, 96), "kernel": (0, 624, W, 272), "caption": CAPTION},
+ "zoom": {"head": HEAD, "source": cols(0, 5, 0, 384), "registers": cols(0, 5, 408, 360), "stack": cols(5, 7, 0, 768),
+ "caption": CAPTION},
+}
+PALETTE = ["#9aa6d9", "#e8c46a", "#6fd6e6", "#7fdc9a", "#d59cf5", "#ff6a4a", "#eee6d2", "#17a0b0"]
+
+
+def probe(video):
+ out = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
+ "stream=r_frame_rate,nb_frames", "-of", "json", video], capture_output=True, text=True, check=True)
+ s = json.loads(out.stdout)["streams"][0]
+ num, den = s["r_frame_rate"].split("/")
+ return int(num) / int(den), int(s["nb_frames"])
+
+
+def frames(video, w, h, first=0, count=None, step=1):
+ """Yield (index, HxWx3 uint8) for frames first, first+step, ..."""
+ vf = f"select='gte(n\\,{first})*not(mod(n-{first}\\,{step}))',scale={w}:{h}:flags=area"
+ cmd = ["ffmpeg", "-v", "error", "-i", video, "-vf", vf, "-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", "rgb24"]
+ if count is not None:
+ cmd += ["-frames:v", str(count)]
+ p = subprocess.Popen(cmd + ["-"], stdout=subprocess.PIPE)
+ i = first
+ while True:
+ buf = p.stdout.read(w * h * 3)
+ if len(buf) < w * h * 3:
+ break
+ yield i, np.frombuffer(buf, np.uint8).reshape(h, w, 3)
+ i += step
+ p.stdout.close()
+ p.wait()
+
+
+def when(text, fps):
+ """'12.4' seconds or 'f372' frames, as a frame index."""
+ return int(text[1:]) if text.startswith("f") else int(round(float(text) * fps))
+
+
+def font(size):
+ for path in ("/usr/share/fonts/go/Go-Mono.ttf", "/usr/share/fonts/TTF/DejaVuSansMono.ttf"):
+ if os.path.exists(path):
+ return ImageFont.truetype(path, size)
+ return ImageFont.load_default()
+
+
+def regions_for(video, name):
+ stem = os.path.splitext(os.path.basename(video))[0]
+ return REGIONS.get(name or stem, REGIONS["bands"])
+
+
+# ---- measure --------------------------------------------------------------
+def measure(video, regions):
+ """Per frame and region, against the frame before (at AW by AH): how many pixels moved, and
+ how many changed at all (moved or fading). Returns names, moved, soft, region sizes."""
+ names = ["all"] + list(regions)
+ boxes = [(0, 0, AW, AH)] + [tuple(int(round(v * AW / W)) for v in r) for r in regions.values()]
+ rows, soft, prev = [], [], None
+ for _, f in frames(video, AW, AH):
+ f = f.astype(np.int16)
+ if prev is None:
+ rows.append([0] * len(names))
+ soft.append([0] * len(names))
+ else:
+ d = np.abs(f - prev).max(axis=2)
+ rows.append([int((d[y:y + h, x:x + w] > THRESH).sum()) for x, y, w, h in boxes])
+ soft.append([int((d[y:y + h, x:x + w] > SOFT).sum()) for x, y, w, h in boxes])
+ prev = f
+ return names, np.array(rows), np.array(soft), [w * h for _, _, w, h in boxes]
+
+
+def busy(counts, soft):
+ """Per frame and region: anything changing, by moving or by fading."""
+ return (counts >= MIN_PX) | (soft >= MIN_SOFT)
+
+
+def runs(active):
+ """[(first, last)] of consecutive true frames."""
+ out, start = [], None
+ for i, a in enumerate(active):
+ if a and start is None:
+ start = i
+ if not a and start is not None:
+ out.append((start, i - 1))
+ start = None
+ if start is not None:
+ out.append((start, len(active) - 1))
+ return out
+
+
+def events(names, counts, soft, fps, gap=2):
+ """Stretches of change, joined across gaps of up to `gap` still frames."""
+ spans = []
+ on_ = busy(counts, soft)
+ for a, b in runs(on_[:, 0]):
+ if spans and a - spans[-1][1] - 1 <= gap:
+ spans[-1] = (spans[-1][0], b)
+ else:
+ spans.append((a, b))
+ out = []
+ for a, b in spans:
+ parts = {}
+ for k, name in enumerate(names[1:], 1):
+ on = np.nonzero(on_[a:b + 1, k])[0]
+ if len(on):
+ hard = np.nonzero(counts[a:b + 1, k] >= MIN_PX)[0]
+ # (first, last, peak, frames that moved): a region that only fades moved in none
+ parts[name] = (a + int(on[0]), a + int(on[-1]), int(counts[a:b + 1, k].max()), len(hard))
+ hard = int((counts[a:b + 1, 0] >= MIN_PX).sum())
+ out.append({"from": a, "to": b, "frames": b - a + 1, "seconds": round((b - a + 1) / fps, 3),
+ "peak": int(counts[a:b + 1, 0].max()), "moved": hard, "regions": parts})
+ return out
+
+
+def report(video, names, counts, soft, sizes, fps):
+ ev = events(names, counts, soft, fps)
+ n = len(counts)
+ lines = [f"{os.path.basename(video)}: {n} frames at {fps:g} fps, {n / fps:.2f} s", ""]
+ moving = sum(e["frames"] for e in ev)
+ lines.append(f"moving {moving / n:.0%} of the time, still {1 - moving / n:.0%}; {len(ev)} events")
+ lines.append("")
+ lines.append(" from to frames peak% what changed (first frame..last, from the event's first; ~ fades only)")
+ last = -1
+ for e in ev:
+ still = e["from"] - last - 1
+ if still / fps >= HOLD:
+ lines.append(f" hold {still / fps:5.2f} s")
+ parts = " ".join(f"{'~' if n_ == 0 else ''}{k} {a - e['from']}..{b - e['from']}" for k, (a, b, _, n_) in e["regions"].items())
+ # A pop: a lot changes in a frame or two with nothing either side of it.
+ pop = " POP" if e["moved"] <= 2 and e["frames"] <= 3 and e["peak"] / sizes[0] > 0.002 else ""
+ many = len([k for k in e["regions"] if k not in ("head", "caption")])
+ lines.append(f"{e['from'] / fps:6.2f} {e['to'] / fps:6.2f} {e['frames']:6d} {100 * e['peak'] / sizes[0]:5.2f} {parts}"
+ + (f" [{many} panels]" if many > 2 else "") + pop)
+ last = e["to"]
+ if (n - last - 1) / fps >= HOLD:
+ lines.append(f" hold {(n - last - 1) / fps:5.2f} s")
+ lines += ["", "checks (each should come out empty)"] + lint(names, counts, soft, sizes, fps, ev)
+ return ev, "\n".join(lines)
+
+
+WORDS = ("head", "caption") # regions that are not panels
+
+
+def lint(names, counts, soft, sizes, fps, ev):
+ """What a viewer would trip on, found by rule. Each line: what, how often, the first few times."""
+ def some(times):
+ return ", ".join(f"{t:.2f}" for t in times[:4]) + (" ..." if len(times) > 4 else "")
+ out = []
+ on = busy(counts, soft)
+ panels = [k for k, n_ in enumerate(names) if k > 0 and n_ not in WORDS]
+
+ # POP: a lot changes in a frame or two, with stillness either side: nothing eased it.
+ pops = [e["from"] / fps for e in ev if e["moved"] <= 2 and e["frames"] <= 3 and e["peak"] / sizes[0] > 0.002]
+ if pops:
+ out.append(f" POP {len(pops):3d} a change of over 0.2% of the frame in a frame or two: {some(pops)}")
+
+ # CROWD: three or more panels begin to move within two frames of each other.
+ crowd = []
+ for e in ev:
+ first = [a - e["from"] for k, (a, b, p, n_) in e["regions"].items() if k not in WORDS and n_ > 0]
+ if sum(1 for f in first if f <= 1) >= 3:
+ crowd.append(e["from"] / fps)
+ if crowd:
+ out.append(f" CROWD {len(crowd):3d} three or more panels start together: {some(crowd)}")
+
+ # METRONOME: six or more events in a row, evenly spaced.
+ starts = [e["from"] for e in ev if e["peak"] / sizes[0] >= 0.005] # the beats, not their echoes
+ i, runs_ = 0, []
+ while i < len(starts) - 1:
+ j = i + 1
+ gap = starts[j] - starts[i]
+ while j + 1 < len(starts) and abs((starts[j + 1] - starts[j]) - gap) <= 1:
+ j += 1
+ if j - i >= 6:
+ runs_.append((starts[i] / fps, starts[j] / fps, j - i + 1, gap / fps))
+ i = j
+ for a, b, n_, gap in runs_:
+ out.append(f" METRONOME {n_:3d} events {gap:.2f} s apart, from {a:.2f} to {b:.2f}")
+
+ # RUSH: a caption arrives or changes, and the picture moves before it can be read.
+ cap = names.index("caption") if "caption" in names else None
+ rush = []
+ if cap is not None:
+ for a, b in runs(on[:, cap]):
+ if b - a < 3:
+ continue
+ later = np.nonzero(on[b + 1:, panels].any(axis=1))[0]
+ if len(later) and later[0] / fps < 1.0:
+ rush.append(b / fps)
+ if rush:
+ out.append(f" RUSH {len(rush):3d} the picture moves within a second of a caption arriving: {some(rush)}")
+
+ # AIR: nothing at all changes for three seconds.
+ air = [(a / fps, (b - a + 1) / fps) for a, b in runs(~on[:, 0]) if (b - a + 1) / fps >= 3.0]
+ if air:
+ out.append(f" AIR {len(air):3d} three seconds or more with nothing changing: " + ", ".join(f"{t:.2f} ({d:.1f} s)" for t, d in air[:4]))
+ return out or [" none"]
+
+
+def chart(video, names, counts, soft, sizes, fps, ev, path, per_row=10.0):
+ """Time left to right, a strip a region; bar height is how much of the region changed (log)."""
+ px = max(2, int(round(180 / fps))) # pixels a frame: 6 at 30 fps, 3 at 60
+ row_frames = int(per_row * fps)
+ n = len(counts)
+ nrows = -(-n // row_frames)
+ left, strip, pad = 130, 34, 34
+ rh = len(names) * strip + pad
+ img = Image.new("RGB", (left + row_frames * px + 20, nrows * rh + 30), "#0a1030")
+ d = ImageDraw.Draw(img)
+ small, tiny = font(15), font(12)
+ still = ~busy(counts, soft)[:, 0]
+ for r in range(nrows):
+ y0 = 20 + r * rh
+ a = r * row_frames
+ # holds, shaded and timed
+ for s, e in runs(still):
+ if (e - s + 1) / fps < HOLD:
+ continue
+ s2, e2 = max(s, a), min(e, a + row_frames - 1)
+ if s2 > e2:
+ continue
+ d.rectangle([left + (s2 - a) * px, y0, left + (e2 - a + 1) * px - 1, y0 + len(names) * strip - 4], fill="#141c48")
+ if s >= a:
+ d.text((left + (s - a) * px + 4, y0 + 2), f"{(e - s + 1) / fps:.2f}", font=tiny, fill="#6a7ab8")
+ for k, name in enumerate(names):
+ y = y0 + k * strip
+ d.text((8, y + 8), name, font=small, fill="#eee6d2" if k == 0 else PALETTE[(k - 1) % len(PALETTE)])
+ d.line([left, y + strip - 5, left + row_frames * px, y + strip - 5], fill="#3a4a8a")
+ ink = "#eee6d2" if k == 0 else PALETTE[(k - 1) % len(PALETTE)]
+ for i in range(a, min(a + row_frames, n)):
+ x = left + (i - a) * px
+ for c, ok, fill in ((soft[i, k], soft[i, k] >= MIN_SOFT, None), (counts[i, k], counts[i, k] >= MIN_PX, ink)):
+ if not ok:
+ continue
+ hgt = int(np.clip((np.log10(c / sizes[k]) + 4.3) / 3.6, 0.08, 1) * (strip - 8)) # 0.005% .. 20%
+ # A fade is an outline, a move is solid.
+ d.rectangle([x, y + strip - 5 - hgt, x + px - 2, y + strip - 5], fill=fill, outline=ink)
+ for i in range(a, min(a + row_frames, n) + 1):
+ if i % int(fps) == 0:
+ x = left + (i - a) * px
+ d.line([x, y0 + len(names) * strip - 4, x, y0 + len(names) * strip + 4], fill="#9aa6d9")
+ d.text((x + 3, y0 + len(names) * strip), f"{i / fps:g}s", font=tiny, fill="#9aa6d9")
+ elif i % int(fps / 2) == 0:
+ x = left + (i - a) * px
+ d.line([x, y0 + len(names) * strip - 4, x, y0 + len(names) * strip], fill="#3a4a8a")
+ img.save(path)
+
+
+def cmd_chart(a):
+ fps, _ = probe(a.video)
+ regions = regions_for(a.video, a.regions)
+ names, counts, soft, sizes = measure(a.video, regions)
+ ev, text = report(a.video, names, counts, soft, sizes, fps)
+ os.makedirs(a.out, exist_ok=True)
+ stem = os.path.splitext(os.path.basename(a.video))[0]
+ chart(a.video, names, counts, soft, sizes, fps, ev, os.path.join(a.out, stem + ".png"))
+ open(os.path.join(a.out, stem + ".txt"), "w").write(text + "\n")
+ np.savetxt(os.path.join(a.out, stem + ".csv"), np.hstack([counts, soft]), fmt="%d", delimiter=",",
+ header=",".join(names + [n + "~" for n in names]), comments="")
+ print(text)
+ return fps, ev
+
+
+# ---- look -----------------------------------------------------------------
+def crop_of(text):
+ return tuple(int(v) for v in text.split(",")) if text else (0, 0, W, H)
+
+
+def grab(video, first, last, step, crop, width):
+ """Frames first..last as PIL images of `crop` (design units), `width` pixels wide."""
+ x, y, w, h = crop
+ fps, total = probe(video)
+ last = min(last, total - 1)
+ out = []
+ for i, f in frames(video, W, H, first, (last - first) // step + 1, step):
+ im = Image.fromarray(f).crop((x, y, x + w, y + h))
+ out.append((i, im.resize((width, max(1, round(width * h / w))), Image.LANCZOS)))
+ return fps, out
+
+
+def tile(shots, fps, cols_, diff=False):
+ w, h = shots[0][1].size
+ rows = -(-len(shots) // cols_)
+ lab = 20
+ sheet = Image.new("RGB", (cols_ * (w + 4) + 4, rows * (h + lab + 4) + 4), "#444444")
+ d = ImageDraw.Draw(sheet)
+ f = font(14)
+ prev = None
+ for k, (i, im) in enumerate(shots):
+ x, y = 4 + (k % cols_) * (w + 4), 4 + (k // cols_) * (h + lab + 4)
+ show = im
+ if diff and prev is not None:
+ # What changed since the tile before, white on the frame dimmed.
+ a, b = np.asarray(im, np.int16), np.asarray(prev, np.int16)
+ moved = np.abs(a - b).max(axis=2) > THRESH
+ dim = (np.asarray(im, np.float32) * 0.35).astype(np.uint8)
+ dim[moved] = (255, 255, 255)
+ show = Image.fromarray(dim)
+ sheet.paste(show, (x, y + lab))
+ d.text((x + 2, y + 1), f"f{i} {i / fps:.3f}s", font=f, fill="#ffffff")
+ prev = im
+ return sheet
+
+
+def cmd_sheet(a):
+ fps, _ = probe(a.video)
+ first, last = when(a.start, fps), when(a.end, fps)
+ crop = crop_of(a.crop)
+ n = (last - first) // a.step + 1
+ cols_ = a.cols or min(n, 6)
+ width = a.width or min(crop[2], 1900 // cols_ - 4)
+ _, shots = grab(a.video, first, last, a.step, crop, width)
+ out = a.out or f"sheet-{os.path.splitext(os.path.basename(a.video))[0]}-f{first}-f{last}.png"
+ os.makedirs(os.path.dirname(out) or ".", exist_ok=True)
+ tile(shots, fps, cols_, a.diff).save(out)
+ print(out, f"{len(shots)} frames")
+
+
+def cmd_auto(a):
+ """A sheet for each event: every frame of it (every 2nd or 3rd if long), and a frame either side."""
+ fps, ev = cmd_chart(a)
+ stem = os.path.splitext(os.path.basename(a.video))[0]
+ made = short = 0
+ for e in ev:
+ if e["frames"] <= 2: # a pop: three of them show what the rest look like
+ short += 1
+ if short > 3:
+ continue
+ first, last = max(e["from"] - 1, 0), e["to"] + 1
+ step = max(1, -(-(last - first + 1) // 24))
+ _, shots = grab(a.video, first, last, step, (0, 0, W, H), 472)
+ path = os.path.join(a.out, f"{stem}-{e['from'] / fps:06.2f}s.png")
+ tile(shots, fps, 4).save(path)
+ made += 1
+ print(f"{made} sheets in {a.out}")
+
+
+def cmd_track(a):
+ """The centre and area of everything `color` in the crop, a frame a line: an object's spacing."""
+ fps, _ = probe(a.video)
+ first, last = when(a.start, fps), when(a.end, fps)
+ x, y, w, h = crop_of(a.crop)
+ key = np.array([int(a.color[i:i + 2], 16) for i in (0, 2, 4)], np.int16)
+ print(" frame time x y area dx dy box: left top width height")
+ px = py = None
+ for i, f in frames(a.video, W, H, first, last - first + 1):
+ m = (np.abs(f[y:y + h, x:x + w].astype(np.int16) - key).max(axis=2) <= a.tol)
+ area = int(m.sum())
+ if area < 4:
+ print(f"{i:6d} {i / fps:6.3f} - - 0")
+ px = py = None
+ continue
+ ys, xs = np.nonzero(m)
+ cx, cy = x + xs.mean(), y + ys.mean()
+ step = f"{cx - px:5.1f} {cy - py:5.1f}" if px is not None else " "
+ box = f"{x + xs.min():5d} {y + ys.min():5d} {xs.max() - xs.min() + 1:5d} {ys.max() - ys.min() + 1:5d}"
+ print(f"{i:6d} {i / fps:6.3f} {cx:6.1f} {cy:6.1f} {area:6d} {step} {box}")
+ px, py = cx, cy
+
+
+def cmd_onion(a):
+ """Every frame of a move in one picture: the first frame, and over it each later frame where it differs."""
+ fps, _ = probe(a.video)
+ first, last = when(a.start, fps), when(a.end, fps)
+ crop = crop_of(a.crop)
+ _, shots = grab(a.video, first, last, a.step, crop, min(crop[2], 1900))
+ base = np.asarray(shots[0][1], np.int16)
+ out = base.copy().astype(np.float32) * 0.5
+ for k, (_, im) in enumerate(shots[1:], 1):
+ f = np.asarray(im, np.int16)
+ moved = np.abs(f - base).max(axis=2) > THRESH
+ out[moved] = np.maximum(out[moved], f[moved] * (0.45 + 0.55 * k / (len(shots) - 1)))
+ path = a.out or f"onion-{os.path.splitext(os.path.basename(a.video))[0]}-f{first}-f{last}.png"
+ os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+ Image.fromarray(out.clip(0, 255).astype(np.uint8)).save(path)
+ print(path, f"{len(shots)} frames")
+
+
+def main():
+ p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ sub = p.add_subparsers(dest="cmd", required=True)
+ for name, fn in (("chart", cmd_chart), ("auto", cmd_auto)):
+ c = sub.add_parser(name)
+ c.add_argument("video")
+ c.add_argument("--regions")
+ c.add_argument("--out", default="tmp")
+ c.set_defaults(fn=fn)
+ c = sub.add_parser("sheet")
+ c.add_argument("video"), c.add_argument("start"), c.add_argument("end")
+ c.add_argument("--step", type=int, default=1), c.add_argument("--cols", type=int), c.add_argument("--crop")
+ c.add_argument("--width", type=int), c.add_argument("--diff", action="store_true"), c.add_argument("--out")
+ c.set_defaults(fn=cmd_sheet)
+ c = sub.add_parser("track")
+ c.add_argument("video"), c.add_argument("start"), c.add_argument("end")
+ c.add_argument("--color", required=True), c.add_argument("--crop"), c.add_argument("--tol", type=int, default=40)
+ c.set_defaults(fn=cmd_track)
+ c = sub.add_parser("onion")
+ c.add_argument("video"), c.add_argument("start"), c.add_argument("end")
+ c.add_argument("--step", type=int, default=1), c.add_argument("--crop"), c.add_argument("--out")
+ c.set_defaults(fn=cmd_onion)
+ a = p.parse_args()
+ a.fn(a)
+
+
+if __name__ == "__main__":
+ sys.exit(main())