summaryrefslogtreecommitdiff
path: root/anim/build.zig
blob: 21abbce6af13c0250cec1faf65d1dd4dc37d25fb (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
//! canim: the animation library (`src/`, the module `canim`), the small C
//! programs it records (`tools/`), and its scenes (`scenes/`), each a
//! player built by sketchbook.
//!
//!   zig build record                       the recordings, into zig-out/recordings
//!   zig build run -Dscene=list             a scene in a window, with hot reload
//!   zig build export -Dscene=list -- DIR   its frames (see sketchbook's README)
//!   zig build videos -Doptimize=ReleaseFast   every scene as out/NAME-1080.mp4 and -2160
//!   zig build test                         unit tests and the scripted runs
//!   zig build bench                        what a recording costs a frame
//!   zig build motion -Dscene=list -Doptimize=ReleaseFast   how a scene moves: contact sheets and change plots in out/motion/list
//!
//! Needs, besides what sketchbook and ztrace need: `setarch`, and for
//! `videos`, `ffmpeg`. The sqlite scene also needs `clang` and SQLite's
//! amalgamation (`-Dsqlite=DIR`); without it that scene is not made.
const std = @import("std");
const sketchbook = @import("sketchbook");

const scenes = [_][]const u8{ "title", "fib", "list", "sort", "syscalls", "zoom", "gallery" };

pub fn build(b: *std.Build) void {
    const target = b.standardTargetOptions(.{});
    const optimize = b.standardOptimizeOption(.{});
    const chosen = b.option([]const u8, "scene", "The scene `run`, `lib` and `export` are about (default fib)") orelse "fib";

    const sb = sketchbook.init(b, .{ .target = target, .optimize = optimize });
    const zt = b.dependency("ztrace", .{ .target = target, .optimize = optimize });
    const pardes = b.dependency("pardes", .{ .target = target, .optimize = optimize, .@"tree-sitter" = .minimal });
    // The tools a recording is made with, always optimized: a step through
    // the agent's or the runtime's code is a moment of the recording, and
    // a slow tracer stretches the clock's retry loops it steps through.
    const tools = b.dependency("ztrace", .{ .target = target, .optimize = .ReleaseFast });
    const ztrace = tools.artifact("ztrace");

    const imports: []const std.Build.Module.Import = &.{
        .{ .name = "anim", .module = sb.modules.anim },
        .{ .name = "sketch", .module = sb.modules.sketch },
        .{ .name = "ztrace", .module = zt.module("ztrace") },
        .{ .name = "ztrace-types", .module = zt.module("ztrace-types") },
        .{ .name = "pardes-syntax", .module = pardes.module("pardes-syntax") },
    };
    const canim = b.addModule("canim", .{ .root_source_file = b.path("src/root.zig"), .imports = imports });

    // ---- The recordings: each program of tools/ run once under ztrace's
    // debugger, without address randomisation (`setarch -R`), so a recording
    // and the frames made from it are the same on every build.
    const record_step = b.step("record", "Record the programs of tools/ into zig-out/recordings");
    const embedded = b.addWriteFiles();
    var table: std.ArrayList(u8) = .empty;

    // The pass and the agent to preload, as ztrace's own build makes and names them.
    const plugin = tools.namedLazyPath("ztrace-pass");
    const agent = tools.namedLazyPath("ztrace-agent");
    const musl = b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .linux, .abi = .musl });

    const Recording = struct {
        name: []const u8,
        source: []const u8,
        /// Where the debugger stops it: every instruction, or every line.
        /// Each stop records the registers, the stack under them, what
        /// they point at, and the instructions at the pc.
        by: []const u8,
        /// Compiled by `ztrace cc` to tell these of itself as well (its
        /// variables with their types), for the functions and globals
        /// `only` names: everything else is the compiler's alone. Null for
        /// the program as the compiler alone makes it (static, unoptimised),
        /// whose instructions are then all its own.
        instr: ?[]const u8 = null,
        only: []const u8 = "*",
        /// Run through to here before the first stop: the loader, the C
        /// library's start, and what the scene does not show.
        from: ?[]const u8 = null,
        /// With the agent preloaded: `malloc` and `free` too.
        alloc: bool = false,
        /// A file of tools/ it reads, by its bare name.
        reads: ?[]const u8 = null,
        /// Bytes of stack above rsp each stop records. An instrumented
        /// program's runtime reads the clock through the vDSO, whose retry
        /// loop is stepped: with 512 or more the loop seldom gets through
        /// (`list` took 27 000 to 42 000 moments at 512, and 0.3 to 1.6
        /// million at 1024, against 3 900 at 256), so those take 256.
        stack: []const u8 = "1024",
    };
    for ([_]Recording{
        .{ .name = "steps", .source = "fib.c", .by = "insn" },
        .{ .name = "hello", .source = "hello.c", .by = "line", .reads = "note" },
        // By instruction, as `steps` is: the registers are the program's
        // own, so they strike and link. Only the variables the scenes name
        // are told by the instrumentation (each a call into its runtime,
        // stepped like any other); `list` starts at `reverse`, its pushes
        // (the agent's and the runtime's first calls) run through.
        .{ .name = "sort", .source = "sort.c", .by = "insn", .instr = "vars", .only = "sort,a", .from = "main", .stack = "256" },
        .{ .name = "list", .source = "list.c", .by = "insn", .instr = "vars", .only = "push,reverse,head", .from = "reverse", .alloc = true, .stack = "256" },
    }) |r| {
        const source = b.path(b.fmt("tools/{s}", .{r.source}));
        var program: std.Build.LazyPath = undefined;
        if (r.instr) |instr| {
            const cc = b.addRunArtifact(ztrace);
            cc.addArgs(&.{ "cc", "--instr", instr, "--only", r.only, "--plugin" });
            cc.addFileArg(plugin);
            cc.addArg("--rt");
            cc.addArtifactArg(tools.artifact("ztrace-rt"));
            // Not position-independent: code and heap at short addresses, as
            // the design's columns are drawn for (seven digits, not twelve).
            // Bound at load and with no canary: no resolver and no check is
            // stepped through that the source does not say.
            cc.addArgs(&.{ "--", "clang", "-O0", "-g", "-no-pie", "-fno-stack-protector", "-Wl,-z,now" });
            cc.addFileArg(source);
            cc.addArg("-o");
            program = cc.addOutputFileArg(r.name);
        } else {
            const mod = b.createModule(.{ .target = musl, .optimize = .Debug, .link_libc = true, .omit_frame_pointer = false, .sanitize_c = .off, .stack_protector = false });
            mod.addCSourceFile(.{ .file = source, .flags = &.{ "-O0", "-g" } });
            program = b.addExecutable(.{ .name = r.name, .root_module = mod, .linkage = .static }).getEmittedBin();
        }
        // It runs by a bare name in a directory of its own, with no
        // environment: the strings on its stack (its name, its variables)
        // end up on screen, and should say nothing of the machine it was made on.
        const where = b.addWriteFiles();
        _ = where.addCopyFile(program, r.name);
        if (r.reads) |name| _ = where.addCopyFile(b.path(b.fmt("tools/{s}", .{name})), name);
        const run = b.addSystemCommand(&.{ "env", "-i", "setarch", "-R" });
        run.addArtifactArg(ztrace);
        // More round each stop than the working frame shows: the map of
        // the whole context wants the words of the stack, 256 bytes round
        // what a pointer leads to and what a step touches, and 69
        // instructions round the pc.
        run.addArgs(&.{ "record", "--by", r.by, "--stack", r.stack, "--around", "256", "--disasm", "69", "-s", "64" });
        if (r.from) |place| run.addArgs(&.{ "--from", place });
        // What an instrumented program tells of itself comes through a ring.
        if (r.instr != null) run.addArg("--ring");
        if (r.alloc) {
            run.addArgs(&.{ "--alloc", "--preload" });
            run.addFileArg(agent);
        }
        run.addArg("-o");
        const trace = run.addOutputFileArg(b.fmt("{s}.ztrace", .{r.name}));
        run.addArgs(&.{ "--", b.fmt("./{s}", .{r.name}) });
        run.setCwd(where.getDirectory());
        run.expectExitCode(0);
        run.expectStdErrMatch("exit 0");

        // Into the scene's own binary: the recording's bytes and the source it ran.
        const file = b.fmt("{s}.ztrace", .{r.name});
        record_step.dependOn(&b.addInstallFile(trace, b.fmt("recordings/{s}", .{file})).step);
        _ = embedded.addCopyFile(trace, file);
        _ = embedded.addCopyFile(source, r.source);
        table.print(b.allocator, "pub const {s}: Program = .{{ .trace = @embedFile(\"{s}\"), .file = \"{s}\", .source = @embedFile(\"{s}\") }};\n", .{ r.name, file, r.source, r.source }) catch @panic("OOM");
    }
    // SQLite, the next rung: its amalgamation is not ours to copy, so the
    // scene is made only where it is. Built as the bench builds it (-O2,
    // frame pointers kept), and as the instrumented ones are (no PIE, bound
    // at load, no canary); run once under the debugger, stopping where a
    // statement's bytecode is run (`sqlite3_step`), where a row enters the
    // B-tree (`sqlite3BtreeInsert`), where a page is written
    // (`unixWrite`) and where its `pwrite64` has returned to it (the
    // instruction after the call, found by objdump: there is no name for
    // it); every syscall recorded with the bytes it points at, up to a page.
    // In a directory of its own each time: the database it makes must not be there.
    const sqlite_dir = b.option([]const u8, "sqlite", "SQLite's amalgamation (sqlite3.c, sqlite3.h): the sqlite scene is made only with it") orelse b.pathJoin(&.{ b.graph.environ_map.get("HOME") orelse "/", "05-genizah/sqlite/sqlite-amalgamation-3500400" });
    const sqlite_c = b.pathJoin(&.{ sqlite_dir, "sqlite3.c" });
    const has_sqlite = if (std.Io.Dir.cwd().access(b.graph.io, sqlite_c, .{})) |_| true else |_| false;
    if (has_sqlite) {
        const cc = b.addSystemCommand(&.{ "clang", "-O2", "-g", "-fno-omit-frame-pointer", "-no-pie", "-fno-stack-protector", "-Wl,-z,now", "-w" });
        cc.addPrefixedDirectoryArg("-I", .{ .cwd_relative = sqlite_dir });
        cc.addFileArg(b.path("tools/sqlite.c"));
        cc.addFileArg(.{ .cwd_relative = sqlite_c });
        cc.addArgs(&.{ "-lm", "-o" });
        const program = cc.addOutputFileArg("sqlite");
        const run = b.addSystemCommand(&.{ "sh", "-c",
            \\set -e
            \\ztrace=$(realpath "$0") program=$(realpath "$1")
            \\case $2 in /*) out=$2 ;; *) out=$PWD/$2 ;; esac
            \\back=$(objdump -d --no-show-raw-insn "$program" | awk '/<unixWrite>:/ { f = 1 } f && /call.*pwrite64/ { getline; sub(":", "", $1); print "0x" $1; exit }')
            \\[ -n "$back" ] || { echo "no call of pwrite64 in unixWrite" >&2; exit 1; }
            \\d=$(mktemp -d)
            \\trap 'rm -rf "$d"' EXIT
            \\cp "$program" "$d/sqlite"
            \\cd "$d"
            \\env -i setarch -R "$ztrace" record --by break -b sqlite3_step -b sqlite3BtreeInsert -b unixWrite -b "*$back" --stack 1024 --around 256 --disasm 69 -s 4096 -o "$out" -- ./sqlite
        });
        run.addArtifactArg(ztrace);
        run.addFileArg(program);
        const trace = run.addOutputFileArg("sqlite.ztrace");
        run.expectStdErrMatch("exit 0");
        record_step.dependOn(&b.addInstallFile(trace, "recordings/sqlite.ztrace").step);
        _ = embedded.addCopyFile(trace, "sqlite.ztrace");
        _ = embedded.addCopyFile(.{ .cwd_relative = sqlite_c }, "sqlite3.c");
        table.print(b.allocator, "pub const sqlite: Program = .{{ .trace = @embedFile(\"sqlite.ztrace\"), .file = \"sqlite3.c\", .source = @embedFile(\"sqlite3.c\") }};\n", .{}) catch @panic("OOM");
    }

    // The gallery's own: a recording of the list kept in test/gallery, so its
    // pictures change when the drawing does and not when a new recording does.
    _ = embedded.addCopyFile(b.path("test/gallery/list.ztrace"), "gallery.ztrace");
    table.print(b.allocator, "pub const gallery: Program = .{{ .trace = @embedFile(\"gallery.ztrace\"), .file = \"list.c\", .source = @embedFile(\"list.c\") }};\n", .{}) catch @panic("OOM");
    const recordings = b.createModule(.{ .root_source_file = embedded.add("recordings.zig", b.fmt(
        \\//! Generated by canim's build.zig: the recordings and the sources they ran.
        \\pub const Program = struct {{ trace: []const u8, file: []const u8, source: []const u8 }};
        \\{s}
    , .{table.items})) });

    // ---- Tests: the library's own, then on the real recordings.
    const test_step = b.step("test", "Unit tests and the scripted runs");
    test_step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = b.createModule(.{ .root_source_file = b.path("src/root.zig"), .target = target, .optimize = optimize, .imports = imports }) })).step);
    test_step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = b.createModule(.{
        .root_source_file = b.path("test/recordings.zig"),
        .target = target,
        .optimize = optimize,
        .imports = &.{ .{ .name = "canim", .module = canim }, .{ .name = "recordings", .module = recordings }, .{ .name = "anim", .module = sb.modules.anim }, .{ .name = "ztrace", .module = zt.module("ztrace") } },
    }) })).step);

    // What a recording costs a frame: always optimized, whatever the build is.
    const fast = b.createModule(.{ .root_source_file = b.path("src/root.zig"), .target = target, .optimize = .ReleaseFast, .imports = imports });
    const bench = b.addRunArtifact(b.addExecutable(.{ .name = "bench", .root_module = b.createModule(.{
        .root_source_file = b.path("tools/bench.zig"),
        .target = target,
        .optimize = .ReleaseFast,
        .imports = &.{ .{ .name = "canim", .module = fast }, .{ .name = "recordings", .module = recordings }, .{ .name = "ztrace", .module = zt.module("ztrace") } },
    }) }));
    bench.has_side_effects = true;
    bench.addPassthruArgs();
    b.step("bench", "Time opening each recording and the queries a frame makes of it").dependOn(&bench.step);

    // ---- Scenes.
    const videos = b.step("videos", "Export every scene to out/NAME-1080.mp4 and out/NAME-2160.mp4 (use -Doptimize=ReleaseFast)");
    for (scenes ++ [_][]const u8{"sqlite"}) |name| {
        if (std.mem.eql(u8, name, "sqlite") and !has_sqlite) continue;
        const scene = sb.addScene(.{ .root_source_file = b.path(b.fmt("scenes/{s}.zig", .{name})), .imports = &.{
            .{ .name = "canim", .module = canim },
            .{ .name = "recordings", .module = recordings },
        } });
        const exe = scene.app.addExecutable();
        if (std.mem.eql(u8, name, chosen)) {
            b.installArtifact(exe);
            const library = scene.app.addLibrary();
            b.step("run", "Play -Dscene with hot reload (rebuild with `zig build lib --watch`)").dependOn(&scene.app.addRun(library, .{}).step);
            b.step("lib", "Build and install -Dscene's hot-reloadable library").dependOn(&b.addInstallArtifact(library, .{}).step);
            const exporting = b.addRunArtifact(exe);
            exporting.addArgs(&.{ "--state", b.fmt("{s}.state", .{name}), "--export" });
            exporting.addPassthruArgs();
            exporting.setCwd(b.path("."));
            exporting.stdio = .inherit;
            b.step("export", "Export -Dscene's frames: -- DIR for PNGs, or - for raw RGBA on stdout; then --fps N, --frames N, --size WxH").dependOn(&exporting.step);
            // How it moves: contact sheets of every transition and plots of
            // what changes from frame to frame (tools/motion.py).
            const motion = b.addSystemCommand(&.{ "python3", "tools/motion.py" });
            motion.addArtifactArg(exe);
            motion.addArg(b.fmt("out/motion/{s}", .{name}));
            motion.addPassthruArgs();
            motion.setCwd(b.path("."));
            motion.stdio = .inherit;
            b.step("motion", "Judge -Dscene's motion: out/motion/NAME/ gets contact sheets of each transition and plots of change per frame").dependOn(&motion.step);
        }
        if (std.mem.eql(u8, name, "gallery")) {
            // The catalogue drawn: its pages against the pictures in test/gallery
            // (`zig build gallery` draws them anew), at the design's size.
            for ([_]struct { []const u8, []const u8 }{ .{ "test", ".zig-cache/canim" }, .{ "gallery", "test/gallery" } }) |use| {
                const run = b.addRunArtifact(exe);
                run.addArgs(&.{ "--headless", "--fresh", "--size", "1920x1080", "--define", b.fmt("dir={s}", .{use[1]}), "--script" });
                run.addFileArg(b.path("test/gallery.script"));
                run.addArg("--state");
                _ = run.addOutputFileArg(b.fmt("gallery-{s}.state", .{use[0]}));
                run.setCwd(b.path("."));
                run.expectExitCode(0);
                run.has_side_effects = true;
                if (std.mem.eql(u8, use[0], "test")) test_step.dependOn(&run.step) else b.step("gallery", "Draw the gallery's pages anew into test/gallery, the pictures the test compares with").dependOn(&run.step);
            }
            continue;
        }
        test_step.dependOn(&scene.app.addScript(exe, b.path(b.fmt("test/{s}.script", .{name})), .{}).step);

        for ([_][]const u8{ "1920x1080", "3840x2160" }) |size| {
            // The design's master: x264 from the raw frames, tagged bt709.
            const video = b.addSystemCommand(&.{ "sh", "-c",
                \\set -e
                \\mkdir -p out
                \\"$0" --state "$1" --export - --size "$2" --fps 60 |
                \\    ffmpeg -loglevel error -y -f rawvideo -pix_fmt rgba -s "$2" -r 60 -i - \
                \\        -c:v libx264 -crf 14 -preset slow -g 600 -pix_fmt yuv420p \
                \\        -colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range tv "$3"
            });
            video.addArtifactArg(exe);
            _ = video.addOutputFileArg(b.fmt("{s}-video.state", .{name}));
            video.addArgs(&.{ size, b.fmt("out/{s}-{s}.mp4", .{ name, size[5..] }) });
            video.setCwd(b.path("."));
            video.has_side_effects = true;
            videos.dependOn(&video.step);
        }
    }
}