//! Every run of a sketch, recorded: one xitdb database per sketch, beside //! its state file, holding an append-only log of the frames live `update` //! was fed, and the State at the moments that matter (keyframes). Nothing //! in it is ever overwritten: marks and clips are names pointing into it. //! //! root = map { //! version 1 //! sessions list of session, one a run //! marks list of { name, session, key }: a named keyframe (the newest of a name wins) //! clips list of { name, session, key, start, end }: frames start..end of a //! session, played from keyframe `key`; `end` 0 while recording //! } //! session = map { //! started wall-clock ns at open ended ns at a clean exit, else 0 //! from "" or where it came from ("import ") //! stretches list of { frame, build, source, layout }: one a load (start, reloads); //! `layout` is the state file's header (memory.readLayout reads it) //! batches list of bytes: a little-endian u64, the first frame's index, then //! whole frames of raw `abi.Event`s, each frame ending with its tick //! keys list of { frame, batch, stretch, why, time, state, line }: the //! State before frame `frame` is fed, in stretch `stretch`'s layout; //! `batch` is where reading frames from it starts //! frames frames committed //! } //! //! One transaction (`commit`) a second of session time, and at once for a //! reload, a mark, a clip's start or end, and exit: a crash loses less than //! a second of play; fsync only at exit and for a mark. const Sessions = @This(); const std = @import("std"); const xitdb = @import("xitdb"); const abi = @import("abi"); pub const Disk = xitdb.Database(.buffered_file, u64); pub const Memory = xitdb.Database(.memory, u64); /// `file` for interactive runs of a build that records, `memory` for a /// scripted run from fresh state (time travel, nothing on disk), `off` /// otherwise (an export, a release build). pub const Mode = enum { off, memory, file }; pub const Why = enum(u64) { start, periodic, jump, before_reload, after_reload, mark, exit, import }; /// Pending frames are committed once they span a second of session time /// (less a millisecond: 60 ticks of a 60 Hz clock that rounds down are one). pub const commit_ns: u64 = std.time.ns_per_s - std.time.ns_per_ms; /// A keyframe at least this often while the State changes. pub const periodic_ns: u64 = 60 * std.time.ns_per_s; /// Mark and clip names: no spaces, at most this long. pub const max_name = 32; const K = enum(u64) { version = 1, sessions, marks, clips, started, ended, from, stretches, batches, keys, frames, frame, build, source, layout, batch, stretch, why, time, state, line, name, session, key, start, end }; fn key(k: K) u64 { return @intFromEnum(k); } pub const Stretch = struct { frame: u64 = 0, build: u64, source: []const u8, layout: []const u8 }; const Keyframe = struct { frame: u64, batch: u64, stretch: u64, why: Why, time: u64, state: []u8, line: []u8 }; gpa: std.mem.Allocator, /// Files and the wall clock; a memory database runs on `std.Io.failing`. io: std.Io, db: union(Mode) { off, memory: Memory, file: Disk } = .off, file: ?std.Io.File = null, /// The database's write buffer, an allocation of its own: the database /// points at it. buffer: ?*std.Io.Writer.Allocating = null, /// The database's path (file mode), owned. path: []const u8 = "", /// This run's index in `sessions`. session: u64 = 0, /// Frames logged, committed or not, and committed. frames: u64 = 0, committed: u64 = 0, /// This session's committed batches, its stretches and its keyframes /// (queued ones too). batches: u64 = 0, stretches: u64 = 0, keys: u64 = 0, /// The frames since the last commit, as the batch they become. batch: std.ArrayList(u8) = .empty, /// The first pending frame's tick. batch_time: u64 = 0, /// The latest event time logged: times never go back. last_time: u64 = 0, /// The last keyframe's time, stretch and bytes' hash. key_time: u64 = 0, key_stretch: u64 = 0, key_hash: u64 = 0, queued: std.ArrayList(Keyframe) = .empty, /// The clip being recorded, an index into `clips`. recording: ?u64 = null, /// State may have changed other than by a logged live frame (a replay ran): /// the next live frame starts with a keyframe. away: bool = false, /// The console line that last changed State, for the next keyframe. line: [256]u8 = undefined, line_len: usize = 0, /// Opens the database at `path` (file mode: created with its directory when /// missing; a file xitdb refuses is renamed aside, never deleted) and starts /// this run's session in it. pub fn open(gpa: std.mem.Allocator, io: std.Io, want: Mode, path: []const u8, first: Stretch, started: u64) !Sessions { var s: Sessions = .{ .gpa = gpa, .io = io }; if (want == .off) return s; errdefer s.deinit(); const buffer = try gpa.create(std.Io.Writer.Allocating); buffer.* = .init(gpa); s.buffer = buffer; switch (want) { .off => unreachable, .memory => s.db = .{ .memory = try Memory.init(std.Io.failing, .{ .buffer = buffer, .file_id = 0 }) }, .file => { s.path = try gpa.dupe(u8, path); const cwd = std.Io.Dir.cwd(); if (std.fs.path.dirname(path)) |dir| try cwd.createDirPath(io, dir); var aside = false; while (true) { const file = try cwd.createFile(io, path, .{ .read = true, .truncate = false }); if (Disk.init(io, .{ .file = file, .buffer = buffer, .fsync = false })) |db| { s.db = .{ .file = db }; s.file = file; break; } else |err| { file.close(io); switch (err) { error.InvalidDatabase, error.InvalidVersion, error.InvalidHashSize, error.TruncatedDatabase, error.InvalidEnumTag, error.EndOfStream => {}, else => return err, } if (aside) return err; aside = true; var buf: [std.fs.max_path_bytes + 32]u8 = undefined; const to = try std.fmt.bufPrint(&buf, "{s}.broken.{d}", .{ path, started / std.time.ns_per_s }); try cwd.rename(path, cwd, to, io); std.log.warn("{s} is no sessions database this host reads: moved to {s}", .{ path, to }); } } }, } s.session = try s.count(.sessions); try s.commit(.{ .open = .{ .started = started, .first = first } }); s.stretches = 1; return s; } pub fn deinit(s: *Sessions) void { s.dropQueued(); s.queued.deinit(s.gpa); s.batch.deinit(s.gpa); if (s.file) |f| f.close(s.io); if (s.buffer) |b| { b.deinit(); s.gpa.destroy(b); } s.gpa.free(s.path); s.* = .{ .gpa = s.gpa, .io = s.io }; } /// The last frames, an exit keyframe of `state`, and the session's end, in /// one transaction synced to disk; then closed. pub fn close(s: *Sessions, state: []const u8, ended: u64) void { defer s.deinit(); if (s.db == .off) return; _ = s.keyframe(.exit, state) catch {}; s.commitSynced(.{ .end = ended }) catch |err| std.log.warn("sessions: last commit failed: {t}", .{err}); } pub fn mode(s: *const Sessions) Mode { return s.db; } /// Remembers the console line that changed State, for the next keyframe. pub fn noteLine(s: *Sessions, line: []const u8) void { s.line_len = @min(line.len, s.line.len); @memcpy(s.line[0..s.line_len], line[0..s.line_len]); } /// One live frame, its tick last, before `update` is fed it: its times are /// made never to go back (what `update` then sees is what a replay will), /// and it is logged, after a keyframe of `state` when `why` says so or the /// last is a minute old. pub fn frame(s: *Sessions, one: []abi.Event, state: []const u8, why: ?Why) !void { for (one) |*e| { e.time = @max(e.time, s.last_time); s.last_time = e.time; } if (s.db == .off) return; const tick = one[one.len - 1].time; if (s.batch.items.len > 0 and tick -| s.batch_time >= commit_ns) try s.commit(.none); if (why) |w| { _ = try s.keyframe(w, state); } else if (tick -| s.key_time >= periodic_ns) _ = try s.keyframe(.periodic, state); if (s.batch.items.len == 0) { try s.batch.appendSlice(s.gpa, std.mem.asBytes(&std.mem.nativeToLittle(u64, s.frames))); s.batch_time = tick; } try s.batch.appendSlice(s.gpa, std.mem.sliceAsBytes(one)); s.frames += 1; } /// Queues a keyframe of `state` before the next frame, and returns its /// index; an unchanged State is the last keyframe again (but for a /// `before_reload` one: the last word in the old layout). pub fn keyframe(s: *Sessions, why: Why, state: []const u8) !u64 { if (s.db == .off) return error.NotRecording; const hash = std.hash.Wyhash.hash(0, state); defer s.line_len = 0; if (s.keys > 0 and why != .before_reload and hash == s.key_hash and s.key_stretch == s.stretches - 1) return s.keys - 1; const state_copy = try s.gpa.dupe(u8, state); errdefer s.gpa.free(state_copy); const line_copy = try s.gpa.dupe(u8, s.line[0..s.line_len]); errdefer s.gpa.free(line_copy); try s.queued.append(s.gpa, .{ .frame = s.frames, .batch = s.batches, .stretch = s.stretches - 1, .why = why, .time = s.last_time, .state = state_copy, .line = line_copy }); s.key_time = s.last_time; s.key_hash = hash; s.key_stretch = s.stretches - 1; s.keys += 1; return s.keys - 1; } /// A new stretch from the next frame: a reload's build, source and layout. pub fn stretch(s: *Sessions, st: Stretch) !void { if (s.db == .off) return; var with = st; with.frame = s.frames; try s.commit(.{ .stretch = with }); s.stretches += 1; } /// Mark `name`: a keyframe of `state`, synced to disk. pub fn mark(s: *Sessions, name: []const u8, state: []const u8) !void { const k = try s.keyframe(.mark, state); try s.commitSynced(.{ .mark = .{ .name = name, .key = k } }); } /// Starts clip `name` at the next frame, played from keyframe `k`. pub fn startClip(s: *Sessions, name: []const u8, k: u64) !void { const index = try s.count(.clips); try s.commit(.{ .clip = .{ .name = name, .key = k, .start = s.frames } }); s.recording = index; } /// Ends the clip being recorded after the frames logged so far. pub fn stopClip(s: *Sessions) !void { const index = s.recording orelse return; try s.commit(.{ .clip_end = .{ .index = index, .end = s.frames } }); s.recording = null; } fn dropQueued(s: *Sessions) void { for (s.queued.items) |k| { s.gpa.free(k.state); s.gpa.free(k.line); } s.queued.clearRetainingCapacity(); } const Extra = union(enum) { none, open: struct { started: u64, first: Stretch }, stretch: Stretch, mark: struct { name: []const u8, key: u64 }, clip: struct { name: []const u8, key: u64, start: u64 }, clip_end: struct { index: u64, end: u64 }, end: u64, }; fn commitSynced(s: *Sessions, extra: Extra) !void { if (s.db == .file) s.db.file.core.file.fsync = true; defer if (s.db == .file) { s.db.file.core.file.fsync = false; }; try s.commit(extra); } /// One transaction: the pending frames, the queued keyframes and `extra`. pub fn commit(s: *Sessions, extra: Extra) !void { switch (s.db) { .off => return, inline .memory, .file => |*db| try db.runContext(Commit(@TypeOf(db.*)){ .s = s, .extra = extra }), } if (s.batch.items.len > 0) { s.batches += 1; s.committed = s.frames; s.batch.clearRetainingCapacity(); } s.dropQueued(); } fn Commit(comptime DB: type) type { return struct { s: *Sessions, extra: Extra, const Map = DB.HashMap(.read_write); const List = DB.ArrayList(.read_write); pub fn run(ctx: @This(), cursor: *DB.Cursor(.read_write)) !void { const s = ctx.s; const root = try Map.init(cursor.*); const sessions = try List.init(try root.putCursor(key(.sessions))); if (ctx.extra == .open) { const o = ctx.extra.open; try root.put(key(.version), .{ .uint = 1 }); const session = try Map.init(try sessions.appendCursor()); try session.put(key(.started), .{ .uint = o.started }); try session.put(key(.ended), .{ .uint = 0 }); try session.put(key(.from), .{ .bytes = "" }); try session.put(key(.frames), .{ .uint = 0 }); try putStretch(try List.init(try session.putCursor(key(.stretches))), o.first); return; } const session = try Map.init(try sessions.putCursor(@intCast(s.session))); if (s.batch.items.len > 0) { try (try List.init(try session.putCursor(key(.batches)))).append(.{ .bytes = s.batch.items }); try session.put(key(.frames), .{ .uint = s.frames }); } if (s.queued.items.len > 0) { const keys = try List.init(try session.putCursor(key(.keys))); for (s.queued.items) |q| { const k = try Map.init(try keys.appendCursor()); try k.put(key(.frame), .{ .uint = q.frame }); try k.put(key(.batch), .{ .uint = q.batch }); try k.put(key(.stretch), .{ .uint = q.stretch }); try k.put(key(.why), .{ .uint = @intFromEnum(q.why) }); try k.put(key(.time), .{ .uint = q.time }); try k.put(key(.state), .{ .bytes = q.state }); try k.put(key(.line), .{ .bytes = q.line }); } } switch (ctx.extra) { .none, .open => {}, .stretch => |st| try putStretch(try List.init(try session.putCursor(key(.stretches))), st), .mark => |m| { const item = try Map.init(try (try List.init(try root.putCursor(key(.marks)))).appendCursor()); try item.put(key(.name), .{ .bytes = m.name }); try item.put(key(.session), .{ .uint = s.session }); try item.put(key(.key), .{ .uint = m.key }); }, .clip => |c| { const item = try Map.init(try (try List.init(try root.putCursor(key(.clips)))).appendCursor()); try item.put(key(.name), .{ .bytes = c.name }); try item.put(key(.session), .{ .uint = s.session }); try item.put(key(.key), .{ .uint = c.key }); try item.put(key(.start), .{ .uint = c.start }); try item.put(key(.end), .{ .uint = 0 }); }, .clip_end => |c| { const item = try Map.init(try (try List.init(try root.putCursor(key(.clips)))).putCursor(@intCast(c.index))); try item.put(key(.end), .{ .uint = c.end }); }, .end => |t| try session.put(key(.ended), .{ .uint = t }), } } fn putStretch(list: List, st: Stretch) !void { const item = try Map.init(try list.appendCursor()); try item.put(key(.frame), .{ .uint = st.frame }); try item.put(key(.build), .{ .uint = st.build }); try item.put(key(.source), .{ .bytes = st.source }); try item.put(key(.layout), .{ .bytes = st.layout }); } }; } // --------------------------------------------------------------------------- // Reading pub const Found = struct { session: u64, key: u64 }; pub const Clip = struct { session: u64, key: u64, start: u64, end: u64 }; /// A keyframe, its bytes `gpa`'s (`free`). pub const Loaded = struct { frame: u64, batch: u64, why: Why, time: u64, state: []const u8, line: []const u8, /// Its stretch: the layout of `state`, and the build that wrote it. stretch: u64, build: u64, source: []const u8, layout: []const u8, pub fn free(l: Loaded, gpa: std.mem.Allocator) void { for ([_][]const u8{ l.state, l.line, l.source, l.layout }) |b| gpa.free(b); } }; /// The newest mark named `name`. pub fn findMark(s: *Sessions, name: []const u8) !?Found { const n = try s.count(.marks); var i = n; while (i > 0) { i -= 1; const e = try s.entry(.marks, i, name) orelse continue; return .{ .session = e.session, .key = e.key }; } return null; } /// The newest clip named `name`. pub fn findClip(s: *Sessions, name: []const u8) !?Clip { var i = try s.count(.clips); while (i > 0) { i -= 1; return try s.entry(.clips, i, name) orelse continue; } return null; } /// Entry `i` of `marks` or `clips` when it is named `name`. fn entry(s: *Sessions, which: K, i: u64, name: []const u8) !?Clip { switch (s.db) { .off => return error.NotRecording, inline .memory, .file => |*db| { const R = Read(@TypeOf(db.*)); const e = try R.at(try R.list(try R.root(db), which) orelse return null, i); var buf: [max_name]u8 = undefined; const got = (try R.field(e, .name)).readBytes(&buf) catch return null; if (!std.mem.eql(u8, got, name)) return null; return .{ .session = try R.uint(e, .session), .key = try R.uint(e, .key), .start = if (which == .clips) try R.uint(e, .start) else 0, .end = if (which == .clips) try R.uint(e, .end) else 0, }; }, } } /// Entries in the root list `which`, or sessions. pub fn count(s: *Sessions, which: K) !u64 { switch (s.db) { .off => return 0, inline .memory, .file => |*db| { const R = Read(@TypeOf(db.*)); return if (try R.list(try R.root(db), which)) |l| try l.count() else 0; }, } } /// Keyframe `k` of `session`, with its stretch. pub fn load(s: *Sessions, session: u64, k: u64) !Loaded { switch (s.db) { .off => return error.NotRecording, inline .memory, .file => |*db| { const R = Read(@TypeOf(db.*)); const ses = try R.session(db, session); const e = try R.at(try R.list(ses, .keys) orelse return error.NoSuchKeyframe, k); const st = try R.uint(e, .stretch); const stretch_entry = try R.at(try R.list(ses, .stretches) orelse return error.CorruptSession, st); const why = std.enums.fromInt(Why, try R.uint(e, .why)) orelse return error.CorruptSession; const state = try R.bytes(e, .state, s.gpa); errdefer s.gpa.free(state); const line = try R.bytes(e, .line, s.gpa); errdefer s.gpa.free(line); const source = try R.bytes(stretch_entry, .source, s.gpa); errdefer s.gpa.free(source); return .{ .frame = try R.uint(e, .frame), .batch = try R.uint(e, .batch), .why = why, .time = try R.uint(e, .time), .state = state, .line = line, .stretch = st, .build = try R.uint(stretch_entry, .build), .source = source, .layout = try R.bytes(stretch_entry, .layout, s.gpa), }; }, } } /// The newest keyframe of `session` whose `by` (`.frame` or `.time`) is at /// most `value`: a binary search. pub fn keyBefore(s: *Sessions, session: u64, by: K, value: u64) !?u64 { switch (s.db) { .off => return error.NotRecording, inline .memory, .file => |*db| { const R = Read(@TypeOf(db.*)); const keys = try R.list(try R.session(db, session), .keys) orelse return null; var lo: u64 = 0; var hi = try keys.count(); while (lo < hi) { const mid = lo + (hi - lo) / 2; if (try R.uint(try R.at(keys, mid), by) <= value) lo = mid + 1 else hi = mid; } return if (lo == 0) null else lo - 1; }, } } /// Appends to `out` the events of frames `start..end` of `session`, read /// from batch `from` on. pub fn events(s: *Sessions, session: u64, from: u64, start: u64, end: u64, out: *std.ArrayList(abi.Event)) !void { switch (s.db) { .off => return error.NotRecording, inline .memory, .file => |*db| { const R = Read(@TypeOf(db.*)); const batches = try R.list(try R.session(db, session), .batches) orelse return; const n = try batches.count(); var i = from; while (i < n) : (i += 1) { const blob = try (try batches.getCursor(@intCast(i)) orelse return error.CorruptSession).readBytesAlloc(s.gpa, null); defer s.gpa.free(blob); if (blob.len < 8 or (blob.len - 8) % @sizeOf(abi.Event) != 0) return error.CorruptSession; var f = std.mem.readInt(u64, blob[0..8], .little); if (f >= end) return; var at: usize = 8; while (at < blob.len) : (at += @sizeOf(abi.Event)) { var e: abi.Event = undefined; @memcpy(std.mem.asBytes(&e), blob[at..][0..@sizeOf(abi.Event)]); if (f >= start) try out.append(s.gpa, e); if (e.kind == .tick) f += 1; if (f >= end) return; } } }, } } /// Session `i`'s summary. pub const Summary = struct { started: u64, ended: u64, frames: u64, stretches: u64, keys: u64 }; pub fn summary(s: *Sessions, i: u64) !Summary { switch (s.db) { .off => return error.NotRecording, inline .memory, .file => |*db| { const R = Read(@TypeOf(db.*)); const ses = try R.session(db, i); return .{ .started = try R.uint(ses, .started), .ended = try R.uint(ses, .ended), .frames = try R.uint(ses, .frames), .stretches = if (try R.list(ses, .stretches)) |l| try l.count() else 0, .keys = if (try R.list(ses, .keys)) |l| try l.count() else 0, }; }, } } fn Read(comptime DB: type) type { return struct { const Map = DB.HashMap(.read_only); const List = DB.ArrayList(.read_only); fn root(db: *DB) !Map { return Map.init(db.rootCursor().readOnly()); } fn session(db: *DB, i: u64) !Map { return at(try list(try root(db), .sessions) orelse return error.NoSuchSession, i); } fn field(map: Map, k: K) !DB.Cursor(.read_only) { return try map.getCursor(key(k)) orelse error.CorruptSession; } fn uint(map: Map, k: K) !u64 { return (try field(map, k)).readUint(); } fn bytes(map: Map, k: K, gpa: std.mem.Allocator) ![]const u8 { return (try field(map, k)).readBytesAlloc(gpa, null); } fn list(map: Map, k: K) !?List { return try List.init(try map.getCursor(key(k)) orelse return null); } fn at(l: List, i: u64) !Map { if (i >= try l.count()) return error.NoSuchEntry; return Map.init(try l.getCursor(@intCast(i)) orelse return error.NoSuchEntry); } }; } test "frames written to a session read back byte for byte, in memory and through a reopened file" { const gpa = std.testing.allocator; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var buf: [std.fs.max_path_bytes]u8 = undefined; const path = try std.fmt.bufPrint(&buf, ".zig-cache/tmp/{s}/t.xitdb", .{&tmp.sub_path}); const layout = "a layout"; for ([_]Mode{ .memory, .file }) |m| { var s = try open(gpa, std.testing.io, m, path, .{ .build = 7, .source = "jj:abc", .layout = layout }, 1000); var live: std.ArrayList(abi.Event) = .empty; defer live.deinit(gpa); // Three seconds at 60 Hz, a move in every third frame; the state is // the frame count, so keyframes differ. var state: u64 = 0; for (0..180) |f| { const t = std.time.ns_per_s + f * std.time.ns_per_s / 60; var frame_events: [2]abi.Event = .{ .{ .kind = .mouse_move, .time = t, .x = @floatFromInt(f) }, abi.Event.tick(t, std.time.ns_per_s / 60) }; const one = if (f % 3 == 0) frame_events[0..] else frame_events[1..]; try s.frame(one, std.mem.asBytes(&state), if (f == 0) .start else if (f == 90) .jump else null); try live.appendSlice(gpa, one); state += 1; } // Two commits of a second each; the third second is pending. try std.testing.expectEqual(@as(u64, 2), s.batches); try s.commit(.none); try std.testing.expectEqual(@as(u64, 2), s.keys); var back: std.ArrayList(abi.Event) = .empty; defer back.deinit(gpa); try s.events(s.session, 0, 0, 180, &back); try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(live.items), std.mem.sliceAsBytes(back.items)); // From the jump's keyframe: frame 90 on, its state 90. const k = (try s.keyBefore(s.session, .frame, 100)).?; try std.testing.expectEqual(@as(u64, 1), k); const loaded = try s.load(s.session, k); defer loaded.free(gpa); try std.testing.expectEqual(@as(u64, 90), loaded.frame); try std.testing.expectEqual(@as(u64, 90), std.mem.bytesToValue(u64, loaded.state)); try std.testing.expectEqualStrings(layout, loaded.layout); try std.testing.expectEqual(@as(u64, 7), loaded.build); back.clearRetainingCapacity(); try s.events(s.session, loaded.batch, 90, 92, &back); try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(live.items[120..][0..3]), std.mem.sliceAsBytes(back.items)); try std.testing.expectEqual(@as(?u64, 0), try s.keyBefore(s.session, .frame, 0)); if (m == .memory) { s.deinit(); continue; } s.close(std.mem.asBytes(&state), 2000); // Reopened: the first session ended, its frames all there; a new one. var again = try open(gpa, std.testing.io, .file, path, .{ .build = 8, .source = "", .layout = layout }, 3000); defer again.deinit(); try std.testing.expectEqual(@as(u64, 1), again.session); const first = try again.summary(0); try std.testing.expectEqual(@as(u64, 180), first.frames); try std.testing.expectEqual(@as(u64, 2000), first.ended); back.clearRetainingCapacity(); try again.events(0, 0, 0, 180, &back); try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(live.items), std.mem.sliceAsBytes(back.items)); } }