summaryrefslogtreecommitdiff
path: root/src/walk.zig
diff options
context:
space:
mode:
Diffstat (limited to 'src/walk.zig')
-rw-r--r--src/walk.zig55
1 files changed, 55 insertions, 0 deletions
diff --git a/src/walk.zig b/src/walk.zig
index 2dc0a721..d0d20e8a 100644
--- a/src/walk.zig
+++ b/src/walk.zig
@@ -112,6 +112,10 @@ fn hashBlob(comptime E: type, items: []const E, h: *Hasher) void {
break :kept if (b.len == items.len) b.word else null;
} else null;
if (!rehash) if (kept) |w| break :word w;
+ if (E == u8) if (unhashed_out) |list| if (kept == null) {
+ list.append(std.heap.c_allocator, items) catch {};
+ break :word 0;
+ };
in_blob = false;
defer in_blob = true;
var one = Hasher.init(0);
@@ -130,6 +134,37 @@ fn hashBlob(comptime E: type, items: []const E, h: *Hasher) void {
h.update(std.mem.asBytes(&word));
}
+/// Where `unhashed` notes the texts a digest would hash for want of a word.
+threadlocal var unhashed_out: ?*std.ArrayListUnmanaged([]const u8) = null;
+
+/// The blob texts a digest of `v` would hash now, having no word yet, onto
+/// `out` (the C heap's): a walk with no blob hashed, which `makeWords` then
+/// makes on another thread. A replica's join (src/replica.zig) does that
+/// first: the session's step never hashes a text it never digested before.
+pub fn unhashed(comptime T: type, v: *const T, out: *std.ArrayListUnmanaged([]const u8)) void {
+ unhashed_out = out;
+ defer unhashed_out = null;
+ _ = digest(T, v);
+}
+
+/// The words of `texts` (`unhashed`'s), made on any thread: they must stay
+/// as they are meanwhile (pinned, memory.zig), and the next digest takes
+/// them.
+pub fn makeWords(texts: []const []const u8) void {
+ for (texts) |items| {
+ // As hashBlob makes it (hashSlice's length, then the bytes).
+ var one = Hasher.init(0);
+ one.update(std.mem.asBytes(&items.len));
+ one.update(items);
+ const w = one.final();
+ lockBlobs();
+ defer unlockBlobs();
+ const got = blob_words.getOrPut(std.heap.page_allocator, @intFromPtr(items.ptr)) catch continue;
+ if (!got.found_existing) got.value_ptr.* = .{ .len = items.len, .word = w };
+ blob_count.store(blob_words.count(), .release);
+ }
+}
+
pub fn digest(comptime T: type, v: *const T) u64 {
var h = Hasher.init(0);
hash(T, v, &h);
@@ -801,6 +836,9 @@ fn tableOf(comptime types: []const type) []const u8 {
return Once.bytes;
}
+/// A blob's bytes a dump left out (`Dump.deferred`): they go at `at`.
+pub const Deferred = struct { at: usize, bytes: []const u8 };
+
/// Writes a dump of `roots`' values, one `put` each in order.
pub fn Dump(comptime roots: []const type) type {
return struct {
@@ -811,6 +849,13 @@ pub fn Dump(comptime roots: []const type) type {
out: std.Io.Writer.Allocating,
memo: std.AutoHashMapUnmanaged(Key, u32) = .empty,
shared: u32 = 0,
+ /// Set: a `.blob`'s bytes (of `blob_min` and up) are not written but
+ /// noted, where they go and what they are; the dump with them put
+ /// back in place is the dump. A replica's join streams them from
+ /// its writer thread (src/replica.zig), the session's step paying
+ /// for the rest alone.
+ deferred: ?*std.ArrayListUnmanaged(Deferred) = null,
+ in_blob: bool = false,
const Key = struct { addr: usize, len: usize, ty: u32 };
@@ -893,6 +938,13 @@ pub fn Dump(comptime roots: []const type) type {
.slice => {
const E = @typeInfo(T).pointer.child;
if (try d.seen(T, @intFromPtr(v.ptr), v.len)) return;
+ if (comptime flat(E)) if (d.in_blob) if (d.deferred) |later| {
+ const bytes = std.mem.sliceAsBytes(v.*);
+ if (bytes.len >= blob_min) {
+ try w.writeAll(std.mem.asBytes(&@as(u64, v.len)));
+ return later.append(d.gpa, .{ .at = d.out.writer.end, .bytes = bytes });
+ }
+ };
try d.items(E, v.*);
},
.array => {
@@ -905,6 +957,9 @@ pub fn Dump(comptime roots: []const type) type {
const packed_struct = comptime @typeInfo(T).@"struct".layout == .@"packed";
inline for (comptime dumpFields(T)) |name| {
const F = @FieldType(T, name);
+ const was = d.in_blob;
+ d.in_blob = was or comptime blob(T, name);
+ defer d.in_blob = was;
if (comptime cutArray(T, name)) {
const all = &@field(v, name);
try d.items(@typeInfo(F).array.child, all[0..@min(all.len, lengthIn(v, comptime lengthOf(T, name).?))]);