//! The disk engine: a query pane's Grep, and the file listing its Find //! scores, run on a worker thread away from the loop (lifted from the //! stream prototype). ripgrep when `rg` is on PATH (`rg --json --sort path //! --fixed-strings --smart-case`, `rg --files --sort path`: it knows //! .gitignore and is 5-15x faster on a real tree), else pardes's own sorted //! walk with fs.zig's skips. Rows come back in batches, each stamped with //! the generation it was started for; a new query bumps the generation and //! cancels the worker, and the loop drops any batch that is not the live //! generation's (the race test proves it). A typed Grep starts once the //! typing pauses (`debounce_ns`); a worker stops at `soft_cap` hits and //! waits there for `More`. Embedded in Pardes as `stream`: one search on //! disk at a time, the query pane `target` names. //! //! Hosts with workers perform the `search` effect (host_io.Host.search): //! `start` runs `work` on io.concurrent, posting through the shell's //! inbox. A host without (a test, a board) runs it on the loop (`runHere`), //! its cap a hard stop. const Stream = @This(); const std = @import("std"); const pardes = @import("pardes.zig"); const config = @import("config.zig"); const filesystem = @import("fs.zig"); const look = @import("look.zig"); const panes = @import("panes.zig"); const syntax = @import("syntax.zig"); const Pardes = pardes.Pardes; pub const Kind = enum { grep, files }; pub const Engine = enum { rg, walk }; pub const State = enum { idle, waiting, searching, capped, done, stopped, failed }; /// A keystroke's Grep starts when no other has come for this long. pub const debounce_ns = 80 * std.time.ns_per_ms; /// Hits (a Grep's lines, a listing's files) a worker sends before it waits /// for More. pub const soft_cap = 2000; /// A listing of files keeps going this far before it waits: the names are /// scored on the loop, the cap is for a runaway tree. pub const files_cap = 200_000; /// A worker's batch goes once its first row is this old, or it is this big; /// the first batch goes at its first file with a hit. const batch_ns = 16 * std.time.ns_per_ms; const batch_bytes = 256 * 1024; /// A bigger file is not searched. const max_file_bytes = 16 << 20; /// The walk goes no deeper than this many directories below its root. const max_walk_depth = 16; /// What a worker sends: rows as fs.zig wrote them (`path:line:col-end /// text`, a listing's `path`), relative to the search's directory. Owned by /// the message until taken; the core copies what it keeps. pub const Batch = struct { gen: u32, rows: []u8 = &.{}, /// Directories it might not open. skipped: u32 = 0, engine: Engine = .walk, /// The worker waits at its soft cap until More. capped: bool = false, end: ?End = null, /// A parse's own time on the worker (`parse_ns`). took_ns: u64 = 0, /// A parse's answer (`parseFor`): the declarations of pane `serial`'s /// text at `revision`. parsed: ?struct { serial: u32, revision: u32 } = null, decls: []syntax.ContextDeclaration = &.{}, /// A paint's answer (`paintAhead`): the colours of pane `serial`'s text /// at `revision` from byte `start`, one syntax.Syn a byte. painted: ?struct { serial: u32, revision: u32, start: usize } = null, styles: []u8 = &.{}, pub const End = enum { done, failed }; pub fn deinit(b: *const Batch, gpa: std.mem.Allocator) void { gpa.free(b.rows); gpa.free(b.decls); gpa.free(b.styles); } }; /// A text a worker works on, copied from the core: a pane's text parsed /// for its declarations (`parseFor`: the rows of a `/` group under them, /// and Symbols lists them, once it is done; the loop never parses a big /// text, a 1.2 MB file is seconds in Debug), or its colours painted /// (`paintAhead`, `paintRows`). pub const TextJob = struct { gpa: std.mem.Allocator, /// What its answer is made with: the shell's, which frees a batch. out: std.mem.Allocator, gen: u32 = 0, serial: u32, revision: u32, /// Where `source` starts in the text. start: usize, path: []u8, source: []u8, /// Set when it is no longer wanted: the work ends early. stop: std.atomic.Value(bool) = .init(false), /// A job holding its own copies of `path` and `source`, its `gen` the /// caller's to set; null when they cannot be had. fn make(gpa: std.mem.Allocator, serial: u32, revision: u32, from: usize, path: []const u8, source: []const u8) ?*TextJob { const job = gpa.create(TextJob) catch return null; const path_copy = gpa.dupe(u8, path) catch { gpa.destroy(job); return null; }; const source_copy = gpa.dupe(u8, source) catch { gpa.free(path_copy); gpa.destroy(job); return null; }; job.* = .{ .gpa = gpa, .out = gpa, .serial = serial, .revision = revision, .start = from, .path = path_copy, .source = source_copy }; return job; } pub fn deinit(job: *TextJob) void { job.gpa.free(job.path); job.gpa.free(job.source); job.gpa.destroy(job); } }; const TextTask = struct { // The worker the host started for it, live: a replica has none. pub const replica = .{ .authority = .{ "future", "io" } }; job: *TextJob, future: ?std.Io.Future(void) = null, io: ?std.Io = null, }; /// A worker joined, the turn out for it as for any blocking call (it may /// be reading through a mount of this very session): canceled, or waited /// for once it has sent its last. fn join(future: std.Io.Future(void), io: std.Io, how: enum { cancel, wait }) void { pardes.turn.yield(); var f = future; if (how == .cancel) f.cancel(io) else f.await(io); pardes.turn.back(); } /// A paint's generation has the top bit, so it can never be a search's or /// a parse's (those count from 0 in `gen`, and it would take two billion /// searches to reach it). const paint_bit: u32 = 1 << 31; /// A query rows' paint's has the next bit too: counted apart from the /// paints a view's frames ask for (`rows_count`), which are its own. const rows_bit: u32 = 1 << 30; const max_paint_bytes = 512 << 10; /// Where a worker hands a batch: false when it was turned away (the inbox /// closed, the wait for room canceled), and the worker stops. pub const Sink = *const fn (ctx: ?*anyopaque, batch: Batch) bool; /// One search, the worker's: copied from the core when it starts. pub const Job = struct { gpa: std.mem.Allocator, gen: u32, kind: Kind, pattern: []u8, /// Rows name files relative to this, the query pane's directory, which /// is also the one root searched. base: []u8, /// Forced (tests); else rg when there is one. engine: ?Engine, /// Hits it may send before it waits; More raises it and wakes it. Run /// on the loop (no worker) it is a hard stop. cap: std.atomic.Value(u32) = .init(soft_cap), waits: bool = true, pub fn deinit(job: *Job) void { const gpa = job.gpa; gpa.free(job.pattern); gpa.free(job.base); gpa.destroy(job); } }; const Task = struct { job: *Job, future: std.Io.Future(void), io: std.Io }; /// What the replay digest leaves out (src/walk.zig): the workers' timings, /// and the colours painted ahead of a view for its frames, which is the /// render's own prefetch (a replica that draws nothing paints none). pub const replica = .{ // The worker a search runs on, which the host starts. .authority = .{"task"}, .local = .{ "parse_ns", "paint", "paint_retired", "paint_count", "ahead", "ahead_for", "painted_ahead", "painted_on_loop", "ahead_seen", "ahead_ask", }, }; gen: u32 = 0, state: State = .idle, kind: Kind = .grep, /// The query pane the rows are for, by serial. target: u32 = 0, pattern: std.ArrayList(u8) = .empty, dir: std.ArrayList(u8) = .empty, /// A typed query's start, once the pause runs out (step). due_ns: ?u64 = null, task: ?Task = null, /// Forced engine (tests); null, rg when it runs. engine: ?Engine = null, used: Engine = .walk, skipped: u32 = 0, /// Stale batches dropped (the race test counts them). dropped: usize = 0, /// The parse running or about to (one at a time), its generation, and the /// one asked for while it ran (pane serial and revision), started after it. parse: ?TextTask = null, parse_gen: u32 = 0, parse_next: ?struct { serial: u32, revision: u32 } = null, /// What the last parse cost the worker. parse_ns: u64 = 0, /// The paint running (one at a time), its generation's count, and the /// painted stretch it gave, kept until the view gets there (tree_sitter_gpa). paint: ?TextTask = null, /// One told to stop and not waited for: joined when its answer comes (a /// paint of a long line runs on in its query; waiting for it here held the /// loop for half a second). No new paint starts while it is out. paint_retired: ?TextTask = null, paint_count: u32 = 0, rows_count: u32 = 0, /// The paint a frame asked for, its worker started once the frame is /// presented (`askAhead`): starting one is a thread, not a frame's to wait. ahead_ask: ?u32 = null, ahead: []u8 = &.{}, ahead_for: struct { serial: u32 = 0, revision: u32 = 0, start: usize = 0 } = .{}, /// Stretches a view took painted ahead, and painted on the loop itself /// (the tests count them). painted_ahead: u32 = 0, painted_on_loop: u32 = 0, /// The view the last paint was asked from, and which way it was going. ahead_seen: struct { serial: u32 = 0, scroll: usize = 0, dir: i2 = 1 } = .{}, /// A file's colours for query rows being worked out (paintRows), one at a /// time, its own task beside the scroll's. ponytail: one file at a time, /// the next asked when it lands (Query.recolor); a worker per file if a /// Grep of hundreds of files colours too slowly. rows_paint: ?TextTask = null, /// The files (a hash of path and revision) asked since the last Grep /// began: each asked once, so a file whose colours never fit its rows /// (changed on disk since) is not painted again and again. ponytail: a /// ring of 256; the oldest forgotten. rows_asked: [256]u64 = @splat(0), rows_asked_at: u8 = 0, /// Rows asked for their colours while a 9P write was served: asked again /// on the next step (Query.produceAll). rows_later: bool = false, pub fn deinit(s: *Stream, gpa: std.mem.Allocator) void { s.cancelTask(); s.cancelParse(); s.cancelPaint(); s.pattern.deinit(gpa); s.dir.deinit(gpa); } /// The parse's worker told to stop and joined: short, its progress /// callback hears the flag. pub fn cancelParse(s: *Stream) void { const t = s.parse orelse return; s.parse = null; t.job.stop.store(true, .release); if (t.future) |f| join(f, t.io.?, .cancel); t.job.deinit(); } /// The paints' workers told to stop and joined (the session's end). pub fn cancelPaint(s: *Stream) void { for ([_]*?TextTask{ &s.paint, &s.paint_retired, &s.rows_paint }) |slot| { const t = slot.* orelse continue; slot.* = null; t.job.stop.store(true, .release); if (t.future) |f| join(f, t.io.?, .cancel); t.job.deinit(); } } /// The painted stretch kept for pane `serial`'s text at `revision`, if it /// covers bytes `need_start..need_end`: handed over (the caller owns it, /// tree_sitter_gpa), with where it starts. pub fn takeAhead(p: *Pardes, serial: u32, revision: u32, need_start: usize, need_end: usize) ?struct { styles: []u8, start: usize } { const s = &p.stream; const a = s.ahead_for; if (s.ahead.len == 0 or a.serial != serial or a.revision != revision) return null; if (need_start < a.start or need_end > a.start + s.ahead.len) return null; defer s.ahead = &.{}; return .{ .styles = s.ahead, .start = a.start }; } /// The colours of the text a scroll is heading into, worked out on a /// worker before it gets there, as a game streams in the level ahead of /// the player: refreshHighlights paints only the rows a view shows (and a /// view's worth either side), so a scroll past them used to parse the next /// stretch on the frame (6-58 ms in Debug on a 1.6 MB file, a hitch every /// few pages). Now it finds them painted already, and only a jump past /// what was foreseen (ge, a Look) paints on the frame. /// ponytail: one paint at a time, for the pane last scrolled, in the way it /// went; a pane per worker if two scroll at once. /// /// Asks a worker to paint bytes `start..end` of pane `serial`'s text ahead /// of its view, unless that is painted or being painted already. Only with /// workers, and only for a grammar a worker can paint (syntax.readyApart). pub fn paintAhead(p: *Pardes, serial: u32, revision: u32, path: []const u8, content: []const u8, from: usize, to: usize) void { const s = &p.stream; if (p.host.vtable.search == null or p.fs.serving or to <= from or to > content.len) return; const a = s.ahead_for; if (s.ahead.len != 0 and a.serial == serial and a.revision == revision and from >= a.start and to <= a.start + s.ahead.len) return; // A stretch bigger than this (a few lines of a minified file) is not // painted ahead: the loop paints what it shows of it. if (to - from > max_paint_bytes) return; if (s.paint) |t| { // One under way for the same text that reaches into the stretch // asked for is let finish: held keys move the stretch every frame, // and a paint restarted every frame would never land. if (t.job.serial == serial and t.job.revision == revision and t.job.start < to and t.job.start + t.job.source.len > from) return; // Else it is told to stop and left to end on its own, once. if (s.paint_retired != null) return; t.job.stop.store(true, .release); s.paint_retired = t; s.paint = null; } if (!syntax.readyApart(path)) return; const job = TextJob.make(p.gpa, serial, revision, from, path, content[from..to]) orelse return; s.paint_count +%= 1; job.gen = paint_bit | (s.paint_count & ~paint_bit); s.paint = .{ .job = job }; // Asked of the host once the frame is presented, not queued as an // effect of the session's: a replica that draws nothing asks for none. s.ahead_ask = job.gen; } /// The worker for the paint the last frame asked for: the host starts it /// after the present (Pardes.pump), off the keystroke's way to the screen. pub fn askAhead(p: *Pardes) void { const gen = p.stream.ahead_ask orelse return; p.stream.ahead_ask = null; if (p.host.vtable.search) |f| f(p.host.ctx, gen); } /// A file's colours for the query rows that show its lines, the whole /// file painted on a worker (rows coloured as the file's language colours /// them, as Callstack's are): the text of the pane that shows it, else read /// from disk there. Kept in the locations cache when it lands, where the /// rows find them (Query.Shades). Not without workers (the rows stay /// plain), nor for a file bigger than `max_rows_paint_bytes`, nor twice for /// one revision of a file. pub fn paintRows(p: *Pardes, path: []const u8) void { const s = &p.stream; if (s.rows_paint != null or p.host.vtable.search == null) return; // Asked while a 9P write is served (a script's Grep): not now, as no // paint is (the write would wait for it to settle); the next step asks. if (p.fs.serving) { s.rows_later = true; return; } if (!syntax.readyApart(path)) return; var content: ?[]const u8 = null; var revision: u32 = 0; for (p.panes) |slot| if (slot) |pane| if (pane.file) |f| if (f.output == null and std.mem.eql(u8, f.path, path)) { content = f.content; revision = f.revision; break; }; if (content) |c| if (c.len > max_rows_paint_bytes) return; const key = std.hash.Wyhash.hash(revision, path) | 1; if (std.mem.indexOfScalar(u64, &s.rows_asked, key) != null) return; const job = TextJob.make(p.gpa, 0, revision, 0, path, content orelse "") orelse return; s.rows_asked[s.rows_asked_at] = key; s.rows_asked_at +%= 1; s.rows_count +%= 1; job.gen = paint_bit | rows_bit | (s.rows_count & ~(paint_bit | rows_bit)); s.rows_paint = .{ .job = job }; p.emit(.{ .search = .{ .gen = job.gen } }); } /// The biggest file whose colours query rows are painted from. const max_rows_paint_bytes = 4 << 20; fn paintWork(io: std.Io, job: *TextJob, ctx: ?*anyopaque, sink: Sink) void { // A file's colours for query rows (paintRows, serial 0) with no text // given: read here, off the loop, and sent back with the colours, the // text they are of. const read: []u8 = if (job.serial == 0 and job.source.len == 0) std.Io.Dir.cwd().readFileAlloc(io, job.path, job.out, .limited(max_rows_paint_bytes)) catch @constCast(&[_]u8{}) else @constCast(&[_]u8{}); const source = if (read.len > 0) read else job.source; // Stopped or failed, it still answers (with nothing), so the loop // knows to join it. const styles: []u8 = syntax.highlightApart(job.out, job.path, source, &job.stop) catch @constCast(&[_]u8{}); const text: []u8 = if (job.serial != 0 or read.len > 0) read else job.out.dupe(u8, job.source) catch @constCast(&[_]u8{}); _ = sink(ctx, .{ .gen = job.gen, .rows = text, .styles = styles, .painted = .{ .serial = job.serial, .revision = job.revision, .start = job.start }, .end = .done }); } /// A paint's answer: its task joined, its stretch kept for when the view /// gets there, if the text is still what was painted. fn receivePaint(p: *Pardes, b: *const Batch) void { const s = &p.stream; const painted = b.painted.?; // A file's colours for query rows: kept where the rows find them, and // the rows coloured from them (which asks for the next file). if (s.rows_paint) |t| if (t.job.gen == b.gen) { s.rows_paint = null; if (t.future) |f| join(f, t.io.?, .wait); defer t.job.deinit(); p.locations_cache.put(p.tree_sitter_gpa, t.job.path, b.rows, b.styles) catch {}; panes.Query.recolor(p); p.needs_frame = true; return; }; // Its task joined: it has sent its last, so the join is a return away. for ([_]*?TextTask{ &s.paint, &s.paint_retired }) |slot| if (slot.*) |t| if (t.job.gen == b.gen) { slot.* = null; if (t.future) |f| join(f, t.io.?, .wait); t.job.deinit(); // A retired one's answer is for a stretch nobody wants now. if (slot == &s.paint_retired) return; }; const id = p.paneBySerial(painted.serial) orelse return; const f = p.panes[id].?.file orelse return; if (f.revision != painted.revision or b.styles.len == 0) return; const copy = p.tree_sitter_gpa.dupe(u8, b.styles) catch return; if (s.ahead.len > 0) p.tree_sitter_gpa.free(s.ahead); s.ahead = copy; s.ahead_for = .{ .serial = painted.serial, .revision = painted.revision, .start = painted.start }; // The next frame looks again: the view may be there already. p.panes[id].?.file.?.syntax_dirty = true; } /// Pane `id`'s declarations asked for, as its text is now, unless a parse of /// that text is already on its way. With workers, on one; without (a test), /// here only for a small text, else not at all (the rows stay ungrouped). pub fn parseFor(p: *Pardes, id: usize) void { const s = &p.stream; const pane = p.panes[id] orelse return; const f = pane.file orelse return; if (!syntax.supportsPath(f.path) or f.content.len == 0) return; if (s.parse) |t| { if (t.job.serial == pane.serial and t.job.revision == f.revision) return; // A parse of a text that has changed since: told to stop; this one // after it. t.job.stop.store(true, .release); s.parse_next = .{ .serial = pane.serial, .revision = f.revision }; return; } const workers = p.host.vtable.search != null and !p.fs.serving; if (!workers and f.content.len > 64 << 10) return; const job = TextJob.make(p.gpa, pane.serial, f.revision, 0, f.path, f.content) orelse return; s.gen +%= 1; s.parse_gen = s.gen; job.gen = s.parse_gen; s.parse = .{ .job = job }; if (workers) return p.emit(.{ .search = .{ .gen = s.parse_gen } }); // Here and now: a small text, a host with no workers. var got: Collect = .{ .gpa = p.gpa }; defer got.batches.deinit(p.gpa); parseWork(if (comptime pardes.hosted) std.Io.Threaded.global_single_threaded.io() else std.Io.failing, job, &got, Collect.take); for (got.batches.items) |*b| { receive(p, b); b.deinit(p.gpa); } } fn parseWork(io: std.Io, job: *TextJob, ctx: ?*anyopaque, sink: Sink) void { const began = std.Io.Clock.awake.now(io); const decls: []syntax.ContextDeclaration = syntax.declarationsApart(job.out, job.path, job.source, &job.stop) catch @constCast(&[_]syntax.ContextDeclaration{}); const took: u64 = @intCast(@max(0, began.durationTo(std.Io.Clock.awake.now(io)).nanoseconds)); _ = sink(ctx, .{ .gen = job.gen, .decls = decls, .parsed = .{ .serial = job.serial, .revision = job.revision }, .end = .done, .took_ns = took }); } fn now() u64 { if (comptime !filesystem.platform_has_fs) return 0; const t = std.Io.Clock.awake.now(std.Io.Threaded.global_single_threaded.io()); return @intCast(@max(0, t.nanoseconds)); } /// Joins the worker, if one runs: a cancel interrupts its waits and reads, /// so this is short. pub fn cancelTask(s: *Stream) void { const t = s.task orelse return; s.task = null; join(t.future, t.io, .cancel); t.job.deinit(); } /// A new question for query pane `target`: a Grep of `pattern`, or the /// listing of the files (`.files`), under `dir`. The generation before it /// is over (its worker canceled, its batches dropped when they come); /// `typed`, it starts once the typing pauses, else at the next step. pub fn query(p: *Pardes, target: u32, kind: Kind, pattern: []const u8, dir: []const u8, how: enum { typed, now }) void { const s = &p.stream; s.cancelTask(); s.gen +%= 1; s.kind = kind; s.target = target; s.pattern.clearRetainingCapacity(); s.pattern.appendSlice(p.gpa, pattern) catch return; s.dir.clearRetainingCapacity(); s.dir.appendSlice(p.gpa, dir) catch return; s.skipped = 0; s.state = .waiting; s.due_ns = if (how == .typed) p.now_ns + debounce_ns else p.now_ns; if (kind == .grep) s.rows_asked = @splat(0); } /// Over: the worker canceled, nothing more of it lands. pub fn stop(s: *Stream) bool { if (s.state != .waiting and s.state != .searching and s.state != .capped) return false; s.cancelTask(); s.due_ns = null; s.gen +%= 1; s.state = .stopped; return true; } /// More: a worker waiting at its cap goes on for another `soft_cap` hits. pub fn more(s: *Stream) bool { if (s.state != .capped) return false; const t = s.task orelse return false; _ = t.job.cap.fetchAdd(soft_cap, .release); t.io.futexWake(u32, &t.job.cap.raw, 1); s.state = .searching; return true; } /// The `More` word: the query pane's disk search past its cap. pub fn moreFor(p: *Pardes, id: usize) bool { if (!p.stream.more()) return false; if (p.panes[id]) |pane| if (pane.query) |q| { q.capped = false; }; if (p.paneBySerial(p.stream.target)) |tid| if (p.panes[tid].?.query) |q| { q.capped = false; }; return true; } /// Whether query pane `serial`'s rows are still coming. pub fn busyFor(s: *const Stream, serial: u32) bool { return s.target == serial and (s.state == .waiting or s.state == .searching); } /// On each tick: a query whose pause ran out starts, on a worker where the /// host has one, else here and now. pub fn step(p: *Pardes) void { const s = &p.stream; const due = s.due_ns orelse return; if (p.now_ns < due) return; s.due_ns = null; s.state = .searching; // A script's write (whose answer says whether it found anything) and a // host with no workers: done before this returns. if (p.host.vtable.search != null and !p.fs.serving) p.emit(.{ .search = .{ .gen = s.gen } }) else runHere(p, s.gen); } /// When the core next needs a tick for the stream: the end of the pause. pub fn wake(s: *const Stream, next_frame: u64) ?u64 { const due = s.due_ns orelse return null; return @max(due, next_frame); } /// Start generation `gen` on a worker that sends to `sink` (a shell's /// inbox). Called by the shell for the `search` effect, on the loop. pub fn start(io: std.Io, gpa: std.mem.Allocator, p: *Pardes, gen: u32, ctx: ?*anyopaque, sink: Sink) void { const s = &p.stream; // A paint's generation: its own task, its own worker. if (gen & paint_bit != 0) { for ([_]*?TextTask{ &s.paint, &s.rows_paint }) |slot| if (slot.*) |*t| if (t.job.gen == gen and t.future == null) { t.job.out = gpa; t.future = io.concurrent(paintWork, .{ io, t.job, ctx, sink }) catch { const job = t.job; slot.* = null; job.deinit(); return; }; t.io = io; }; return; } // A parse's generation: its own task. if (s.parse) |*t| if (gen == s.parse_gen and t.job.gen == gen and t.future == null) { t.job.out = gpa; t.future = io.concurrent(parseWork, .{ io, t.job, ctx, sink }) catch { const job = t.job; s.parse = null; job.deinit(); return; }; t.io = io; return; }; if (gen != s.gen or s.state != .searching) return; s.cancelTask(); const job = makeJob(gpa, s) catch |err| { s.state = .failed; return p.reportError(p.active, "search", err); }; const future = io.concurrent(work, .{ io, job, ctx, sink }) catch { // No worker to be had: it runs here, on the loop. job.deinit(); return runHere(p, gen); }; s.task = .{ .job = job, .future = future, .io = io }; } fn makeJob(gpa: std.mem.Allocator, s: *const Stream) !*Job { const job = try gpa.create(Job); errdefer gpa.destroy(job); const pattern = try gpa.dupe(u8, s.pattern.items); errdefer gpa.free(pattern); const base = try gpa.dupe(u8, s.dir.items); job.* = .{ .gpa = gpa, .gen = s.gen, .kind = s.kind, .pattern = pattern, .base = base, .engine = s.engine }; if (s.kind == .files) job.cap = .init(files_cap); return job; } /// A host with no workers (a test's, the fallback when none can be had): /// the whole search on the loop, the turn out for it, its batches taken in /// as a worker's would be. Its cap is a hard stop. pub fn runHere(p: *Pardes, gen: u32) void { // A paint with no worker to run it (a shell not threaded yet): let go, // never left to hold its slot; the rows or the view ask again. if (gen & paint_bit != 0) { const s = &p.stream; for ([_]*?TextTask{ &s.paint, &s.rows_paint }) |slot| if (slot.*) |t| if (t.job.gen == gen and t.future == null) { slot.* = null; t.job.deinit(); }; return; } if (comptime !filesystem.platform_has_fs) { p.stream.state = .failed; return; } const s = &p.stream; if (gen != s.gen or s.state != .searching) return; const job = makeJob(p.gpa, s) catch |err| { s.state = .failed; return p.reportError(p.active, "search", err); }; defer job.deinit(); job.waits = false; // Its reads a site of the tape's (a 9P write's Grep or Find runs here): // a replica takes the batches the session got, as it takes a worker's // (Event.search), and reads no file. const batches = pardes.tape.site(.search, p.gpa, searchHere, .{ p.gpa, job }); defer p.gpa.free(batches); for (batches) |*b| { receive(p, b); b.deinit(p.gpa); } } /// The whole search, its batches gpa's. fn searchHere(gpa: std.mem.Allocator, job: *Job) []Batch { var got: Collect = .{ .gpa = gpa }; work(std.Io.Threaded.global_single_threaded.io(), job, &got, Collect.take); return got.batches.toOwnedSlice(gpa) catch { for (got.batches.items) |b| b.deinit(gpa); got.batches.deinit(gpa); return &.{}; }; } const Collect = struct { gpa: std.mem.Allocator, batches: std.ArrayList(Batch) = .empty, fn take(ctx: ?*anyopaque, batch: Batch) bool { const c: *Collect = @ptrCast(@alignCast(ctx.?)); c.batches.append(c.gpa, batch) catch { batch.deinit(c.gpa); return false; }; return true; } }; /// A batch from a worker (`Event.search`), borrowed: the live generation's /// rows go to its query pane (a Grep's lines, a listing's names); any other /// generation's are dropped whole. pub fn receive(p: *Pardes, b: *const Batch) void { const s = &p.stream; if (b.parsed) |parsed| return receiveParse(p, b, parsed.serial, parsed.revision); if (b.painted != null) return receivePaint(p, b); if (b.gen != s.gen or (s.state != .searching and s.state != .capped)) { s.dropped += 1; return; } s.used = b.engine; s.skipped += b.skipped; if (b.capped) s.state = .capped; if (b.end) |end| { s.state = if (end == .failed) .failed else .done; // It sent its last: its join is a return away. if (s.task) |t| { s.task = null; join(t.future, t.io, .wait); t.job.deinit(); } } p.needs_frame = true; const id = p.paneBySerial(s.target) orelse return; const q = p.panes[id].?.query orelse return; q.capped = s.state == .capped; panes.Query.take(p, q, s.kind, b.rows); } /// A parse's answer: its task joined; the declarations kept as the pane's /// (File.State.context_declarations, for its revision) when its text is /// still what was parsed; the query panes of that text told; and a parse /// asked for while it ran started. fn receiveParse(p: *Pardes, b: *const Batch, serial: u32, revision: u32) void { const s = &p.stream; if (s.parse) |t| if (t.job.gen == b.gen) { s.parse = null; if (t.future) |f| join(f, t.io.?, .wait); t.job.deinit(); }; s.parse_ns = b.took_ns; if (p.paneBySerial(serial)) |id| if (p.panes[id].?.file) |*f| if (f.revision == revision and b.decls.len > 0) { if (p.tree_sitter_gpa.dupe(syntax.ContextDeclaration, b.decls)) |copy| { if (f.context_declarations.len > 0) p.tree_sitter_gpa.free(f.context_declarations); f.context_declarations = copy; f.context_revision = revision; panes.Query.declsArrived(p, serial); } else |_| {} }; p.needs_frame = true; if (s.parse_next) |next| { s.parse_next = null; if (p.paneBySerial(next.serial)) |id| parseFor(p, id); } } // ---- the worker ---- const Worker = struct { io: std.Io, gpa: std.mem.Allocator, job: *Job, ctx: ?*anyopaque, sink: Sink, engine: Engine = .walk, exact: bool, rows: std.ArrayList(u8) = .empty, skipped: u32 = 0, total: u32 = 0, sent: bool = false, batch_began: ?std.Io.Timestamp = null, /// One read buffer for every file: grown to the biggest, never 0xAA'd per file. buf: std.ArrayList(u8) = .empty, fn deinit(w: *Worker) void { w.rows.deinit(w.gpa); w.buf.deinit(w.gpa); } fn send(w: *Worker, end: ?Batch.End, capped: bool) error{ Refused, OutOfMemory }!void { const rows = try w.rows.toOwnedSlice(w.gpa); const batch: Batch = .{ .gen = w.job.gen, .rows = rows, .skipped = w.skipped, .engine = w.engine, .capped = capped, .end = end, }; w.skipped = 0; w.sent = true; w.batch_began = null; if (!w.sink(w.ctx, batch)) return error.Refused; } /// After a file with hits: the first batch goes at once (what the user /// sees first), the others once old or big enough. fn maybeSend(w: *Worker) !void { if (w.rows.items.len == 0) return; const began = w.batch_began orelse { w.batch_began = std.Io.Clock.awake.now(w.io); if (!w.sent) try w.send(null, false); return; }; const age = began.durationTo(std.Io.Clock.awake.now(w.io)).nanoseconds; if (age >= batch_ns or w.rows.items.len >= batch_bytes) try w.send(null, false); } /// At the cap, what is gathered goes, and the worker waits for More (a /// cancel ends the wait); on the loop it is the end. fn checkCap(w: *Worker) !bool { var cap = w.job.cap.load(.acquire); if (w.total < cap) return false; if (!w.job.waits) return true; try w.send(null, true); while (w.total >= cap) { try w.io.futexWait(u32, &w.job.cap.raw, cap); cap = w.job.cap.load(.acquire); } return false; } /// Smart case: a pattern with a capital matches exactly, else in any case. fn find(w: *const Worker, hay: []const u8) ?usize { return if (w.exact) std.mem.indexOf(u8, hay, w.job.pattern) else std.ascii.findIgnoreCase(hay, w.job.pattern); } fn grepText(w: *Worker, path: []const u8, text: []const u8) !u32 { var found: u32 = 0; var line: usize = 0; var it = std.mem.splitScalar(u8, text, '\n'); while (it.next()) |raw| { line += 1; if (w.total >= w.job.cap.load(.acquire)) break; const at = w.find(raw) orelse continue; try w.row(path, line, at, at + w.job.pattern.len, raw); found += 1; } return found; } fn row(w: *Worker, path: []const u8, line: usize, from: usize, end: usize, raw: []const u8) !void { const ln = std.mem.trimEnd(u8, raw, " \t\r\n"); var cut = @min(ln.len, 400); while (cut > 0 and cut < ln.len and ln[cut] & 0xc0 == 0x80) cut -= 1; try w.rows.print(w.gpa, "{s}:{d}:{d}{c}{d} {s}\n", .{ path, line, from + 1, config.range_sep, end, ln[0..cut] }); w.total += 1; } /// A listing's file: its name, relative to the base. fn file(w: *Worker, path: []const u8) !bool { try w.rows.print(w.gpa, "{s}\n", .{path}); w.total += 1; try w.maybeSend(); return w.checkCap(); } /// The name a row gives `abs`: under the base, relative to it. fn shown(w: *const Worker, abs: []const u8) []const u8 { const base = std.mem.trimEnd(u8, w.job.base, "/"); if (abs.len > base.len + 1 and std.mem.startsWith(u8, abs, base) and abs[base.len] == '/') return abs[base.len + 1 ..]; if (std.mem.startsWith(u8, abs, "./")) return abs[2..]; return abs; } // ---- our own walk: sorted, depth first, fs.zig's skips ---- fn walk(w: *Worker) !void { var dir = std.Io.Dir.cwd().openDir(w.io, w.job.base, .{ .iterate = true }) catch |err| switch (err) { error.Canceled => return error.Canceled, else => return, }; defer dir.close(w.io); _ = try w.walkDir(dir, "", 0); } /// Whether the cap ended it (on the loop). fn walkDir(w: *Worker, dir: std.Io.Dir, prefix: []const u8, depth: usize) anyerror!bool { try w.io.checkCancel(); var arena: std.heap.ArenaAllocator = .init(w.gpa); defer arena.deinit(); const a = arena.allocator(); const Entry = struct { name: []const u8, kind: std.Io.File.Kind }; var entries: std.ArrayList(Entry) = .empty; var it = dir.iterate(); while (true) { const e = it.next(w.io) catch |err| switch (err) { error.Canceled => return error.Canceled, else => { w.skipped += 1; break; }, } orelse break; try entries.append(a, .{ .name = try a.dupe(u8, e.name), .kind = e.kind }); } std.mem.sort(Entry, entries.items, {}, struct { fn lt(_: void, x: Entry, y: Entry) bool { return std.mem.lessThan(u8, x.name, y.name); } }.lt); for (entries.items) |e| { const path = if (prefix.len == 0) e.name else try std.fmt.allocPrint(a, "{s}/{s}", .{ prefix, e.name }); switch (e.kind) { .directory => { if (depth + 1 >= max_walk_depth or e.name[0] == '.') continue; if (for (filesystem.find_skip) |skip| { if (std.mem.eql(u8, e.name, skip)) break true; } else false) continue; var sub = dir.openDir(w.io, e.name, .{ .iterate = true }) catch |err| switch (err) { error.Canceled => return error.Canceled, else => { w.skipped += 1; continue; }, }; defer sub.close(w.io); if (try w.walkDir(sub, path, depth + 1)) return true; }, .file => switch (w.job.kind) { .grep => if (try w.grepFile(dir, e.name, path)) return true, .files => if (try w.file(path)) return true, }, else => {}, } } return false; } fn grepFile(w: *Worker, dir: std.Io.Dir, name: []const u8, path: []const u8) !bool { const f = dir.openFile(w.io, name, .{}) catch |err| switch (err) { error.Canceled => return error.Canceled, else => return false, }; defer f.close(w.io); const len = f.length(w.io) catch |err| switch (err) { error.Canceled => return error.Canceled, else => return false, }; // ponytail: a file past 16 MiB is passed over unsaid; a chunked // read, as fs.grep's, would search it. if (len > max_file_bytes) return false; try w.buf.resize(w.gpa, @intCast(len)); const n = f.readPositionalAll(w.io, w.buf.items, 0) catch |err| switch (err) { error.Canceled => return error.Canceled, else => return false, }; const text = w.buf.items[0..n]; if (std.mem.indexOfScalar(u8, text[0..@min(text.len, 1024)], 0) != null) return false; const found = try w.grepText(path, text); if (found == 0) return false; try w.maybeSend(); return w.checkCap(); } // ---- ripgrep: a child on the worker, parsed, never shown ---- /// error.NoRipgrep when there is none to run: the walk does it. fn ripgrep(w: *Worker) !void { var argv: std.ArrayList([]const u8) = .empty; defer argv.deinit(w.gpa); switch (w.job.kind) { .grep => { try argv.appendSlice(w.gpa, &.{ "rg", "--json", "--sort", "path", "--fixed-strings", "--smart-case", "--no-messages", "--" }); try argv.append(w.gpa, w.job.pattern); }, .files => try argv.appendSlice(w.gpa, &.{ "rg", "--files", "--sort", "path", "--no-messages" }), } // As selection_pipe's children: this thread's mask is the child's. const none = std.posix.sigemptyset(); var kept: std.posix.sigset_t = undefined; std.posix.sigprocmask(std.posix.SIG.SETMASK, &none, &kept); var child = std.process.spawn(w.io, .{ .argv = argv.items, .cwd = if (w.job.base.len == 0) .inherit else .{ .path = w.job.base }, .stdin = .ignore, .stdout = .pipe, .stderr = .ignore, }) catch |err| { std.posix.sigprocmask(std.posix.SIG.SETMASK, &kept, null); return if (err == error.Canceled) error.Canceled else error.NoRipgrep; }; std.posix.sigprocmask(std.posix.SIG.SETMASK, &kept, null); w.engine = .rg; // Canceled, capped on the loop, failed: it is killed and reaped. defer if (child.id != null) child.kill(w.io); var line: std.ArrayList(u8) = .empty; defer line.deinit(w.gpa); var json_arena: std.heap.ArenaAllocator = .init(w.gpa); defer json_arena.deinit(); var file_hits: u32 = 0; var file_path: std.ArrayList(u8) = .empty; defer file_path.deinit(w.gpa); // Cut at the cap (on the loop), rg is left to the kill. var cut = false; var chunk: [64 * 1024]u8 = undefined; read: while (true) { const n = child.stdout.?.readStreaming(w.io, &.{&chunk}) catch |err| switch (err) { error.EndOfStream => break, error.Canceled => return error.Canceled, else => return error.ReadFailed, }; if (n == 0) break; var at: usize = 0; while (std.mem.indexOfScalarPos(u8, chunk[0..n], at, '\n')) |nl| { try line.appendSlice(w.gpa, chunk[at..nl]); at = nl + 1; defer line.clearRetainingCapacity(); if (w.job.kind == .files) { // a dot-file is passed over, as the walk passes over // dot-directories if (std.mem.startsWith(u8, std.fs.path.basename(line.items), ".")) continue; if (try w.file(w.shown(line.items))) { cut = true; break :read; } continue; } _ = json_arena.reset(.retain_capacity); const value = std.json.parseFromSliceLeaky(std.json.Value, json_arena.allocator(), line.items, .{}) catch continue; const obj = if (value == .object) value.object else continue; const kind = (obj.get("type") orelse continue); if (kind != .string) continue; const data = if (obj.get("data")) |d| (if (d == .object) d.object else continue) else continue; if (std.mem.eql(u8, kind.string, "begin")) { file_hits = 0; file_path.clearRetainingCapacity(); const text = jsonText(data.get("path")) orelse continue; try file_path.appendSlice(w.gpa, w.shown(text)); } else if (std.mem.eql(u8, kind.string, "match")) { if (file_path.items.len == 0 or w.total >= w.job.cap.load(.acquire)) continue; const text = jsonText(data.get("lines")) orelse continue; const number = data.get("line_number") orelse continue; if (number != .integer) continue; var from: usize = 0; var end: usize = w.job.pattern.len; if (data.get("submatches")) |subs| if (subs == .array and subs.array.items.len > 0 and subs.array.items[0] == .object) { const first = subs.array.items[0].object; if (first.get("start")) |v| if (v == .integer) { from = @intCast(v.integer); }; if (first.get("end")) |v| if (v == .integer) { end = @intCast(v.integer); }; }; try w.row(file_path.items, @intCast(number.integer), from, end, text); file_hits += 1; } else if (std.mem.eql(u8, kind.string, "end")) { if (file_hits == 0) continue; file_hits = 0; try w.maybeSend(); // At the cap rg is left blocked on its full pipe: its // own backpressure, until More or a cancel. if (try w.checkCap()) { cut = true; break :read; } } } try line.appendSlice(w.gpa, chunk[at..n]); } if (!cut and child.id != null) { _ = child.wait(w.io) catch |err| if (err == error.Canceled) return error.Canceled; child.id = null; } } }; /// rg's `{"text": …}` (a name or line that is not UTF-8 comes as bytes, /// and is passed over). fn jsonText(v: ?std.json.Value) ?[]const u8 { const o = v orelse return null; if (o != .object) return null; const t = o.object.get("text") orelse return null; return if (t == .string) t.string else null; } fn work(io: std.Io, job: *Job, ctx: ?*anyopaque, sink: Sink) void { var exact = false; for (job.pattern) |c| exact = exact or std.ascii.isUpper(c); var w: Worker = .{ .io = io, .gpa = job.gpa, .job = job, .ctx = ctx, .sink = sink, .exact = exact }; defer w.deinit(); const ran = if ((job.engine orelse .rg) == .rg) w.ripgrep() else error.NoRipgrep; ran catch |err| switch (err) { error.Canceled, error.Refused => return, error.NoRipgrep => { w.engine = .walk; w.walk() catch |walk_err| switch (walk_err) { error.Canceled, error.Refused => return, else => { w.send(.failed, false) catch {}; return; }, }; }, else => { w.send(.failed, false) catch {}; return; }, }; w.send(.done, false) catch {}; } // ---- tests ---- const host_io = @import("host_io.zig"); const testing = std.testing; /// The shell's half, in a test: perform the `search` effect on a real /// worker, hand what it posts to the core. const TestShell = struct { q: host_io.Inbox, p: ?*Pardes = null, fn init() TestShell { return .{ .q = .{ .gpa = testing.allocator, .lsp_allocator = testing.allocator, .io = testing.io } }; } /// A shell's search: a worker started for the generation, as the tty's. const vtable: pardes.Host.VTable = .{ .search = struct { fn f(ctx: ?*anyopaque, gen: u32) void { const t: *TestShell = @ptrCast(@alignCast(ctx.?)); start(testing.io, testing.allocator, t.p.?, gen, &t.q, host_io.Inbox.pushSearch); } }.f }; fn attach(t: *TestShell, p: *Pardes) void { t.p = p; p.host = .{ .ctx = t, .vtable = &vtable }; } fn pump(t: *TestShell, p: *Pardes) void { // What the last frame asked for, as Pardes.pump asks after a present. askAhead(p); while (p.nextEffect()) |e| switch (e) { .search => p.perform(e), else => {}, }; var batch: host_io.Inbox.Batch = .{}; t.q.takeInto(&batch); for (batch.slice()) |m| switch (m) { .search => |b| { p.update(.{ .search = b }); b.deinit(testing.allocator); }, else => m.deinit(testing.allocator, testing.allocator), }; } /// Pump until the search is over (or `limit_ms` passes: a failure). fn finish(t: *TestShell, p: *Pardes, limit_ms: i64) !void { const until = std.Io.Clock.awake.now(testing.io).toMilliseconds() + limit_ms; while (p.stream.state == .searching or p.stream.state == .waiting) { if (std.Io.Clock.awake.now(testing.io).toMilliseconds() > until) return error.Timeout; if (p.stream.state == .waiting) p.update(.tick); t.pump(p); testing.io.sleep(.fromMicroseconds(200), .awake) catch {}; } } }; /// A worker that never heard of the cancel: it pushes rows of a stale /// generation for as long as the live one streams. const Rogue = struct { q: *host_io.Inbox, gen: u32, stop: std.atomic.Value(bool) = .init(false), done: std.atomic.Value(bool) = .init(false), pushed: usize = 0, fn run(r: *Rogue) void { defer r.done.store(true, .release); while (!r.stop.load(.acquire) and r.pushed < 200) { const rows = testing.allocator.dupe(u8, "d00/f000.txt:1:3-8 stale stale\n") catch return; if (!host_io.Inbox.pushSearch(r.q, .{ .gen = r.gen, .rows = rows })) return; r.pushed += 1; std.Thread.yield() catch {}; } } }; test "Stream: a stale generation's rows never land in the query pane, however two generations race (walk and rg)" { if (comptime !filesystem.platform_has_fs) return; var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); var root_buf: [4096]u8 = undefined; const files = 120; var name_buf: [64]u8 = undefined; for (0..files) |i| { const name = try std.fmt.bufPrint(&name_buf, "d{d:0>2}/f{d:0>3}.txt", .{ i / 10, i }); try tmp.dir.createDirPath(testing.io, std.fs.path.dirname(name).?); try tmp.dir.writeFile(testing.io, .{ .sub_path = name, .data = "a needle line\nsome hay here\nneedle again\nhay hay\nmore hay\n" }); } const root = root_buf[0..try tmp.dir.realPath(testing.io, &root_buf)]; const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 120, .rows = 40 }); defer p.deinit(); const pane = try p.setTestFile("x\n"); p.gpa.free(pane.file.?.path); pane.file.?.path = try std.fs.path.join(p.gpa, &.{ root, "main.txt" }); p.advance(1); var shell: TestShell = .init(); shell.attach(p); defer { p.stream.cancelTask(); shell.q.deinit(); } // The query pane the rows are for: a Grep's. panes.Query.open(p, 0, .grep, "", false); const qid = for (p.panes, 0..) |slot, i| { if (slot) |qp| if (qp.query != null) break i; } else return error.NoQueryPane; const qp = p.panes[qid].?; const q = qp.query.?; var dropped_total: usize = 0; for ([_]Engine{ .walk, .rg }) |engine| { p.stream.engine = engine; for (0..12) |round| { // A starts and runs a little (0, 1 or 2 pumps: its batches in // flight, queued, or taken in), then B replaces it, typed (its // pause run out on the clock) or at once. const a = if (round % 2 == 0) "needle" else "hay"; const b = if (round % 2 == 0) "hay" else "needle"; q.found.clearRetainingCapacity(); q.found_bytes.clearRetainingCapacity(); query(p, qp.serial, .grep, a, root, .now); p.update(.tick); for (0..round % 3) |_| shell.pump(p); var rogue: Rogue = .{ .q = &shell.q, .gen = p.stream.gen }; q.found.clearRetainingCapacity(); q.found_bytes.clearRetainingCapacity(); if (round % 4 == 3) { query(p, qp.serial, .grep, b, root, .typed); try testing.expectEqual(State.waiting, p.stream.state); p.advance(p.now_ns + debounce_ns + std.time.ns_per_ms); } else query(p, qp.serial, .grep, b, root, .now); try testing.expect(p.stream.gen != rogue.gen); const thread = try std.Thread.spawn(.{}, Rogue.run, .{&rogue}); const finished = shell.finish(p, 20_000); // It may be waiting for room: the queue is taken until it ends. rogue.stop.store(true, .release); while (!rogue.done.load(.acquire)) shell.pump(p); thread.join(); shell.pump(p); // whatever it pushed last try finished; try testing.expectEqual(State.done, p.stream.state); // b's lines, every one, and not one of a's nor the rogue's. const per_file: usize = if (std.mem.eql(u8, b, "hay")) 3 else 2; try testing.expectEqual(files * per_file, q.found.items.len); try testing.expect(std.mem.indexOf(u8, q.found_bytes.items, "stale") == null); for (q.found.items) |fd| { const code = q.found_bytes.items[fd.code..][0..fd.code_len]; try testing.expect(std.ascii.findIgnoreCase(code, b) != null); } if (engine == .rg and p.stream.used != .rg) return; // no rg here: the walk was the test dropped_total += p.stream.dropped; p.stream.dropped = 0; } } try testing.expect(dropped_total > 0); } test "Stream: the listing of files for Find on a worker, a cap of hits stops the walk on the loop" { if (comptime !filesystem.platform_has_fs) return; var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); var root_buf: [4096]u8 = undefined; try tmp.dir.createDirPath(testing.io, "src"); try tmp.dir.createDirPath(testing.io, ".git"); for ([_][]const u8{ "src/a.zig", "src/b.zig", "notes.md", ".git/config" }) |name| try tmp.dir.writeFile(testing.io, .{ .sub_path = name, .data = "needle\n" }); const root = root_buf[0..try tmp.dir.realPath(testing.io, &root_buf)]; const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 120, .rows = 40 }); defer p.deinit(); const pane = try p.setTestFile("x\n"); p.gpa.free(pane.file.?.path); pane.file.?.path = try std.fs.path.join(p.gpa, &.{ root, "main.txt" }); p.advance(1); p.stream.engine = .walk; // No host workers: the listing runs on the loop at the next tick. panes.Query.open(p, 0, .find, "", false); p.update(.tick); p.update(.tick); const qid = for (p.panes, 0..) |slot, i| { if (slot) |qp| if (qp.query != null) break i; } else return error.NoQueryPane; const q = p.panes[qid].?.query.?; // the three files, not the dot-directory's try testing.expectEqual(@as(usize, 3), q.names.cands.items.len); try testing.expectEqual(State.done, p.stream.state); } test "Stream: a big text's declarations are parsed on a worker, never the loop, and the rows of its / regroup under them" { if (comptime !filesystem.platform_has_fs) return; const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 120, .rows = 40 }); defer p.deinit(); var text: std.ArrayList(u8) = .empty; defer text.deinit(testing.allocator); for (0..4000) |i| try text.print(testing.allocator, "fn f{d}() void {{\n _ = needle;\n}}\n", .{i}); const pane = try p.setTestFile(text.items); p.gpa.free(pane.file.?.path); pane.file.?.path = try p.gpa.dupe(u8, "/tmp/big.zig"); p.advance(1); var shell: TestShell = .init(); shell.attach(p); defer { p.stream.cancelTask(); p.stream.cancelParse(); shell.q.deinit(); } for ("/needle") |c| p.update(.{ .key = .{ .cp = c, .text = &[_]u8{c} } }); const qid = for (p.panes, 0..) |slot, i| { if (slot) |qp| if (qp.query != null) break i; } else return error.NoQueryPane; const qp = p.panes[qid].?; // Not grouped yet: no parse on the loop; one asked of the worker. try testing.expect(p.stream.parse != null); try testing.expect(std.mem.indexOf(u8, qp.file.?.content, "fn f") == null); const until = std.Io.Clock.awake.now(testing.io).toMilliseconds() + 60_000; while (pane.file.?.context_revision == null or pane.file.?.context_revision.? != pane.file.?.revision) { if (std.Io.Clock.awake.now(testing.io).toMilliseconds() > until) return error.Timeout; shell.pump(p); p.update(.tick); testing.io.sleep(.fromMilliseconds(2), .awake) catch {}; } p.update(.tick); // Now under their declarations: `fn fN() void {` headers. try testing.expect(std.mem.indexOf(u8, qp.file.?.content, "fn f") != null); try testing.expect(p.stream.parse == null); } test "Stream: a scroll's colours are painted ahead on a worker, so PageDown after PageDown parses nothing on the frame" { if (comptime !filesystem.platform_has_fs or !syntax.enabled) return; const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 120, .rows = 40 }); defer p.deinit(); var text: std.ArrayList(u8) = .empty; defer text.deinit(testing.allocator); for (0..6000) |i| try text.print(testing.allocator, "fn f{d}() void {{\n _ = \"needle\";\n}}\n", .{i}); const pane = try p.setTestFile(text.items); p.gpa.free(pane.file.?.path); pane.file.?.path = try p.gpa.dupe(u8, "/tmp/big.zig"); p.advance(1); var shell: TestShell = .init(); shell.attach(p); defer { p.stream.cancelTask(); p.stream.cancelParse(); p.stream.cancelPaint(); shell.q.deinit(); } var arena: std.heap.ArenaAllocator = .init(testing.allocator); defer arena.deinit(); _ = try p.render(arena.allocator()); const first = p.stream.painted_on_loop; for (0..20) |_| { // The worker has had its time (a frame's work at a held key's pace // is all it needs; here it is waited for, so the test is not a race). const until = std.Io.Clock.awake.now(testing.io).toMilliseconds() + 60_000; while (p.stream.paint != null) { if (std.Io.Clock.awake.now(testing.io).toMilliseconds() > until) return error.Timeout; shell.pump(p); testing.io.sleep(.fromMilliseconds(1), .awake) catch {}; } p.update(.{ .key = .{ .cp = pardes.Key.page_down } }); _ = arena.reset(.retain_capacity); _ = try p.render(arena.allocator()); // Coloured: the rows shown have their colours. const f = &pane.file.?; try testing.expect(f.highlights.len > 0 and f.highlight_start <= f.line_starts[f.scroll]); } try testing.expect(p.stream.painted_on_loop == first); try testing.expect(p.stream.painted_ahead >= 5); } test "Stream: a Grep stops at its 2000 lines and waits; More in the query pane's tag goes on" { if (comptime !filesystem.platform_has_fs) return; var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); var root_buf: [4096]u8 = undefined; var name_buf: [64]u8 = undefined; var text: std.ArrayList(u8) = .empty; defer text.deinit(testing.allocator); for (0..20) |_| try text.appendSlice(testing.allocator, "a needle here\n"); for (0..150) |i| try tmp.dir.writeFile(testing.io, .{ .sub_path = try std.fmt.bufPrint(&name_buf, "f{d:0>3}.txt", .{i}), .data = text.items }); const root = root_buf[0..try tmp.dir.realPath(testing.io, &root_buf)]; const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 120, .rows = 40 }); defer p.deinit(); const pane = try p.setTestFile("x\n"); p.gpa.free(pane.file.?.path); pane.file.?.path = try std.fs.path.join(p.gpa, &.{ root, "main.txt" }); p.advance(1); var shell: TestShell = .init(); shell.attach(p); defer { p.stream.cancelTask(); shell.q.deinit(); } p.stream.engine = .walk; panes.Query.open(p, 0, .grep, "needle", false); const qid = for (p.panes, 0..) |slot, i| { if (slot) |qp| if (qp.query != null) break i; } else return error.NoQueryPane; const qp = p.panes[qid].?; p.advance(p.now_ns + debounce_ns + std.time.ns_per_ms); const until = std.Io.Clock.awake.now(testing.io).toMilliseconds() + 20_000; while (p.stream.state != .capped) { if (std.Io.Clock.awake.now(testing.io).toMilliseconds() > until) return error.Timeout; p.update(.tick); shell.pump(p); testing.io.sleep(.fromMicroseconds(200), .awake) catch {}; } p.update(.tick); try testing.expectEqual(@as(usize, soft_cap), qp.query.?.found.items.len); try testing.expect(qp.query.?.capped); try testing.expectEqualStrings(" More Del", @import("tagline.zig").curTail(qp)); pardes.exec.runBuiltin(p, .More, qid, "", null); try shell.finish(p, 20_000); p.update(.tick); try testing.expectEqual(State.done, p.stream.state); try testing.expectEqual(@as(usize, 3000), qp.query.?.found.items.len); try testing.expect(!qp.query.?.capped); } test "Stream: a Grep's rows recoloured are made again, never written, and a dump of them dropped by an edit loads to its digest" { if (comptime !filesystem.platform_has_fs) return; if (!syntax.enabled or !syntax.readyApart("c.zig")) return; var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); // Rows enough that a digest keeps a word for them (walk's blob_min). var text: std.ArrayList(u8) = .empty; defer text.deinit(testing.allocator); for (0..80) |i| try text.print(testing.allocator, "pub const value_{d}: u32 = {d};\n", .{ i, i }); try tmp.dir.writeFile(testing.io, .{ .sub_path = "c.zig", .data = text.items }); var root_buf: [4096]u8 = undefined; const root = root_buf[0..try tmp.dir.realPath(testing.io, &root_buf)]; const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 120, .rows = 40 }); defer p.deinit(); const pane = try p.setTestFile(text.items); p.gpa.free(pane.file.?.path); pane.file.?.path = try std.fs.path.join(p.gpa, &.{ root, "c.zig" }); var shell: TestShell = .init(); shell.attach(p); defer { p.stream.cancelTask(); p.stream.cancelPaint(); shell.q.deinit(); } panes.Query.open(p, 0, .grep, "value", false); const qid = for (p.panes, 0..) |slot, i| { if (slot) |qp| if (qp.query != null) break i; } else return error.NoQueryPane; const f = &p.panes[qid].?.file.?; const coloured = struct { fn count(rows: []const @import("locations.zig").Row) usize { var n: usize = 0; for (rows) |row| n += @intFromBool(row.kind == .match and row.location_end > 0 and std.mem.indexOfNone(u8, row.colors, &.{0}) != null); return n; } }.count; var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); const until = now() + 10 * std.time.ns_per_s; // The rows, plain, digested (their word kept) before their colours land. while (now() < until and f.location_rows.len < 80) { p.advance(now()); shell.pump(p); p.update(.tick); testing.io.sleep(.fromMilliseconds(2), .awake) catch {}; } try testing.expect(f.location_rows.len >= 80 and coloured(f.location_rows) == 0); _ = pardes.walk.digest(Pardes, p); const plain = f.location_rows.ptr; while (now() < until and (coloured(f.location_rows) < 80 or p.stream.rows_paint != null)) { p.advance(now()); shell.pump(p); p.update(.tick); _ = arena.reset(.retain_capacity); _ = try p.render(arena.allocator()); testing.io.sleep(.fromMilliseconds(2), .awake) catch {}; } try testing.expectEqual(@as(usize, 80), coloured(f.location_rows)); // Made again: a Debug build's digest finds no blob written in place // (the plain rows' kept word is theirs), and the rows are new ones. _ = pardes.walk.digest(Pardes, p); try testing.expect(f.location_rows.ptr != plain); // An edit drops them: a join's load of the session is it, digest for // digest. panes.File.setContent(p, f, try p.gpa.dupe(u8, "value\n")); const bytes = try pardes.dump.dumpBytes(p); defer p.gpa.free(bytes); const q = try pardes.dump.load(testing.allocator, p.opts, bytes, null); defer q.deinit(); try testing.expectEqual(pardes.walk.digest(panes.File.State, f), pardes.walk.digest(panes.File.State, &q.panes[qid].?.file.?)); } test "Stream: a Grep's rows are coloured as their file's language colours them, the file painted on a worker, the match tint on top" { if (comptime !filesystem.platform_has_fs) return; if (!syntax.enabled or !syntax.readyApart("c.zig")) return; var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); const text = "const std = @import(\"std\");\npub fn value() u32 {\n return 42; // value\n}\n"; try tmp.dir.writeFile(testing.io, .{ .sub_path = "c.zig", .data = text }); var root_buf: [4096]u8 = undefined; const root = root_buf[0..try tmp.dir.realPath(testing.io, &root_buf)]; const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 120, .rows = 40 }); defer p.deinit(); // the file open in the pane that asks: its text painted, not the disk's const pane = try p.setTestFile(text); p.gpa.free(pane.file.?.path); pane.file.?.path = try std.fs.path.join(p.gpa, &.{ root, "c.zig" }); var shell: TestShell = .init(); shell.attach(p); defer { p.stream.cancelTask(); p.stream.cancelPaint(); shell.q.deinit(); } panes.Query.open(p, 0, .grep, "value", false); const qid = for (p.panes, 0..) |slot, i| { if (slot) |qp| if (qp.query != null) break i; } else return error.NoQueryPane; const f = &p.panes[qid].?.file.?; var halt: std.atomic.Value(bool) = .init(false); const own = try syntax.highlightApart(testing.allocator, "c.zig", text, &halt); defer testing.allocator.free(own); var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); const until = now() + 10 * std.time.ns_per_s; var coloured: usize = 0; while (now() < until) { p.advance(now()); shell.pump(p); p.update(.tick); // a frame each, as a shell draws: the rows painted before their colours land _ = arena.reset(.retain_capacity); _ = try p.render(arena.allocator()); coloured = 0; for (f.location_rows) |row| if (row.kind == .match and row.location_end > 0 and std.mem.indexOfNone(u8, row.colors, &.{0}) != null) { coloured += 1; }; if (coloured == 2 and p.stream.rows_paint == null) break; testing.io.sleep(.fromMilliseconds(2), .awake) catch {}; } try testing.expectEqual(@as(usize, 2), coloured); // and painted: the frame's colours are the rows' _ = arena.reset(.retain_capacity); _ = try p.render(arena.allocator()); // each row's colours its line's, byte for byte; the match tinted over them var lines = std.mem.splitScalar(u8, f.content, '\n'); for (f.location_rows) |row| { const line = lines.next().?; if (row.kind != .match or row.location_end == 0) continue; const ls = @import("modal.zig").lineStartOffset(text, row.at.line - 1); try testing.expectEqualSlices(u8, own[ls + row.source_column ..][0..row.colors.len], row.colors); const at = @intFromPtr(line.ptr) - @intFromPtr(f.content.ptr) + row.code_start - f.highlight_start; try testing.expectEqualSlices(u8, row.colors, f.highlights[at..][0..row.colors.len]); const dec = panes.Output.decorateRow(f.output, line, row); try testing.expectEqualStrings("value", line[dec.mark_start..dec.mark_end]); } }