diff options
| -rw-r--r-- | docs/typ/builtins-gui.json | 1 | ||||
| -rw-r--r-- | docs/typ/builtins-tty.json | 1 | ||||
| -rw-r--r-- | src/Names.zig | 85 | ||||
| -rw-r--r-- | src/Query.zig | 190 | ||||
| -rw-r--r-- | src/Stream.zig | 1177 | ||||
| -rw-r--r-- | src/builtins.zig | 8 | ||||
| -rw-r--r-- | src/config.zig | 1 | ||||
| -rw-r--r-- | src/detached/server.zig | 10 | ||||
| -rw-r--r-- | src/detached/wire.zig | 6 | ||||
| -rw-r--r-- | src/gui/gui.zig | 16 | ||||
| -rw-r--r-- | src/host_io.zig | 35 | ||||
| -rw-r--r-- | src/macos.zig | 2 | ||||
| -rw-r--r-- | src/pardes.zig | 23 | ||||
| -rw-r--r-- | src/syntax.zig | 45 | ||||
| -rw-r--r-- | src/tagline.zig | 4 | ||||
| -rw-r--r-- | src/tty/tty.zig | 28 |
16 files changed, 1462 insertions, 170 deletions
diff --git a/docs/typ/builtins-gui.json b/docs/typ/builtins-gui.json index a502836e..c238955c 100644 --- a/docs/typ/builtins-gui.json +++ b/docs/typ/builtins-gui.json @@ -81,6 +81,7 @@ {"name": "Messages", "leader": "SPC h m", "also": [], "arg": false, "scope": "root", "setting": false, "choices": null, "doc": "Open the messages shown so far."}, {"name": "Mini", "leader": null, "also": [], "arg": true, "scope": "pane", "setting": false, "choices": null, "doc": "Open a minimap of the file in braille dots."}, {"name": "Mode", "leader": null, "also": [], "arg": false, "scope": "pane", "setting": false, "choices": null, "doc": "Cycle the pane's input modes: terminals add raw tty mode to normal/insert."}, + {"name": "More", "leader": null, "also": [], "arg": false, "scope": "pane", "setting": false, "choices": null, "doc": "A query pane's Grep stopped at its 2000 lines goes on for 2000 more."}, {"name": "Motion", "leader": null, "also": [], "arg": true, "scope": "root", "setting": true, "choices": "off, crisp, smooth, bouncy, playful", "doc": "How animations move: off, crisp, smooth, bouncy or playful."}, {"name": "Mount", "leader": null, "also": [], "arg": true, "scope": "root", "setting": false, "choices": null, "doc": "Mount another pardes's 9P tree under /n/<name>, given its dial address."}, {"name": "Msg", "leader": null, "also": [], "arg": true, "scope": "pane", "setting": false, "choices": null, "doc": "Show its argument on this pane's message row."}, diff --git a/docs/typ/builtins-tty.json b/docs/typ/builtins-tty.json index 7e9ba678..791eef63 100644 --- a/docs/typ/builtins-tty.json +++ b/docs/typ/builtins-tty.json @@ -71,6 +71,7 @@ {"name": "Messages", "leader": "SPC h m", "also": [], "arg": false, "scope": "root", "setting": false, "choices": null, "doc": "Open the messages shown so far."}, {"name": "Mini", "leader": null, "also": [], "arg": true, "scope": "pane", "setting": false, "choices": null, "doc": "Open a minimap of the file in braille dots."}, {"name": "Mode", "leader": null, "also": [], "arg": false, "scope": "pane", "setting": false, "choices": null, "doc": "Cycle the pane's input modes: terminals add raw tty mode to normal/insert."}, + {"name": "More", "leader": null, "also": [], "arg": false, "scope": "pane", "setting": false, "choices": null, "doc": "A query pane's Grep stopped at its 2000 lines goes on for 2000 more."}, {"name": "Motion", "leader": null, "also": [], "arg": true, "scope": "root", "setting": true, "choices": "off, crisp, smooth, bouncy, playful", "doc": "How animations move: off, crisp, smooth, bouncy or playful."}, {"name": "Mount", "leader": null, "also": [], "arg": true, "scope": "root", "setting": false, "choices": null, "doc": "Mount another pardes's 9P tree under /n/<name>, given its dial address."}, {"name": "Msg", "leader": null, "also": [], "arg": true, "scope": "pane", "setting": false, "choices": null, "doc": "Show its argument on this pane's message row."}, diff --git a/src/Names.zig b/src/Names.zig index 48447f56..0c8ba075 100644 --- a/src/Names.zig +++ b/src/Names.zig @@ -1,5 +1,5 @@ //! Names a query pane picks from by a fuzzy query: the files under a -//! directory (walked a slice a frame), the recent files, the open panes, +//! directory (listed by the disk engine, Stream.zig), the recent files, the open panes, //! a file's declarations. Each keystroke scores them again with a small //! fzf-style scorer (subsequence, Smith-Waterman with affine gaps, bonuses //! at the starts of words and path segments), a frame's budget at a time, @@ -12,7 +12,6 @@ const std = @import("std"); const pardes = @import("pardes.zig"); const panes = @import("panes.zig"); const config = @import("config.zig"); -const filesystem = @import("fs.zig"); const Pardes = pardes.Pardes; /// The candidates' rows, back to back, no newlines. @@ -31,9 +30,6 @@ pending: std.ArrayList(u32) = .empty, pending_at: usize = 0, query_buf: [max_query]u8 = undefined, query_len: u8 = 0, -/// The files walk, while it lasts (Find in a query pane until the disk -/// engine has it). -walk: ?Walk = null, /// Bumped whenever `hits` change: the rows are made again only then. gen: u32 = 0, @@ -59,30 +55,10 @@ pub const max_query = 64; /// A candidate is scored over its first this many bytes. const max_text = 512; const max_cands = 200_000; -/// Entries a walk reads a frame: ~1-3 ms warm. -const walk_slice = 2000; -const walk_max_depth = 16; /// A keystroke's scoring, and a frame's: what is left waits a frame. const score_budget_ns = 4 * std.time.ns_per_ms; -const Walk = struct { - root: std.Io.Dir, - walker: std.Io.Dir.SelectiveWalker, - done: bool = false, - skipped: u32 = 0, - - fn deinit(w: *Walk) void { - const io = std.Io.Threaded.global_single_threaded.io(); - // The walker closes a directory only as it leaves it: the ones it - // is still in (all but the root, which is ours) are closed here. - if (w.walker.stack.items.len > 1) for (w.walker.stack.items[1..]) |*item| item.iter.reader.dir.close(io); - w.walker.deinit(); - w.root.close(io); - } -}; - pub fn deinit(n: *Names, gpa: std.mem.Allocator) void { - if (n.walk) |*w| w.deinit(); n.bytes.deinit(gpa); n.lower.deinit(gpa); n.cands.deinit(gpa); @@ -93,8 +69,6 @@ pub fn deinit(n: *Names, gpa: std.mem.Allocator) void { /// No candidates, no walk: for a source filled again. pub fn reset(n: *Names) void { - if (n.walk) |*w| w.deinit(); - n.walk = null; n.bytes.clearRetainingCapacity(); n.lower.clearRetainingCapacity(); n.cands.clearRetainingCapacity(); @@ -136,62 +110,9 @@ pub fn baseOf(path: []const u8) u16 { return @intCast(@min(slash + 1, max_text)); } -/// The files under `dir`, walked from now on a slice a frame (`walkSlice`): -/// the first slice now, so a small tree is whole before the first frame. -pub fn startWalk(n: *Names, gpa: std.mem.Allocator, dir: []const u8) !void { - if (comptime !filesystem.platform_has_fs) return error.NoFilesystem; - const io = std.Io.Threaded.global_single_threaded.io(); - pardes.turn.yield(); - defer pardes.turn.back(); - var dir_handle = try std.Io.Dir.cwd().openDir(io, dir, .{ .iterate = true }); - errdefer dir_handle.close(io); - n.walk = .{ .root = dir_handle, .walker = try dir_handle.walkSelectively(gpa) }; - n.walkSlice(gpa); -} - -/// One frame's slice of a walk: up to `walk_slice` entries, files kept, -/// what a Find skips skipped and dot-directories too. -pub fn walkSlice(n: *Names, gpa: std.mem.Allocator) void { - const w = if (n.walk) |*w| w else return; - if (w.done) return; - const io = std.Io.Threaded.global_single_threaded.io(); - pardes.turn.yield(); - defer pardes.turn.back(); - var steps: usize = 0; - walk: while (steps < walk_slice) : (steps += 1) { - const e = (w.walker.next(io) catch { - w.skipped += 1; - continue; - }) orelse { - w.done = true; - break; - }; - if (e.kind != .directory) { - n.add(gpa, e.path, 0, e.path.len, baseOf(e.path), 0) catch { - w.done = true; - break; - }; - if (n.cands.items.len >= max_cands) { - w.done = true; - break; - } - continue; - } - if (e.depth() >= walk_max_depth or e.basename[0] == '.') continue; - for (filesystem.find_skip) |s| if (std.mem.eql(u8, e.basename, s)) continue :walk; - w.walker.enter(io, e) catch { - w.skipped += 1; - }; - } -} - -pub fn walking(n: *const Names) bool { - return if (n.walk) |w| !w.done else false; -} - -/// Still walking, or still scoring what it has: it wants the next frame. +/// Still scoring what it has: it wants the next frame. pub fn busy(n: *const Names) bool { - return n.walking() or n.scored < n.cands.items.len or n.pending_at < n.pending.items.len; + return n.scored < n.cands.items.len or n.pending_at < n.pending.items.len; } /// A new query: scored again, from what the last one matched when it only diff --git a/src/Query.zig b/src/Query.zig index fbce95b2..48378a18 100644 --- a/src/Query.zig +++ b/src/Query.zig @@ -46,6 +46,7 @@ const colors = @import("colors.zig"); const memory = @import("memory.zig"); const Names = @import("Names.zig"); const Places = @import("Places.zig"); +const Stream = @import("Stream.zig"); const Surface = @import("surface.zig").Surface; const Pardes = pardes.Pardes; const Pane = panes.Pane; @@ -152,16 +153,17 @@ window: struct { lo: usize = 0, hi: usize = 0 } = .{}, /// made every third, the frame's time going to the finding. waited: u8 = 0, names: Names = .{}, -/// A Grep's lines, found a slice a frame (Query.grepSlice). +/// A Grep's lines, as the disk engine sends them (Stream.zig, `take`). found: std.ArrayList(Found) = .empty, found_bytes: std.ArrayList(u8) = .empty, -grep_next: usize = 0, grep_gen: u32 = 0, /// The language server's rows for Refs, Def, Impl, Diagnostics, as it gave /// them (`path:line:col text`, gpa), filtered by what follows the word. lsp_rows: []u8 = &.{}, /// The +Preview under this pane, by serial; 0, none. preview: u32 = 0, +/// Its disk search waits at its cap: the tag says More. +capped: bool = false, /// A reused pane's frozen rows when a new query began in it: Esc brings /// them back. saved: ?Saved = null, @@ -387,7 +389,7 @@ pub fn busy(p: *const Pardes) bool { const pane = slot orelse continue; const q = pane.query orelse continue; if (q.frozen) continue; - if (q.names.busy() or (q.source == .grep and q.grep_next < q.names.cands.items.len) or q.made_rev != q.seen_rev) return true; + if (q.names.busy() or p.stream.busyFor(pane.serial) or q.made_rev != q.seen_rev) return true; } return false; } @@ -482,32 +484,29 @@ fn produce(p: *Pardes, id: usize, tick: bool) void { makeAllRows(p, id) catch |err| p.reportError(id, "search", err); }, .find, .recent, .panes, .symbols => { - if (tick) q.names.walkSlice(p.gpa); if (changed) q.names.typed(p.gpa, a.pat) else if (q.names.busy()) q.names.rescore(p.gpa); if (q.names.gen == q.made_gen and !changed) return; q.made_gen = q.names.gen; makeNameRows(p, id, a.pat) catch |err| p.reportError(id, "search", err); }, .grep => { + const dir = std.fs.path.dirname(f.path) orelse "/"; if (changed) { q.found.clearRetainingCapacity(); q.found_bytes.clearRetainingCapacity(); - q.grep_next = 0; q.grep_gen +%= 1; + q.current = null; + // On the disk engine's worker, once the typing pauses. + if (a.pat.len > 0) Stream.query(p, rp.serial, .grep, a.pat, dir, .typed) else if (p.stream.target == rp.serial) { + _ = p.stream.stop(); + } + feedGrepPlaces(p, q, rp.serial, a.pat); } - if (a.pat.len == 0) { - if (changed) makeGrepRows(p, id) catch {}; - return; - } - if (tick or changed) { - q.names.walkSlice(p.gpa); - grepSlice(p, q, rp, a.pat); - } - const gen: u64 = (@as(u64, q.grep_gen) << 32) | q.found.items.len; - if (gen == q.made_gen and !changed and !(q.grep_next >= q.names.cands.items.len and q.more)) return; + const searching = p.stream.target == rp.serial and p.stream.state != .done and p.stream.state != .stopped and p.stream.state != .failed and p.stream.state != .idle; + const gen: u64 = (@as(u64, q.grep_gen) << 32) | (q.found.items.len << 1) | @intFromBool(searching); + if (gen == q.made_gen and !changed) return; q.made_gen = gen; - makeGrepRows(p, id) catch |err| p.reportError(id, "search", err); - if (changed or q.current == null) feedGrepPlaces(p, q, rp.serial, a.pat); + makeGrepRows(p, id, false) catch |err| p.reportError(id, "search", err); }, .refs, .defs, .impls, .diagnostics => { if (changed and q.lsp_rows.len == 0) { @@ -544,11 +543,12 @@ fn startSource(p: *Pardes, id: usize, q: *Query, source: Source, owner_id: ?usiz q.names.reset(); q.found.clearRetainingCapacity(); q.found_bytes.clearRetainingCapacity(); - q.grep_next = 0; const dir = std.fs.path.dirname(rp.file.?.path) orelse "/"; const gpa = p.gpa; switch (source) { - .find, .grep => q.names.startWalk(gpa, dir) catch |err| p.reportError(id, "search", err), + // The files under the directory, listed once on the disk engine's + // worker (rg --files, else the walk); each keystroke scores them. + .find => Stream.query(p, rp.serial, .files, "", dir, .now), .recent => { var out: std.Io.Writer.Allocating = .init(gpa); defer out.deinit(); @@ -575,7 +575,11 @@ fn startSource(p: *Pardes, id: usize, q: *Query, source: Source, owner_id: ?usiz const oid = owner_id orelse return; const owner = p.panes[oid].?; const f = if (owner.file) |*file| file else return; - const decls = ownerDecls(p, f, true) orelse return; + const decls = ownerDecls(p, f, false) orelse { + // Parsed on the worker; listed when it is done (declsArrived). + Stream.parseFor(p, oid); + return; + }; for (decls) |d| { const header = f.content[d.start_byte..@min(d.header_end_byte, f.content.len)]; const first = std.mem.trim(u8, header[0 .. std.mem.indexOfScalar(u8, header, '\n') orelse header.len], " \t\r"); @@ -821,6 +825,8 @@ fn makeTextRows(p: *Pardes, id: usize, oid: usize, full: bool) !void { defer out.deinit(p.gpa); var rows: std.ArrayList(locations.Row) = .empty; var budget: usize = if (full) max_rows else live_rows; + // Grouped once a parse of the text exists: asked of the worker now. + if (ownerDecls(p, &owner.file.?, false) == null) Stream.parseFor(p, oid); const at = set.current orelse (if (set.seek) |s| Places.firstFrom(set.hits.items, s) else 0); const from = if (full) 0 else at -| live_rows / 2; q.window = .{ .lo = from, .hi = from }; @@ -927,6 +933,7 @@ fn firstHit(q: *const Query) ?usize { /// `k` from the current row: the hits on the rows before it, and its own. fn countCurrent(q: *Query) void { + if (q.source == .grep) return countGrepCurrent(q); q.k = 0; const c = q.current orelse return; for (q.items.items[0..c]) |it| q.k += if (it.kind == .hit) @max(1, it.n) else 0; @@ -992,7 +999,7 @@ fn makeNameRows(p: *Pardes, id: usize, pat: []const u8) !void { const f = &rp.file.?; f.pager_spans = spans.toOwnedSlice(p.gpa) catch &.{}; q.total = q.names.hits.items.len; - q.more = q.names.busy(); + q.more = q.names.busy() or p.stream.busyFor(rp.serial); // The best is current until a step or a click moves it. q.current = if (shown > 0) 0 else null; countCurrent(q); @@ -1000,70 +1007,30 @@ fn makeNameRows(p: *Pardes, id: usize, pat: []const u8) !void { showTarget(p, id); } -/// A slice of a Grep: the files the walk has found so far read and -/// searched, about 4 ms of them a frame, each line holding the pattern -/// (a literal, smart case: a capital makes it exact) kept. -/// ponytail: on the loop, a file a step; the disk engine's worker (phase -/// 4) takes this over. -fn grepSlice(p: *Pardes, q: *Query, rp: *Pane, pat: []const u8) void { - const started = Places.nowNs(); - const dir = std.fs.path.dirname(rp.file.?.path) orelse "/"; - const fold = for (pat) |c| { - if (std.ascii.isUpper(c)) break false; - } else true; - var low_buf: [256]u8 = undefined; - const needle = if (fold) std.ascii.lowerString(low_buf[0..@min(pat.len, low_buf.len)], pat[0..@min(pat.len, low_buf.len)]) else pat; - while (q.grep_next < q.names.cands.items.len) { - if (q.found.items.len >= grep_cap) break; - const c = q.names.cands.items[q.grep_next]; - q.grep_next += 1; - const rel = q.names.bytes.items[c.start..][0..c.len]; - var pathbuf: [4096 + 8]u8 = undefined; - const path = std.fmt.bufPrint(&pathbuf, "{s}/{s}", .{ std.mem.trimEnd(u8, dir, "/"), rel }) catch continue; - const bytes = blk: { - pardes.turn.yield(); - defer pardes.turn.back(); - break :blk filesystem.read(p, path) catch continue; - }; - defer p.gpa.free(bytes); - if (std.mem.indexOfScalar(u8, bytes[0..@min(bytes.len, 8000)], 0) != null) continue; - var line_no: u32 = 1; - var ls: usize = 0; - while (ls <= bytes.len) { - const le = std.mem.indexOfScalarPos(u8, bytes, ls, '\n') orelse bytes.len; - const ln = bytes[ls..le]; - const at = if (fold) std.ascii.findIgnoreCase(ln, needle) else std.mem.indexOf(u8, ln, needle); - if (at) |col| { - const path_at: u32 = @intCast(q.found_bytes.items.len); - q.found_bytes.appendSlice(p.gpa, rel) catch return; - const code_at: u32 = @intCast(q.found_bytes.items.len); - const code = ln[0..@min(ln.len, 400)]; - q.found_bytes.appendSlice(p.gpa, code) catch return; - q.found.append(p.gpa, .{ .path = path_at, .path_len = @intCast(rel.len), .line = line_no, .col = @intCast(col), .end = @intCast(col + needle.len), .code = code_at, .code_len = @intCast(code.len) }) catch return; - if (q.found.items.len >= grep_cap) break; - } - if (le == bytes.len) break; - ls = le + 1; - line_no += 1; - } - if (started != 0 and Places.nowNs() - started > Places.frame_ns) break; - } -} - /// A Grep keeps this many lines: the rest wait (the disk engine's `More`). pub const grep_cap = 2000; -fn makeGrepRows(p: *Pardes, id: usize) !void { +/// A Grep's rows: each file's name over its lines, while typed a window +/// of `live_rows` around the current one, `full` (frozen) every one sent. +fn makeGrepRows(p: *Pardes, id: usize, full: bool) !void { const rp = p.panes[id].?; const q = rp.query.?; const dir = std.fs.path.dirname(rp.file.?.path) orelse "/"; + const at: usize = if (q.current) |c| (if (c < q.items.items.len) q.items.items[c].hit else 0) else 0; q.items.clearRetainingCapacity(); var out: std.ArrayList(u8) = .empty; defer out.deinit(p.gpa); var rows: std.ArrayList(locations.Row) = .empty; var last_path: []const u8 = ""; var pathbuf: [4096 + 8]u8 = undefined; - for (q.found.items, 0..) |fd, i| { + const lo = if (full) 0 else at -| live_rows / 2; + const hi = if (full) q.found.items.len else @min(q.found.items.len, lo + live_rows); + if (lo > 0) { + var hb: [64]u8 = undefined; + try noteRow(p, q, &out, &rows, std.fmt.bufPrint(&hb, " ... {d} lines before these", .{lo}) catch " ..."); + } + var current: ?usize = null; + for (q.found.items[lo..hi], lo..) |fd, i| { const rel = q.found_bytes.items[fd.path..][0..fd.path_len]; const path = std.fmt.bufPrint(&pathbuf, "{s}/{s}", .{ std.mem.trimEnd(u8, dir, "/"), rel }) catch continue; if (!std.mem.eql(u8, rel, last_path)) { @@ -1072,19 +1039,62 @@ fn makeGrepRows(p: *Pardes, id: usize) !void { try writeRow(p, &out, &rows, rel, .{ .kind = .context, .path = @constCast(path), .at = .{}, .location_end = 0, .code_start = 0 }); } const code = q.found_bytes.items[fd.code..][0..fd.code_len]; + if (i == at) current = q.items.items.len; try q.items.append(p.gpa, .{ .kind = .hit, .hit = i, .n = 1 }); try writeHitRow(p, &out, &rows, 9, path, fd.line, code, fd.col, fd.end, &.{}); } - const walking = q.names.walking() or q.grep_next < q.names.cands.items.len; - if (q.found.items.len >= grep_cap) try noteRow(p, q, &out, &rows, " ... stopped at 2000 lines: type more of it"); + if (hi < q.found.items.len) { + var hb: [64]u8 = undefined; + try noteRow(p, q, &out, &rows, std.fmt.bufPrint(&hb, " ... {d} lines after these", .{q.found.items.len - hi}) catch " ..."); + } + if (p.stream.target == rp.serial and p.stream.state == .capped) try noteRow(p, q, &out, &rows, " ... stopped at 2000 lines: More in the tag goes on"); setRows(p, rp, &out, &rows); q.total = q.found.items.len; - q.more = walking and q.found.items.len < grep_cap; - if (q.current == null or q.current.? >= q.items.items.len) q.current = firstHit(q); - countCurrent(q); + q.more = p.stream.busyFor(rp.serial); + q.current = current orelse firstHit(q); + countGrepCurrent(q); showCurrent(rp); } +/// `k` for a Grep: the current row's line among all the lines sent. +fn countGrepCurrent(q: *Query) void { + q.k = 0; + const c = q.current orelse return; + if (c < q.items.items.len and q.items.items[c].kind == .hit) q.k = q.items.items[c].hit + 1; +} + +/// The disk engine's rows for query pane `q` (Stream.receive): a Grep's +/// lines (`path:line:col-end code`, relative to the pane's directory) kept +/// as found, a listing's names added to what Find scores. +pub fn take(p: *Pardes, q: *Query, kind: Stream.Kind, rows: []const u8) void { + var lines = std.mem.splitScalar(u8, rows, '\n'); + while (lines.next()) |ln| { + if (ln.len == 0) continue; + switch (kind) { + .files => q.names.add(p.gpa, ln, 0, ln.len, Names.baseOf(ln), 0) catch return, + .grep => { + const t = look.parsePathLine(ln); + if (t.at.line == 0 or t.path.len == 0) continue; + const code = if (t.end < ln.len) ln[t.end + 1 ..] else ""; + const path_at: u32 = @intCast(q.found_bytes.items.len); + q.found_bytes.appendSlice(p.gpa, t.path) catch return; + const code_at: u32 = @intCast(q.found_bytes.items.len); + q.found_bytes.appendSlice(p.gpa, code) catch return; + q.found.append(p.gpa, .{ + .path = path_at, + .path_len = @intCast(t.path.len), + .line = @intCast(t.at.line), + .col = @intCast(t.at.col -| 1), + .end = @intCast(if (t.at.end_col > 0) t.at.end_col else t.at.col), + .code = code_at, + .code_len = @intCast(code.len), + }) catch return; + }, + } + } + q.names.gen +%= 1; +} + /// A Grep's hits in each open file are that pane's places: its tints, /// ticks and n/N. fn feedGrepPlaces(p: *Pardes, q: *Query, serial: u32, pat: []const u8) void { @@ -1552,6 +1562,8 @@ pub fn land(p: *Pardes, id: usize, row: ?usize) void { } return; }; + // A Grep's whole list, now it is a result. + if (q.source == .grep) makeGrepRows(p, id, true) catch {}; var buf: [4096 + 64]u8 = undefined; const address = if (meta.at.line > 0) std.fmt.bufPrint(&buf, "{s}:{d}:{d}", .{ meta.path, meta.at.line, meta.at.col }) catch meta.path @@ -1634,8 +1646,26 @@ pub fn cancel(p: *Pardes, id: usize) void { } else p.removePane(id, null) catch {}; } -/// The pane going (Del, its owner's Del): the places it fed go with it. +/// A parse of pane `serial`'s text came back (Stream.receiveParse): its +/// query panes' rows are made again, under the declarations now, and a +/// Symbols query gathers them. +pub fn declsArrived(p: *Pardes, serial: u32) void { + for (p.panes, 0..) |slot, id| { + const rp = slot orelse continue; + const q = rp.query orelse continue; + if (q.owner != serial or q.frozen) continue; + q.made_gen = std.math.maxInt(u64); + if (q.source == .symbols) { + startSource(p, id, q, .symbols, p.paneBySerial(serial)); + if (parse(q.asked)) |a| q.names.typed(p.gpa, a.pat); + } + } +} + +/// The pane going (Del, its owner's Del): the places it fed go with it, +/// and a disk search still running for it. pub fn release(p: *Pardes, pane: *Pane) void { + if (p.stream.target == pane.serial) _ = p.stream.stop(); for (p.panes) |slot| if (slot) |tp| if (tp.places.from == pane.serial) { tp.places.forget(p.gpa); tp.places.clear(p.gpa); @@ -1932,6 +1962,8 @@ fn leader(p: *Pardes, keys: []const u8) void { } fn settle(p: *Pardes) void { + // a typed Grep's pause run out on the clock + p.advance(p.now_ns + Stream.debounce_ns + std.time.ns_per_ms); var frames: usize = 0; while (busy(p) and frames < 2000) : (frames += 1) p.update(.tick); p.update(.tick); diff --git a/src/Stream.zig b/src/Stream.zig new file mode 100644 index 00000000..eb420b42 --- /dev/null +++ b/src/Stream.zig @@ -0,0 +1,1177 @@ +//! The disk engine: a query pane's Grep, and the file listing its Find +//! scores, run on a worker thread away from the loop (lifted from the +//! stream prototype). ripgrep when `rg` is on PATH (`rg --json --sort path +//! --fixed-strings --smart-case`, `rg --files --sort path`: it knows +//! .gitignore and is 5-15x faster on a real tree), else pardes's own sorted +//! walk with fs.zig's skips. Rows come back in batches, each stamped with +//! the generation it was started for; a new query bumps the generation and +//! cancels the worker, and the loop drops any batch that is not the live +//! generation's (the race test proves it). A typed Grep starts once the +//! typing pauses (`debounce_ns`); a worker stops at `soft_cap` hits and +//! waits there for `More`. Embedded in Pardes as `stream`: one search on +//! disk at a time, the query pane `target` names. +//! +//! Hosts with workers perform the `search` effect (host_io.Host.search): +//! `start` runs `work` on io.concurrent, posting through the shell's +//! inbox. A host without (a test, a board) runs it on the loop (`runHere`), +//! its cap a hard stop. +const Stream = @This(); + +const std = @import("std"); +const pardes = @import("pardes.zig"); +const config = @import("config.zig"); +const filesystem = @import("fs.zig"); +const look = @import("look.zig"); +const panes = @import("panes.zig"); +const syntax = @import("syntax.zig"); +const Pardes = pardes.Pardes; + +pub const Kind = enum { grep, files }; +pub const Engine = enum { rg, walk }; +pub const State = enum { idle, waiting, searching, capped, done, stopped, failed }; + +/// A keystroke's Grep starts when no other has come for this long. +pub const debounce_ns = 80 * std.time.ns_per_ms; +/// Hits (a Grep's lines, a listing's files) a worker sends before it waits +/// for More. +pub const soft_cap = 2000; +/// A listing of files keeps going this far before it waits: the names are +/// scored on the loop, the cap is for a runaway tree. +pub const files_cap = 200_000; +/// A worker's batch goes once its first row is this old, or it is this big; +/// the first batch goes at its first file with a hit. +const batch_ns = 16 * std.time.ns_per_ms; +const batch_bytes = 256 * 1024; +/// A bigger file is not searched. +const max_file_bytes = 16 << 20; + +/// What a worker sends: rows as fs.zig wrote them (`path:line:col-end +/// text`, a listing's `path`), relative to the search's directory. Owned by +/// the message until taken; the core copies what it keeps. +pub const Batch = struct { + gen: u32, + rows: []u8 = &.{}, + hits: u32 = 0, + /// Files with a hit in this batch's rows. + files: u32 = 0, + /// Files read since the last batch (rg: all of them, at its end). + scanned: u32 = 0, + /// Directories it might not open. + skipped: u32 = 0, + engine: Engine = .walk, + /// The worker waits at its soft cap until More. + capped: bool = false, + end: ?End = null, + /// The worker's own time, start to end (in the last batch). + took_ns: u64 = 0, + /// A parse's answer (`parseFor`): the declarations of pane `serial`'s + /// text at `revision`. + parsed: ?struct { serial: u32, revision: u32 } = null, + decls: []syntax.ContextDeclaration = &.{}, + + pub const End = enum { done, failed }; + + pub fn deinit(b: *const Batch, gpa: std.mem.Allocator) void { + gpa.free(b.rows); + gpa.free(b.decls); + } +}; + +/// A parse of a pane's text for its declarations, on a worker: the rows of +/// a `/` group under them, and Symbols lists them, once it is done. The +/// loop never parses a big text (a 1.2 MB file is seconds in Debug). +pub const ParseJob = struct { + gpa: std.mem.Allocator, + /// What its answer is made with: the shell's, which frees a batch. + out: std.mem.Allocator, + gen: u32, + serial: u32, + revision: u32, + path: []u8, + content: []u8, + /// Set when it is no longer wanted: the parse ends early. + stop: std.atomic.Value(bool) = .init(false), + + pub fn deinit(job: *ParseJob) void { + job.gpa.free(job.path); + job.gpa.free(job.content); + job.gpa.destroy(job); + } +}; + +const ParseTask = struct { job: *ParseJob, future: ?std.Io.Future(void) = null, io: ?std.Io = null }; + +/// Where a worker hands a batch: false when it was turned away (the inbox +/// closed, the wait for room canceled), and the worker stops. +pub const Sink = *const fn (ctx: ?*anyopaque, batch: Batch) bool; + +/// One search, the worker's: copied from the core when it starts. +pub const Job = struct { + gpa: std.mem.Allocator, + gen: u32, + kind: Kind, + pattern: []u8, + /// Rows name files relative to this, the query pane's directory, which + /// is also the one root searched. + base: []u8, + /// Forced (tests, PARDES_STREAM_ENGINE); else rg when there is one. + engine: ?Engine, + /// Hits it may send before it waits; More raises it and wakes it. Run + /// on the loop (no worker) it is a hard stop. + cap: std.atomic.Value(u32) = .init(soft_cap), + waits: bool = true, + + pub fn deinit(job: *Job) void { + const gpa = job.gpa; + gpa.free(job.pattern); + gpa.free(job.base); + gpa.destroy(job); + } +}; + +const Task = struct { job: *Job, future: std.Io.Future(void), io: std.Io }; + +gen: u32 = 0, +state: State = .idle, +kind: Kind = .grep, +/// The query pane the rows are for, by serial. +target: u32 = 0, +pattern: std.ArrayList(u8) = .empty, +dir: std.ArrayList(u8) = .empty, +/// A typed query's start, once the pause runs out (step). +due_ns: ?u64 = null, +task: ?Task = null, +/// Forced engine (tests); null, rg when it runs. +engine: ?Engine = null, +used: Engine = .walk, +hits: u32 = 0, +files: u32 = 0, +scanned: u32 = 0, +skipped: u32 = 0, +/// Stale batches dropped (the race test counts them). +dropped: usize = 0, +started_ns: u64 = 0, +first_ns: ?u64 = null, +finished_ns: u64 = 0, +worker_ns: u64 = 0, +/// The parse running or about to (one at a time), its generation, and the +/// one asked for while it ran (pane serial and revision), started after it. +parse: ?ParseTask = null, +parse_gen: u32 = 0, +parse_next: ?struct { serial: u32, revision: u32 } = null, +/// What the last parse cost the worker. +parse_ns: u64 = 0, + +pub fn deinit(s: *Stream, gpa: std.mem.Allocator) void { + s.cancelTask(); + s.cancelParse(); + s.pattern.deinit(gpa); + s.dir.deinit(gpa); +} + +/// The parse's worker told to stop and joined: short, its progress +/// callback hears the flag. +pub fn cancelParse(s: *Stream) void { + const t = s.parse orelse return; + s.parse = null; + t.job.stop.store(true, .release); + if (t.future) |f| { + pardes.turn.yield(); + var future = f; + future.cancel(t.io.?); + pardes.turn.back(); + } + t.job.deinit(); +} + +/// Pane `id`'s declarations asked for, as its text is now, unless a parse of +/// that text is already on its way. With workers, on one; without (a test), +/// here only for a small text, else not at all (the rows stay ungrouped). +pub fn parseFor(p: *Pardes, id: usize) void { + const s = &p.stream; + const pane = p.panes[id] orelse return; + const f = pane.file orelse return; + if (!syntax.supportsPath(f.path) or f.content.len == 0) return; + if (s.parse) |t| { + if (t.job.serial == pane.serial and t.job.revision == f.revision) return; + // A parse of a text that has changed since: told to stop; this one + // after it. + t.job.stop.store(true, .release); + s.parse_next = .{ .serial = pane.serial, .revision = f.revision }; + return; + } + const workers = p.host.vtable.search != null and !p.fs.serving; + if (!workers and f.content.len > 64 << 10) return; + const job = p.gpa.create(ParseJob) catch return; + const path = p.gpa.dupe(u8, f.path) catch { + p.gpa.destroy(job); + return; + }; + const content = p.gpa.dupe(u8, f.content) catch { + p.gpa.free(path); + p.gpa.destroy(job); + return; + }; + s.gen +%= 1; + s.parse_gen = s.gen; + job.* = .{ .gpa = p.gpa, .out = p.gpa, .gen = s.parse_gen, .serial = pane.serial, .revision = f.revision, .path = path, .content = content }; + s.parse = .{ .job = job }; + if (workers) return p.emit(.{ .search = .{ .gen = s.parse_gen } }); + // Here and now: a small text, a host with no workers. + var got: Collect = .{ .gpa = p.gpa }; + defer got.batches.deinit(p.gpa); + parseWork(std.Io.Threaded.global_single_threaded.io(), job, &got, Collect.take); + for (got.batches.items) |*b| { + receive(p, b); + b.deinit(p.gpa); + } +} + +fn parseWork(io: std.Io, job: *ParseJob, ctx: ?*anyopaque, sink: Sink) void { + const began = std.Io.Clock.awake.now(io); + const decls: []syntax.ContextDeclaration = syntax.declarationsApart(job.out, job.path, job.content, &job.stop) catch @constCast(&[_]syntax.ContextDeclaration{}); + const took: u64 = @intCast(@max(0, began.durationTo(std.Io.Clock.awake.now(io)).nanoseconds)); + _ = sink(ctx, .{ .gen = job.gen, .decls = decls, .parsed = .{ .serial = job.serial, .revision = job.revision }, .end = .done, .took_ns = took }); +} + +fn now() u64 { + if (comptime !filesystem.platform_has_fs) return 0; + const t = std.Io.Clock.awake.now(std.Io.Threaded.global_single_threaded.io()); + return @intCast(@max(0, t.nanoseconds)); +} + +/// Joins the worker, if one runs: a cancel interrupts its waits and reads, +/// so this is short. The turn goes out for it, as for any blocking call: +/// the worker may be reading through a mount of this very session. +pub fn cancelTask(s: *Stream) void { + const t = s.task orelse return; + s.task = null; + pardes.turn.yield(); + var future = t.future; + future.cancel(t.io); + pardes.turn.back(); + t.job.deinit(); +} + +/// A new question for query pane `target`: a Grep of `pattern`, or the +/// listing of the files (`.files`), under `dir`. The generation before it +/// is over (its worker canceled, its batches dropped when they come); +/// `typed`, it starts once the typing pauses, else at the next step. +pub fn query(p: *Pardes, target: u32, kind: Kind, pattern: []const u8, dir: []const u8, how: enum { typed, now }) void { + const s = &p.stream; + s.cancelTask(); + s.gen +%= 1; + s.kind = kind; + s.target = target; + s.pattern.clearRetainingCapacity(); + s.pattern.appendSlice(p.gpa, pattern) catch return; + s.dir.clearRetainingCapacity(); + s.dir.appendSlice(p.gpa, dir) catch return; + s.hits = 0; + s.files = 0; + s.scanned = 0; + s.skipped = 0; + s.first_ns = null; + s.state = .waiting; + s.due_ns = if (how == .typed) p.now_ns + debounce_ns else p.now_ns; +} + +/// Over: the worker canceled, nothing more of it lands. +pub fn stop(s: *Stream) bool { + if (s.state != .waiting and s.state != .searching and s.state != .capped) return false; + s.cancelTask(); + s.due_ns = null; + s.gen +%= 1; + s.state = .stopped; + s.finished_ns = now(); + return true; +} + +/// More: a worker waiting at its cap goes on for another `soft_cap` hits. +pub fn more(s: *Stream) bool { + if (s.state != .capped) return false; + const t = s.task orelse return false; + _ = t.job.cap.fetchAdd(soft_cap, .release); + t.io.futexWake(u32, &t.job.cap.raw, 1); + s.state = .searching; + return true; +} + +/// The `More` word: the query pane's disk search past its cap. +pub fn moreFor(p: *Pardes, id: usize) bool { + if (!p.stream.more()) return false; + if (p.panes[id]) |pane| if (pane.query) |q| { + q.capped = false; + }; + if (p.paneBySerial(p.stream.target)) |tid| if (p.panes[tid].?.query) |q| { + q.capped = false; + }; + return true; +} + +/// Whether query pane `serial`'s rows are still coming. +pub fn busyFor(s: *const Stream, serial: u32) bool { + return s.target == serial and (s.state == .waiting or s.state == .searching); +} + +/// On each tick: a query whose pause ran out starts, on a worker where the +/// host has one, else here and now. +pub fn step(p: *Pardes) void { + const s = &p.stream; + const due = s.due_ns orelse return; + if (p.now_ns < due) return; + s.due_ns = null; + s.state = .searching; + s.started_ns = now(); + // A script's write (whose answer says whether it found anything) and a + // host with no workers: done before this returns. + if (p.host.vtable.search != null and !p.fs.serving) p.emit(.{ .search = .{ .gen = s.gen } }) else runHere(p, s.gen); +} + +/// When the core next needs a tick for the stream: the end of the pause. +pub fn wake(s: *const Stream, next_frame: u64) ?u64 { + const due = s.due_ns orelse return null; + return @max(due, next_frame); +} + +/// Start generation `gen` on a worker that sends to `sink` (a shell's +/// inbox). Called by the shell for the `search` effect, on the loop. +pub fn start(io: std.Io, gpa: std.mem.Allocator, p: *Pardes, gen: u32, ctx: ?*anyopaque, sink: Sink) void { + const s = &p.stream; + // A parse's generation: its own task. + if (s.parse) |*t| if (gen == s.parse_gen and t.job.gen == gen and t.future == null) { + t.job.out = gpa; + t.future = io.concurrent(parseWork, .{ io, t.job, ctx, sink }) catch { + const job = t.job; + s.parse = null; + job.deinit(); + return; + }; + t.io = io; + return; + }; + if (gen != s.gen or s.state != .searching) return; + s.cancelTask(); + const job = makeJob(gpa, s) catch |err| { + s.state = .failed; + return p.reportError(p.active, "search", err); + }; + const future = io.concurrent(work, .{ io, job, ctx, sink }) catch { + // No worker to be had: it runs here, on the loop. + job.deinit(); + return runHere(p, gen); + }; + s.task = .{ .job = job, .future = future, .io = io }; +} + +fn makeJob(gpa: std.mem.Allocator, s: *const Stream) !*Job { + const job = try gpa.create(Job); + errdefer gpa.destroy(job); + const pattern = try gpa.dupe(u8, s.pattern.items); + errdefer gpa.free(pattern); + const base = try gpa.dupe(u8, s.dir.items); + job.* = .{ .gpa = gpa, .gen = s.gen, .kind = s.kind, .pattern = pattern, .base = base, .engine = s.engine }; + if (s.kind == .files) job.cap = .init(files_cap); + return job; +} + +/// A host with no workers (a test's, the fallback when none can be had): +/// the whole search on the loop, the turn out for it, its batches taken in +/// as a worker's would be. Its cap is a hard stop. +pub fn runHere(p: *Pardes, gen: u32) void { + if (comptime !filesystem.platform_has_fs) { + p.stream.state = .failed; + return; + } + const s = &p.stream; + if (gen != s.gen or s.state != .searching) return; + const job = makeJob(p.gpa, s) catch |err| { + s.state = .failed; + return p.reportError(p.active, "search", err); + }; + defer job.deinit(); + job.waits = false; + var got: Collect = .{ .gpa = p.gpa }; + defer got.batches.deinit(p.gpa); + { + pardes.turn.yield(); + defer pardes.turn.back(); + work(std.Io.Threaded.global_single_threaded.io(), job, &got, Collect.take); + } + for (got.batches.items) |*b| { + receive(p, b); + b.deinit(p.gpa); + } +} + +const Collect = struct { + gpa: std.mem.Allocator, + batches: std.ArrayList(Batch) = .empty, + + fn take(ctx: ?*anyopaque, batch: Batch) bool { + const c: *Collect = @ptrCast(@alignCast(ctx.?)); + c.batches.append(c.gpa, batch) catch { + batch.deinit(c.gpa); + return false; + }; + return true; + } +}; + +/// A batch from a worker (`Event.search`), borrowed: the live generation's +/// rows go to its query pane (a Grep's lines, a listing's names); any other +/// generation's are dropped whole. +pub fn receive(p: *Pardes, b: *const Batch) void { + const s = &p.stream; + if (b.parsed) |parsed| return receiveParse(p, b, parsed.serial, parsed.revision); + if (b.gen != s.gen or (s.state != .searching and s.state != .capped)) { + s.dropped += 1; + return; + } + s.used = b.engine; + s.hits += b.hits; + s.files += b.files; + s.scanned += b.scanned; + s.skipped += b.skipped; + if (s.first_ns == null and b.rows.len > 0) s.first_ns = now(); + if (b.capped) s.state = .capped; + if (b.end) |end| { + s.worker_ns = b.took_ns; + s.state = if (end == .failed) .failed else .done; + s.finished_ns = now(); + // It sent its last: its join is a return away. + if (s.task) |t| { + s.task = null; + pardes.turn.yield(); + var future = t.future; + future.await(t.io); + pardes.turn.back(); + t.job.deinit(); + } + } + p.needs_frame = true; + const id = p.paneBySerial(s.target) orelse return; + const q = p.panes[id].?.query orelse return; + q.capped = s.state == .capped; + panes.Query.take(p, q, s.kind, b.rows); +} + +/// A parse's answer: its task joined; the declarations kept as the pane's +/// (File.State.context_declarations, for its revision) when its text is +/// still what was parsed; the query panes of that text told; and a parse +/// asked for while it ran started. +fn receiveParse(p: *Pardes, b: *const Batch, serial: u32, revision: u32) void { + const s = &p.stream; + if (s.parse) |t| if (t.job.gen == b.gen) { + s.parse = null; + if (t.future) |f| { + pardes.turn.yield(); + var future = f; + future.await(t.io.?); + pardes.turn.back(); + } + t.job.deinit(); + }; + s.parse_ns = b.took_ns; + if (p.paneBySerial(serial)) |id| if (p.panes[id].?.file) |*f| if (f.revision == revision and b.decls.len > 0) { + if (p.tree_sitter_gpa.dupe(syntax.ContextDeclaration, b.decls)) |copy| { + if (f.context_declarations.len > 0) p.tree_sitter_gpa.free(f.context_declarations); + f.context_declarations = copy; + f.context_revision = revision; + panes.Query.declsArrived(p, serial); + } else |_| {} + }; + p.needs_frame = true; + if (s.parse_next) |next| { + s.parse_next = null; + if (p.paneBySerial(next.serial)) |id| parseFor(p, id); + } +} + +// ---- the worker ---- + +const Worker = struct { + io: std.Io, + gpa: std.mem.Allocator, + job: *Job, + ctx: ?*anyopaque, + sink: Sink, + engine: Engine = .walk, + exact: bool, + rows: std.ArrayList(u8) = .empty, + hits: u32 = 0, + files: u32 = 0, + scanned: u32 = 0, + skipped: u32 = 0, + total: u32 = 0, + sent: bool = false, + batch_began: ?std.Io.Timestamp = null, + began: std.Io.Timestamp, + /// One read buffer for every file: grown to the biggest, never 0xAA'd per file. + buf: std.ArrayList(u8) = .empty, + + fn deinit(w: *Worker) void { + w.rows.deinit(w.gpa); + w.buf.deinit(w.gpa); + } + + fn send(w: *Worker, end: ?Batch.End, capped: bool) error{ Refused, OutOfMemory }!void { + const rows = try w.rows.toOwnedSlice(w.gpa); + const batch: Batch = .{ + .gen = w.job.gen, + .rows = rows, + .hits = w.hits, + .files = w.files, + .scanned = w.scanned, + .skipped = w.skipped, + .engine = w.engine, + .capped = capped, + .end = end, + .took_ns = if (end != null) @intCast(@max(0, w.began.durationTo(std.Io.Clock.awake.now(w.io)).nanoseconds)) else 0, + }; + w.hits = 0; + w.files = 0; + w.scanned = 0; + w.skipped = 0; + w.sent = true; + w.batch_began = null; + if (!w.sink(w.ctx, batch)) return error.Refused; + } + + /// After a file with hits: the first batch goes at once (what the user + /// sees first), the others once old or big enough. + fn maybeSend(w: *Worker) !void { + if (w.rows.items.len == 0) return; + const began = w.batch_began orelse { + w.batch_began = std.Io.Clock.awake.now(w.io); + if (!w.sent) try w.send(null, false); + return; + }; + const age = began.durationTo(std.Io.Clock.awake.now(w.io)).nanoseconds; + if (age >= batch_ns or w.rows.items.len >= batch_bytes) try w.send(null, false); + } + + /// At the cap, what is gathered goes, and the worker waits for More (a + /// cancel ends the wait); on the loop it is the end. + fn checkCap(w: *Worker) !bool { + var cap = w.job.cap.load(.acquire); + if (w.total < cap) return false; + if (!w.job.waits) return true; + try w.send(null, true); + while (w.total >= cap) { + try w.io.futexWait(u32, &w.job.cap.raw, cap); + cap = w.job.cap.load(.acquire); + } + return false; + } + + /// Smart case: a pattern with a capital matches exactly, else in any case. + fn find(w: *const Worker, hay: []const u8) ?usize { + return if (w.exact) std.mem.indexOf(u8, hay, w.job.pattern) else std.ascii.findIgnoreCase(hay, w.job.pattern); + } + + fn grepText(w: *Worker, path: []const u8, text: []const u8) !u32 { + var found: u32 = 0; + var line: usize = 0; + var it = std.mem.splitScalar(u8, text, '\n'); + while (it.next()) |raw| { + line += 1; + if (w.total >= w.job.cap.load(.acquire)) break; + const at = w.find(raw) orelse continue; + try w.row(path, line, at, at + w.job.pattern.len, raw); + found += 1; + } + return found; + } + + fn row(w: *Worker, path: []const u8, line: usize, from: usize, end: usize, raw: []const u8) !void { + const ln = std.mem.trimEnd(u8, raw, " \t\r\n"); + var cut = @min(ln.len, 400); + while (cut > 0 and cut < ln.len and ln[cut] & 0xc0 == 0x80) cut -= 1; + try w.rows.print(w.gpa, "{s}:{d}:{d}{c}{d} {s}\n", .{ path, line, from + 1, config.range_sep, end, ln[0..cut] }); + w.hits += 1; + w.total += 1; + } + + /// A listing's file: its name, relative to the base. + fn file(w: *Worker, path: []const u8) !bool { + try w.rows.print(w.gpa, "{s}\n", .{path}); + w.hits += 1; + w.total += 1; + w.files += 1; + try w.maybeSend(); + return w.checkCap(); + } + + /// The name a row gives `abs`: under the base, relative to it. + fn shown(w: *const Worker, abs: []const u8) []const u8 { + const base = std.mem.trimEnd(u8, w.job.base, "/"); + if (abs.len > base.len + 1 and std.mem.startsWith(u8, abs, base) and abs[base.len] == '/') return abs[base.len + 1 ..]; + if (std.mem.startsWith(u8, abs, "./")) return abs[2..]; + return abs; + } + + // ---- our own walk: sorted, depth first, fs.zig's skips ---- + + fn walk(w: *Worker) !void { + var dir = std.Io.Dir.cwd().openDir(w.io, w.job.base, .{ .iterate = true }) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return, + }; + defer dir.close(w.io); + _ = try w.walkDir(dir, "", 0); + } + + /// Whether the cap ended it (on the loop). + fn walkDir(w: *Worker, dir: std.Io.Dir, prefix: []const u8, depth: usize) anyerror!bool { + try w.io.checkCancel(); + var arena: std.heap.ArenaAllocator = .init(w.gpa); + defer arena.deinit(); + const a = arena.allocator(); + const Entry = struct { name: []const u8, kind: std.Io.File.Kind }; + var entries: std.ArrayList(Entry) = .empty; + var it = dir.iterate(); + while (true) { + const e = it.next(w.io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => { + w.skipped += 1; + break; + }, + } orelse break; + try entries.append(a, .{ .name = try a.dupe(u8, e.name), .kind = e.kind }); + } + std.mem.sort(Entry, entries.items, {}, struct { + fn lt(_: void, x: Entry, y: Entry) bool { + return std.mem.lessThan(u8, x.name, y.name); + } + }.lt); + for (entries.items) |e| { + const path = if (prefix.len == 0) e.name else try std.fmt.allocPrint(a, "{s}/{s}", .{ prefix, e.name }); + switch (e.kind) { + .directory => { + if (depth + 1 >= filesystem.WalkNotes.max_depth or e.name[0] == '.') continue; + if (for (filesystem.find_skip) |skip| { + if (std.mem.eql(u8, e.name, skip)) break true; + } else false) continue; + var sub = dir.openDir(w.io, e.name, .{ .iterate = true }) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => { + w.skipped += 1; + continue; + }, + }; + defer sub.close(w.io); + if (try w.walkDir(sub, path, depth + 1)) return true; + }, + .file => switch (w.job.kind) { + .grep => if (try w.grepFile(dir, e.name, path)) return true, + .files => if (try w.file(path)) return true, + }, + else => {}, + } + } + return false; + } + + fn grepFile(w: *Worker, dir: std.Io.Dir, name: []const u8, path: []const u8) !bool { + const f = dir.openFile(w.io, name, .{}) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return false, + }; + defer f.close(w.io); + const len = f.length(w.io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return false, + }; + // ponytail: a file past 16 MiB is passed over unsaid; a chunked + // read, as fs.grep's, would search it. + if (len > max_file_bytes) return false; + try w.buf.resize(w.gpa, @intCast(len)); + const n = f.readPositionalAll(w.io, w.buf.items, 0) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return false, + }; + const text = w.buf.items[0..n]; + w.scanned += 1; + if (std.mem.indexOfScalar(u8, text[0..@min(text.len, 1024)], 0) != null) return false; + const found = try w.grepText(path, text); + if (found == 0) return false; + w.files += 1; + try w.maybeSend(); + return w.checkCap(); + } + + // ---- ripgrep: a child on the worker, parsed, never shown ---- + + /// error.NoRipgrep when there is none to run: the walk does it. + fn ripgrep(w: *Worker) !void { + var argv: std.ArrayList([]const u8) = .empty; + defer argv.deinit(w.gpa); + switch (w.job.kind) { + .grep => { + try argv.appendSlice(w.gpa, &.{ "rg", "--json", "--sort", "path", "--fixed-strings", "--smart-case", "--no-messages", "--" }); + try argv.append(w.gpa, w.job.pattern); + }, + .files => try argv.appendSlice(w.gpa, &.{ "rg", "--files", "--sort", "path", "--no-messages" }), + } + // As selection_pipe's children: this thread's mask is the child's. + const none = std.posix.sigemptyset(); + var kept: std.posix.sigset_t = undefined; + std.posix.sigprocmask(std.posix.SIG.SETMASK, &none, &kept); + var child = std.process.spawn(w.io, .{ + .argv = argv.items, + .cwd = if (w.job.base.len == 0) .inherit else .{ .path = w.job.base }, + .stdin = .ignore, + .stdout = .pipe, + .stderr = .ignore, + }) catch |err| { + std.posix.sigprocmask(std.posix.SIG.SETMASK, &kept, null); + return if (err == error.Canceled) error.Canceled else error.NoRipgrep; + }; + std.posix.sigprocmask(std.posix.SIG.SETMASK, &kept, null); + w.engine = .rg; + // Canceled, capped on the loop, failed: it is killed and reaped. + defer if (child.id != null) child.kill(w.io); + var line: std.ArrayList(u8) = .empty; + defer line.deinit(w.gpa); + var json_arena: std.heap.ArenaAllocator = .init(w.gpa); + defer json_arena.deinit(); + var file_hits: u32 = 0; + var file_path: std.ArrayList(u8) = .empty; + defer file_path.deinit(w.gpa); + // Cut at the cap (on the loop), rg is left to the kill. + var cut = false; + var chunk: [64 * 1024]u8 = undefined; + read: while (true) { + const n = child.stdout.?.readStreaming(w.io, &.{&chunk}) catch |err| switch (err) { + error.EndOfStream => break, + error.Canceled => return error.Canceled, + else => return error.ReadFailed, + }; + if (n == 0) break; + var at: usize = 0; + while (std.mem.indexOfScalarPos(u8, chunk[0..n], at, '\n')) |nl| { + try line.appendSlice(w.gpa, chunk[at..nl]); + at = nl + 1; + defer line.clearRetainingCapacity(); + if (w.job.kind == .files) { + // a dot-file is passed over, as the walk passes over + // dot-directories + if (std.mem.startsWith(u8, std.fs.path.basename(line.items), ".")) continue; + w.scanned += 1; + if (try w.file(w.shown(line.items))) { + cut = true; + break :read; + } + continue; + } + _ = json_arena.reset(.retain_capacity); + const value = std.json.parseFromSliceLeaky(std.json.Value, json_arena.allocator(), line.items, .{}) catch continue; + const obj = if (value == .object) value.object else continue; + const kind = (obj.get("type") orelse continue); + if (kind != .string) continue; + const data = if (obj.get("data")) |d| (if (d == .object) d.object else continue) else continue; + if (std.mem.eql(u8, kind.string, "begin")) { + file_hits = 0; + file_path.clearRetainingCapacity(); + const text = jsonText(data.get("path")) orelse continue; + try file_path.appendSlice(w.gpa, w.shown(text)); + } else if (std.mem.eql(u8, kind.string, "match")) { + if (file_path.items.len == 0 or w.total >= w.job.cap.load(.acquire)) continue; + const text = jsonText(data.get("lines")) orelse continue; + const number = data.get("line_number") orelse continue; + if (number != .integer) continue; + var from: usize = 0; + var end: usize = w.job.pattern.len; + if (data.get("submatches")) |subs| if (subs == .array and subs.array.items.len > 0 and subs.array.items[0] == .object) { + const first = subs.array.items[0].object; + if (first.get("start")) |v| if (v == .integer) { + from = @intCast(v.integer); + }; + if (first.get("end")) |v| if (v == .integer) { + end = @intCast(v.integer); + }; + }; + try w.row(file_path.items, @intCast(number.integer), from, end, text); + file_hits += 1; + } else if (std.mem.eql(u8, kind.string, "end")) { + if (file_hits == 0) continue; + w.files += 1; + file_hits = 0; + try w.maybeSend(); + // At the cap rg is left blocked on its full pipe: its + // own backpressure, until More or a cancel. + if (try w.checkCap()) { + cut = true; + break :read; + } + } else if (std.mem.eql(u8, kind.string, "summary")) { + if (data.get("stats")) |stats| if (stats == .object) if (stats.object.get("searches")) |v| if (v == .integer) { + w.scanned = @intCast(v.integer); + }; + } + } + try line.appendSlice(w.gpa, chunk[at..n]); + } + if (!cut and child.id != null) { + _ = child.wait(w.io) catch |err| if (err == error.Canceled) return error.Canceled; + child.id = null; + } + } +}; + +/// rg's `{"text": …}` (a name or line that is not UTF-8 comes as bytes, +/// and is passed over). +fn jsonText(v: ?std.json.Value) ?[]const u8 { + const o = v orelse return null; + if (o != .object) return null; + const t = o.object.get("text") orelse return null; + return if (t == .string) t.string else null; +} + +fn work(io: std.Io, job: *Job, ctx: ?*anyopaque, sink: Sink) void { + var exact = false; + for (job.pattern) |c| exact = exact or std.ascii.isUpper(c); + var w: Worker = .{ .io = io, .gpa = job.gpa, .job = job, .ctx = ctx, .sink = sink, .exact = exact, .began = std.Io.Clock.awake.now(io) }; + defer w.deinit(); + const forced: ?Engine = job.engine orelse if (std.c.getenv("PARDES_STREAM_ENGINE")) |v| + std.meta.stringToEnum(Engine, std.mem.span(v)) + else + null; + const ran = if ((forced orelse .rg) == .rg) w.ripgrep() else error.NoRipgrep; + ran catch |err| switch (err) { + error.Canceled, error.Refused => return, + error.NoRipgrep => { + w.engine = .walk; + w.walk() catch |walk_err| switch (walk_err) { + error.Canceled, error.Refused => return, + else => { + w.send(.failed, false) catch {}; + return; + }, + }; + }, + else => { + w.send(.failed, false) catch {}; + return; + }, + }; + w.send(.done, false) catch {}; +} + +// ---- tests ---- + +const host_io = @import("host_io.zig"); +const testing = std.testing; + +/// The shell's half, in a test: perform the `search` effect on a real +/// worker, hand what it posts to the core. +const TestShell = struct { + q: host_io.Inbox, + + fn init() TestShell { + return .{ .q = .{ .gpa = testing.allocator, .lsp_allocator = testing.allocator, .io = testing.io } }; + } + + const vtable: pardes.Host.VTable = .{ .search = struct { + fn f(_: ?*anyopaque, _: u32) void {} + }.f }; + + fn attach(p: *Pardes) void { + p.host = .{ .vtable = &vtable }; + } + + fn pump(t: *TestShell, p: *Pardes) void { + while (p.nextEffect()) |e| switch (e) { + .search => |s| start(testing.io, testing.allocator, p, s.gen, &t.q, host_io.Inbox.pushSearch), + else => {}, + }; + var batch: host_io.Inbox.Batch = .{}; + t.q.takeInto(&batch); + for (batch.slice()) |m| switch (m) { + .search => |b| { + p.update(.{ .search = b }); + b.deinit(testing.allocator); + }, + else => m.deinit(testing.allocator, testing.allocator), + }; + } + + /// Pump until the search is over (or `limit_ms` passes: a failure). + fn finish(t: *TestShell, p: *Pardes, limit_ms: i64) !void { + const until = std.Io.Clock.awake.now(testing.io).toMilliseconds() + limit_ms; + while (p.stream.state == .searching or p.stream.state == .waiting) { + if (std.Io.Clock.awake.now(testing.io).toMilliseconds() > until) return error.Timeout; + if (p.stream.state == .waiting) p.update(.tick); + t.pump(p); + testing.io.sleep(.fromMicroseconds(200), .awake) catch {}; + } + } +}; + +/// A worker that never heard of the cancel: it pushes rows of a stale +/// generation for as long as the live one streams. +const Rogue = struct { + q: *host_io.Inbox, + gen: u32, + stop: std.atomic.Value(bool) = .init(false), + done: std.atomic.Value(bool) = .init(false), + pushed: usize = 0, + + fn run(r: *Rogue) void { + defer r.done.store(true, .release); + while (!r.stop.load(.acquire) and r.pushed < 200) { + const rows = testing.allocator.dupe(u8, "d00/f000.txt:1:3-8 stale stale\n") catch return; + if (!host_io.Inbox.pushSearch(r.q, .{ .gen = r.gen, .rows = rows, .hits = 1, .files = 1 })) return; + r.pushed += 1; + std.Thread.yield() catch {}; + } + } +}; + +test "Stream: a stale generation's rows never land in the query pane, however two generations race (walk and rg)" { + if (comptime !filesystem.platform_has_fs) return; + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var root_buf: [4096]u8 = undefined; + const files = 120; + var name_buf: [64]u8 = undefined; + for (0..files) |i| { + const name = try std.fmt.bufPrint(&name_buf, "d{d:0>2}/f{d:0>3}.txt", .{ i / 10, i }); + try tmp.dir.createDirPath(testing.io, std.fs.path.dirname(name).?); + try tmp.dir.writeFile(testing.io, .{ .sub_path = name, .data = "a needle line\nsome hay here\nneedle again\nhay hay\nmore hay\n" }); + } + const root = root_buf[0..try tmp.dir.realPath(testing.io, &root_buf)]; + const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 120, .rows = 40 }); + defer p.deinit(); + const pane = try p.setTestFile("x\n"); + p.gpa.free(pane.file.?.path); + pane.file.?.path = try std.fs.path.join(p.gpa, &.{ root, "main.txt" }); + p.advance(1); + var shell: TestShell = .init(); + TestShell.attach(p); + defer { + p.stream.cancelTask(); + shell.q.deinit(); + } + // The query pane the rows are for: a Grep's. + panes.Query.open(p, 0, .grep, "", false); + const qid = for (p.panes, 0..) |slot, i| { + if (slot) |qp| if (qp.query != null) break i; + } else return error.NoQueryPane; + const qp = p.panes[qid].?; + const q = qp.query.?; + var dropped_total: usize = 0; + for ([_]Engine{ .walk, .rg }) |engine| { + p.stream.engine = engine; + for (0..12) |round| { + // A starts and runs a little (0, 1 or 2 pumps: its batches in + // flight, queued, or taken in), then B replaces it, typed (its + // pause run out on the clock) or at once. + const a = if (round % 2 == 0) "needle" else "hay"; + const b = if (round % 2 == 0) "hay" else "needle"; + q.found.clearRetainingCapacity(); + q.found_bytes.clearRetainingCapacity(); + query(p, qp.serial, .grep, a, root, .now); + p.update(.tick); + for (0..round % 3) |_| shell.pump(p); + var rogue: Rogue = .{ .q = &shell.q, .gen = p.stream.gen }; + q.found.clearRetainingCapacity(); + q.found_bytes.clearRetainingCapacity(); + if (round % 4 == 3) { + query(p, qp.serial, .grep, b, root, .typed); + try testing.expectEqual(State.waiting, p.stream.state); + p.advance(p.now_ns + debounce_ns + std.time.ns_per_ms); + } else query(p, qp.serial, .grep, b, root, .now); + try testing.expect(p.stream.gen != rogue.gen); + const thread = try std.Thread.spawn(.{}, Rogue.run, .{&rogue}); + const finished = shell.finish(p, 20_000); + // It may be waiting for room: the queue is taken until it ends. + rogue.stop.store(true, .release); + while (!rogue.done.load(.acquire)) shell.pump(p); + thread.join(); + shell.pump(p); // whatever it pushed last + try finished; + try testing.expectEqual(State.done, p.stream.state); + // b's lines, every one, and not one of a's nor the rogue's. + const per_file: usize = if (std.mem.eql(u8, b, "hay")) 3 else 2; + try testing.expectEqual(files * per_file, q.found.items.len); + try testing.expect(std.mem.indexOf(u8, q.found_bytes.items, "stale") == null); + for (q.found.items) |fd| { + const code = q.found_bytes.items[fd.code..][0..fd.code_len]; + try testing.expect(std.ascii.findIgnoreCase(code, b) != null); + } + if (engine == .rg and p.stream.used != .rg) return; // no rg here: the walk was the test + dropped_total += p.stream.dropped; + p.stream.dropped = 0; + } + } + try testing.expect(dropped_total > 0); +} + +test "Stream: the listing of files for Find on a worker, a cap of hits stops the walk on the loop" { + if (comptime !filesystem.platform_has_fs) return; + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var root_buf: [4096]u8 = undefined; + try tmp.dir.createDirPath(testing.io, "src"); + try tmp.dir.createDirPath(testing.io, ".git"); + for ([_][]const u8{ "src/a.zig", "src/b.zig", "notes.md", ".git/config" }) |name| + try tmp.dir.writeFile(testing.io, .{ .sub_path = name, .data = "needle\n" }); + const root = root_buf[0..try tmp.dir.realPath(testing.io, &root_buf)]; + const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 120, .rows = 40 }); + defer p.deinit(); + const pane = try p.setTestFile("x\n"); + p.gpa.free(pane.file.?.path); + pane.file.?.path = try std.fs.path.join(p.gpa, &.{ root, "main.txt" }); + p.advance(1); + p.stream.engine = .walk; + // No host workers: the listing runs on the loop at the next tick. + panes.Query.open(p, 0, .find, "", false); + p.update(.tick); + p.update(.tick); + const qid = for (p.panes, 0..) |slot, i| { + if (slot) |qp| if (qp.query != null) break i; + } else return error.NoQueryPane; + const q = p.panes[qid].?.query.?; + // the three files, not the dot-directory's + try testing.expectEqual(@as(usize, 3), q.names.cands.items.len); + try testing.expectEqual(State.done, p.stream.state); +} + +test "Stream: a big text's declarations are parsed on a worker, never the loop, and the rows of its / regroup under them" { + if (comptime !filesystem.platform_has_fs) return; + const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 120, .rows = 40 }); + defer p.deinit(); + var text: std.ArrayList(u8) = .empty; + defer text.deinit(testing.allocator); + for (0..4000) |i| try text.print(testing.allocator, "fn f{d}() void {{\n _ = needle;\n}}\n", .{i}); + const pane = try p.setTestFile(text.items); + p.gpa.free(pane.file.?.path); + pane.file.?.path = try p.gpa.dupe(u8, "/tmp/big.zig"); + p.advance(1); + var shell: TestShell = .init(); + TestShell.attach(p); + defer { + p.stream.cancelTask(); + p.stream.cancelParse(); + shell.q.deinit(); + } + for ("/needle") |c| p.update(.{ .key = .{ .cp = c, .text = &[_]u8{c} } }); + const qid = for (p.panes, 0..) |slot, i| { + if (slot) |qp| if (qp.query != null) break i; + } else return error.NoQueryPane; + const qp = p.panes[qid].?; + // Not grouped yet: no parse on the loop; one asked of the worker. + try testing.expect(p.stream.parse != null); + try testing.expect(std.mem.indexOf(u8, qp.file.?.content, "fn f") == null); + const until = std.Io.Clock.awake.now(testing.io).toMilliseconds() + 60_000; + while (pane.file.?.context_revision == null or pane.file.?.context_revision.? != pane.file.?.revision) { + if (std.Io.Clock.awake.now(testing.io).toMilliseconds() > until) return error.Timeout; + shell.pump(p); + p.update(.tick); + testing.io.sleep(.fromMilliseconds(2), .awake) catch {}; + } + p.update(.tick); + // Now under their declarations: `fn fN() void {` headers. + try testing.expect(std.mem.indexOf(u8, qp.file.?.content, "fn f") != null); + try testing.expect(p.stream.parse == null); +} + +test "zz stream timings on the stress file with a worker (QUERY_TIMES=<path>)" { + const path_z = std.c.getenv("QUERY_TIMES") orelse return error.SkipZigTest; + const path = std.mem.span(path_z); + const bytes = try std.Io.Dir.cwd().readFileAlloc(testing.io, path, testing.allocator, .limited(8 << 20)); + defer testing.allocator.free(bytes); + const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 160, .rows = 50 }); + defer p.deinit(); + const pane = try p.setTestFile(bytes); + p.gpa.free(pane.file.?.path); + pane.file.?.path = try p.gpa.dupe(u8, path); + p.advance(1); + var shell: TestShell = .init(); + TestShell.attach(p); + defer { + p.stream.cancelTask(); + p.stream.cancelParse(); + shell.q.deinit(); + } + var frame = std.heap.ArenaAllocator.init(testing.allocator); + defer frame.deinit(); + pane.body.cur_row = 15000; + pane.ensureCursorVisible(); + _ = try p.render(frame.allocator()); + const t0 = now(); + for ("/pane") |c| p.update(.{ .key = .{ .cp = c, .text = &[_]u8{c} } }); + const typed = now() - t0; + var worst_wait: u64 = 0; + while (pane.file.?.context_revision == null or pane.file.?.context_revision.? != pane.file.?.revision) { + const t1 = now(); + shell.pump(p); + p.update(.tick); + _ = frame.reset(.retain_capacity); + _ = try p.render(frame.allocator()); + worst_wait = @max(worst_wait, now() - t1); + testing.io.sleep(.fromMilliseconds(2), .awake) catch {}; + } + var worst_key: u64 = 0; + for (".v") |c| { + const t2 = now(); + p.update(.{ .key = .{ .cp = c, .text = &[_]u8{c} } }); + _ = frame.reset(.retain_capacity); + _ = try p.render(frame.allocator()); + worst_key = @max(worst_key, now() - t2); + for (0..12) |_| p.update(.tick); + } + std.debug.print("stream: '/pane' typed in {d} us; the worker's parse {d} ms; worst frame while it parsed {d} us; a keystroke with grouped rows {d} us\n", .{ typed / 1000, p.stream.parse_ns / std.time.ns_per_ms, worst_wait / 1000, worst_key / 1000 }); +} + +test "Stream: a Grep stops at its 2000 lines and waits; More in the query pane's tag goes on" { + if (comptime !filesystem.platform_has_fs) return; + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var root_buf: [4096]u8 = undefined; + var name_buf: [64]u8 = undefined; + var text: std.ArrayList(u8) = .empty; + defer text.deinit(testing.allocator); + for (0..20) |_| try text.appendSlice(testing.allocator, "a needle here\n"); + for (0..150) |i| try tmp.dir.writeFile(testing.io, .{ .sub_path = try std.fmt.bufPrint(&name_buf, "f{d:0>3}.txt", .{i}), .data = text.items }); + const root = root_buf[0..try tmp.dir.realPath(testing.io, &root_buf)]; + const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 120, .rows = 40 }); + defer p.deinit(); + const pane = try p.setTestFile("x\n"); + p.gpa.free(pane.file.?.path); + pane.file.?.path = try std.fs.path.join(p.gpa, &.{ root, "main.txt" }); + p.advance(1); + var shell: TestShell = .init(); + TestShell.attach(p); + defer { + p.stream.cancelTask(); + shell.q.deinit(); + } + p.stream.engine = .walk; + panes.Query.open(p, 0, .grep, "needle", false); + const qid = for (p.panes, 0..) |slot, i| { + if (slot) |qp| if (qp.query != null) break i; + } else return error.NoQueryPane; + const qp = p.panes[qid].?; + p.advance(p.now_ns + debounce_ns + std.time.ns_per_ms); + const until = std.Io.Clock.awake.now(testing.io).toMilliseconds() + 20_000; + while (p.stream.state != .capped) { + if (std.Io.Clock.awake.now(testing.io).toMilliseconds() > until) return error.Timeout; + p.update(.tick); + shell.pump(p); + testing.io.sleep(.fromMicroseconds(200), .awake) catch {}; + } + p.update(.tick); + try testing.expectEqual(@as(usize, soft_cap), qp.query.?.found.items.len); + try testing.expect(qp.query.?.capped); + try testing.expectEqualStrings(" More Del", @import("tagline.zig").curTail(qp)); + pardes.exec.runBuiltin(p, .More, qid, "", null); + try shell.finish(p, 20_000); + p.update(.tick); + try testing.expectEqual(State.done, p.stream.state); + try testing.expectEqual(@as(usize, 3000), qp.query.?.found.items.len); + try testing.expect(!qp.query.?.capped); +} diff --git a/src/builtins.zig b/src/builtins.zig index 7cdee9de..c8909e66 100644 --- a/src/builtins.zig +++ b/src/builtins.zig @@ -1756,6 +1756,14 @@ pub const Search = struct { } }; +/// A query pane's Grep stopped at its 2000 lines goes on for 2000 more. +pub const More = struct { + pub const quiet = true; + pub fn run(c: Ctx) void { + if (!pardes.Stream.moreFor(c.p, c.id)) c.p.reportFailure(c.id, "More: no search is waiting at its cap"); + } +}; + /// Pick an open pane by its name in a query pane under this one (SPC b b). pub const Panes = struct { pub const takes_arg = true; diff --git a/src/config.zig b/src/config.zig index 2fa4a55d..25c9dc5a 100644 --- a/src/config.zig +++ b/src/config.zig @@ -119,6 +119,7 @@ pub const leader_path = paths: { .SearchRows = null, .Search = null, .Panes = null, + .More = null, .BootShell = null, .LookWord = null, .DirLook = null, diff --git a/src/detached/server.zig b/src/detached/server.zig index 124a4c5a..eefce93e 100644 --- a/src/detached/server.zig +++ b/src/detached/server.zig @@ -166,6 +166,8 @@ pub const Session = struct { s.inbox.lsp_task = null; } s.pipe_tasks.cancelAll(s.io); + // Its worker may be waiting on the inbox for room. + s.core.stream.cancelTask(); _ = s.drainCompletions(false); } @@ -197,6 +199,7 @@ pub const Session = struct { var buf: [256]u8 = undefined; s.core.setStatus(s.core.active, message.stamp(&buf, "lsp", status)); }, + .search => |b| s.core.update(.{ .search = b }), // Ptys are read in the loop's own poll, not through the inbox. .output, .eof, .files_changed, .fs_ready => {}, } @@ -350,8 +353,15 @@ pub const Session = struct { .detach = detach, .lsp = lspRequest, .pipe = pipeRequest, + .search = searchRequest, }; + fn searchRequest(ctx: ?*anyopaque, gen: u32) void { + const s = of(ctx); + if (s.wake[0] < 0) return pardes.Stream.runHere(s.core, gen); + pardes.Stream.start(s.io, s.worker_gpa, s.core, gen, &s.inbox, host_io.Inbox.pushSearch); + } + fn lspRequest(ctx: ?*anyopaque, request: host_io.Lsp.Request) void { const s = of(ctx); _ = s.drainCompletions(true); diff --git a/src/detached/wire.zig b/src/detached/wire.zig index 2318d319..82614b4f 100644 --- a/src/detached/wire.zig +++ b/src/detached/wire.zig @@ -856,7 +856,7 @@ fn clientTag(msg: ClientMsg) ClientTag { return switch (msg) { .event => |ev| switch (ev) { // Machine-local reports and 9P requests belong to the session owner. - .output, .eof, .exited, .lsp_resp, .pipe_resp, .file_changed, .tick => unreachable, + .output, .eof, .exited, .lsp_resp, .pipe_resp, .search, .file_changed, .tick => unreachable, inline else => |_, t| @field(ClientTag, @tagName(t)), }, inline else => |_, t| @field(ClientTag, @tagName(t)), @@ -986,7 +986,7 @@ pub fn encodeClient(out: []u8, msg: ClientMsg) Error![]const u8 { .pointer_leave => {}, .focus => |focused| try w.writeByte(@intFromBool(focused)), // See `clientTag`: no tag, so nothing to encode. - .output, .eof, .exited, .lsp_resp, .pipe_resp, .file_changed, .tick => unreachable, + .output, .eof, .exited, .lsp_resp, .pipe_resp, .search, .file_changed, .tick => unreachable, }, } try finishMessage(&w, at); @@ -1137,7 +1137,7 @@ pub fn clientBound(msg: ClientMsg) usize { .paste => |b| b.len, .command => |line| line.len, // See `clientTag`: not on this wire in this direction. - .output, .eof, .exited, .lsp_resp, .pipe_resp, .file_changed, .tick => unreachable, + .output, .eof, .exited, .lsp_resp, .pipe_resp, .search, .file_changed, .tick => unreachable, }, }; } diff --git a/src/gui/gui.zig b/src/gui/gui.zig index 5017ab53..49a98f5b 100644 --- a/src/gui/gui.zig +++ b/src/gui/gui.zig @@ -2202,6 +2202,8 @@ fn localSession( }; defer { if (queue.lsp_task) |*task| task.future.cancel(io); + // Its worker may be waiting on the queue for room. + core.stream.cancelTask(); queue.deinit(); } var pipe_tasks: PipeTasks = .{}; @@ -2573,6 +2575,8 @@ fn runGrid(init: std.process.Init, opts_in: pardes.Options) !void { }; defer { if (queue.lsp_task) |*task| task.future.cancel(io); + // Its worker may be waiting on the queue for room. + core.stream.cancelTask(); queue.deinit(); } var pipe_tasks: PipeTasks = .{}; @@ -3815,6 +3819,7 @@ const Shell = struct { .open_link = openLink, .lsp = lsp, .pipe = pipe, + .search = search, }; const grid_vtable: pardes.Host.VTable = vt: { @@ -3908,6 +3913,11 @@ const Shell = struct { s.saw_event = true; s.pipe_tasks.finish(s.io, response_value.id); }, + .search => |b| { + s.core.update(.{ .search = b }); + b.deinit(s.gpa); + s.saw_event = true; + }, .files_changed => check_files = true, .fs_ready => {}, }; @@ -4473,6 +4483,12 @@ fn pipe(ctx: ?*anyopaque, id: u32) void { s.core.reportError(s.core.active, "pipe", error.WorkersUnavailable); } +fn search(ctx: ?*anyopaque, gen: u32) void { + const s = shellOf(ctx); + if (s.threads_ok) return pardes.Stream.start(s.io, s.gpa, s.core, gen, s.queue, Queue.pushSearch); + pardes.Stream.runHere(s.core, gen); +} + fn pollCwds(core: *pardes.Pardes, ptys: *[pardes.MAX_PANES]?Pty) void { for (ptys, 0..) |slot, id| if (slot) |pt| { if (pt.fd < 0) continue; diff --git a/src/host_io.zig b/src/host_io.zig index b32fff55..f5d6f375 100644 --- a/src/host_io.zig +++ b/src/host_io.zig @@ -48,6 +48,9 @@ pub const Host = struct { open_link: ?*const fn (ctx: ?*anyopaque, url: []const u8) void = null, lsp: ?*const fn (ctx: ?*anyopaque, req: Lsp.Request) void = null, pipe: ?*const fn (ctx: ?*anyopaque, id: u32) void = null, + /// Start the streaming search of this generation on a worker + /// (pardes.Stream.start); absent, it runs on the loop. + search: ?*const fn (ctx: ?*anyopaque, gen: u32) void = null, }; }; @@ -129,6 +132,9 @@ pub const Inbox = struct { lsp: struct { id: u32, rows: ?[]u8 }, lsp_status: []u8, pipe: selection_pipe.Response, + /// A streaming search's rows (pardes.Stream), of whatever generation: + /// the core drops a stale one. + search: pardes.Stream.Batch, files_changed, fs_ready, @@ -141,6 +147,7 @@ pub const Inbox = struct { var response = response_value; response.deinit(gpa); }, + .search => |b| b.deinit(gpa), .eof, .files_changed, .fs_ready => {}, } } @@ -425,6 +432,34 @@ pub const Inbox = struct { q.push(.{ .pipe = response }); } + /// A streaming search's batch (pardes.Stream.start's sink). Never + /// dropped for want of room, as output is not: the worker waits for + /// room, a wait its cancel ends (false: it stops). + pub fn pushSearch(ctx: ?*anyopaque, batch: pardes.Stream.Batch) bool { + const q: *Inbox = @ptrCast(@alignCast(ctx.?)); + q.lock(); + while (!q.closed and q.len >= output_capacity) { + q.waiting += 1; + q.space.wait(q.io, &q.mutex) catch { + q.waiting -= 1; + q.unlock(); + batch.deinit(q.gpa); + return false; + }; + q.waiting -= 1; + } + if (q.closed) { + q.unlock(); + batch.deinit(q.gpa); + return false; + } + q.items[(q.head + q.len) % q.items.len] = .{ .search = batch }; + q.len += 1; + q.unlock(); + q.wake(); + return true; + } + /// A watched child exited (setExitWake): wake the loop, which takes it. pub fn wakeForExit(ctx: ?*anyopaque) void { const q: *Inbox = @ptrCast(@alignCast(ctx.?)); diff --git a/src/macos.zig b/src/macos.zig index 9378359c..7599a02d 100644 --- a/src/macos.zig +++ b/src/macos.zig @@ -1101,6 +1101,8 @@ fn drainInbox(st: *State) bool { } }); st.pipe_tasks.finish(st.io, value.id); }, + // ponytail: no `search` host entry here, so Stream runs on the loop. + .search => |b| st.core.update(.{ .search = b }), // The SDL shell's: this one's file watches and 9P are its own. .files_changed, .fs_ready => {}, } diff --git a/src/pardes.zig b/src/pardes.zig index 1b4e9920..3ccfee5f 100644 --- a/src/pardes.zig +++ b/src/pardes.zig @@ -25,6 +25,7 @@ pub const Messages = @import("Messages.zig"); pub const Registers = @import("Registers.zig"); pub const Macro = @import("Macro.zig"); pub const Pipe = @import("Pipe.zig"); +pub const Stream = @import("Stream.zig"); pub const colors = @import("colors.zig"); pub const Theme = colors.Theme; pub const themes = colors.themes; @@ -488,6 +489,7 @@ test { _ = @import("Places.zig"); _ = @import("Query.zig"); _ = @import("Names.zig"); + _ = @import("Stream.zig"); } pub const Builtin = builtins.registry.Builtin(); @@ -3669,6 +3671,8 @@ pub const Event = union(enum) { failure: ?selection_pipe.Failure = null, }, file_changed: struct { pane: u8, bytes: []const u8 }, + /// A disk search's batch (Stream.zig), borrowed for the step. + search: Stream.Batch, paste: []const u8, command: []const u8, /// Native shells may preserve sub-cell wheel distance in physical pixels. @@ -3725,6 +3729,9 @@ pub const Effect = union(enum) { /// Snapshot the matching request with pipeRequest(id), then run it away /// from the UI/event loop and answer with pipe_resp. pipe: struct { id: u32 }, + /// Start the disk search of this generation (Stream.start) on a worker + /// posting into the shell's inbox. + search: struct { gen: u32 }, /// Write what an Edit's `w` took (`Pipe.writes`), in order. edit_write, watch: struct { pane: u8, on: bool, mode: WatchMode = .reconcile }, @@ -4584,6 +4591,8 @@ pub const Pardes = struct { } = null, pipe: Pipe = .{}, + /// A query pane's Grep, and its Find's listing, on a worker (Stream.zig). + stream: Stream = .{}, fs: filesystem.Namespace = .{}, boot: Boot = .shell, @@ -4950,6 +4959,7 @@ pub const Pardes = struct { if (p.chord_arg) |a| gpa.free(a); for (p.spawn_dirs) |d| if (d) |dir| gpa.free(dir); p.pipe.deinit(gpa); + p.stream.deinit(gpa); p.locations_cache.deinit(p.tree_sitter_gpa); p.fs.deinit(gpa); p.shell_rows.reset(gpa); @@ -5547,7 +5557,7 @@ pub const Pardes = struct { pub fn postEvent(p: *Pardes, ev: Event) void { switch (ev) { .key => |k| std.debug.assert(k.text.len == 0), - .output, .paste, .lsp_resp, .pipe_resp, .file_changed, .command => unreachable, + .output, .paste, .lsp_resp, .pipe_resp, .search, .file_changed, .command => unreachable, else => {}, } if (p.in_len == p.in_q.len) return; @@ -5846,6 +5856,8 @@ pub const Pardes = struct { f(p.host.ctx, q.id) else p.update(.{ .pipe_resp = .{ .id = q.id, .success = false, .outputs = &.{} } }), + // No workers here (a test, a board): it runs on the loop. + .search => |q| if (v.search) |f| f(p.host.ctx, q.gen) else Stream.runHere(p, q.gen), .edit_write => { // Each write takes the bytes `w` took when it ran: the text // as it was before the Edit's changes. @@ -5995,7 +6007,8 @@ pub const Pardes = struct { pane.msg_announcement = false; }; switch (ev) { - .tick => {}, + // A batch of rows coming in is no gesture: a Look's hover stays. + .tick, .search => {}, .mouse => |m| if (!(m.button == .none and m.kind == .motion)) { p.raw_hover_intent = false; look.cancelLookHover(p); @@ -6026,6 +6039,7 @@ pub const Pardes = struct { p.clip_pending = null; }, .tick => { + Stream.step(p); Messages.advanceMessages(p); for (0..p.panes.len) |id| exec.flushReplEnter(p, id, false); }, @@ -6090,6 +6104,7 @@ pub const Pardes = struct { }, .lsp_resp => |r| p.lspResponse(r.id, r.rows), .pipe_resp => |r| Pipe.pipeResponse(p, r.id, r.success, r.outputs, r.failure), + .search => |b| Stream.receive(p, &b), .file_changed => |fc| _ = p.applyWatchedFileChanged(fc.pane, fc.bytes), .key => |key| { p.can_ask = true; @@ -7632,7 +7647,11 @@ pub const Pardes = struct { // Places still to find resume on the next frame. for (p.panes) |slot| if (slot) |pane| if (pane.places.busy(pane)) return next; if (panes.Query.busy(p)) return next; + // A typed Grep waiting out its pause. + const streaming = p.stream.wake(next); + if (streaming) |due| if (due <= next) return next; var frames: ?u64 = null; + if (streaming) |due| frames = @max(1, (due -| p.stepped_ns + animation.frame_ns - 1) / animation.frame_ns); if (p.look_hover_wait) |waiting| { const delay = config.look_preview_delay_frames orelse return next; frames = @max(1, delay -| waiting.frames); diff --git a/src/syntax.zig b/src/syntax.zig index 803d815a..81722ce5 100644 --- a/src/syntax.zig +++ b/src/syntax.zig @@ -572,7 +572,7 @@ pub fn analyzeSource(gpa: std.mem.Allocator, path: []const u8, content: []const defer tree.destroy(); var result: SourceAnalysis = .{}; errdefer result.deinit(gpa); - if (want_declarations) result.declarations = try declarationsFromTree(gpa, content, tree); + if (want_declarations) result.declarations = try declarationsFromTree(gpa, content, tree, null); if (want_colors) { result.colors = try gpa.alloc(u8, content.len); @memset(result.colors, 0); @@ -586,14 +586,55 @@ pub fn contextDeclarations(gpa: std.mem.Allocator, path: []const u8, content: [] return analysis.declarations; } -fn declarationsFromTree(gpa: std.mem.Allocator, content: []const u8, tree: *ts.Tree) ![]ContextDeclaration { +/// The declarations of `content` (its path's language), parsed with a +/// parser of its own, so a worker thread may ask: the loop's parsers +/// (`ensure`'s, one a language) are not touched, and a language is safe to +/// share. `stop`, set from another thread, ends the parse and the walk +/// early (error.Canceled): a 1.2 MB file is seconds in a Debug build. +pub fn declarationsApart(gpa: std.mem.Allocator, path: []const u8, content: []const u8, halt: *const std.atomic.Value(bool)) ![]ContextDeclaration { + if (comptime !enabled) return &.{}; + if (content.len == 0) return &.{}; + const ext = std.fs.path.extension(path); + const spec = specs: for (&specs) |*spec| { + for (spec.exts) |choice| if (std.ascii.eqlIgnoreCase(ext, choice)) break :specs spec; + } else return &.{}; + const parser = ts.Parser.create(); + defer parser.destroy(); + try parser.setLanguage(spec.language()); + const Reader = struct { + text: []const u8, + fn read(payload: ?*anyopaque, byte: u32, _: ts.Point, n: *u32) callconv(.c) [*c]const u8 { + const r: *@This() = @ptrCast(@alignCast(payload.?)); + const at = @min(byte, r.text.len); + n.* = @intCast(r.text.len - at); + return r.text.ptr + at; + } + fn progress(state: *ts.Parser.State) callconv(.c) bool { + const flag: *const std.atomic.Value(bool) = @ptrCast(@alignCast(state.payload.?)); + return flag.load(.monotonic); + } + }; + var reader: Reader = .{ .text = content }; + const tree = parser.parseWithOptions( + .{ .payload = &reader, .read = Reader.read, .encoding = .utf8 }, + null, + .{ .payload = @constCast(halt), .progress_callback = @ptrCast(&Reader.progress) }, + ) orelse return error.Canceled; + defer tree.destroy(); + return declarationsFromTree(gpa, content, tree, halt); +} + +fn declarationsFromTree(gpa: std.mem.Allocator, content: []const u8, tree: *ts.Tree, halt: ?*const std.atomic.Value(bool)) ![]ContextDeclaration { var result: std.ArrayList(ContextDeclaration) = .empty; errdefer result.deinit(gpa); // The cursor retains its ancestor stack: sibling visits do not repeatedly // reconstruct parents in broad syntax trees. Preorder preserves source order. var cursor = tree.rootNode().walk(); defer cursor.destroy(); + var visited: usize = 0; while (true) { + visited += 1; + if (halt) |flag| if (visited % 4096 == 0 and flag.load(.monotonic)) return error.Canceled; const node = cursor.node(); if (node.isNamed()) if (contextSpan(node, content)) |span| { if (result.items.len == 0 or result.items[result.items.len - 1].start_line != span.start_line or diff --git a/src/tagline.zig b/src/tagline.zig index 89152a3a..3c8dd3e1 100644 --- a/src/tagline.zig +++ b/src/tagline.zig @@ -39,6 +39,8 @@ pub const dir_pane_tail = " " ++ config.dir_pane_builtins_str; /// them into text a Save keeps; Unattach leaves it where it stands. pub const query_live_tail = " Del"; pub const query_frozen_tail = " Save Unattach Del"; +/// ...and while its Grep waits at 2000 lines for More (Stream.zig). +pub const query_capped_tail = " More Del"; /// A file with changes it has not saved. pub fn dirty(pane: *const panes.Pane) bool { const f = pane.file orelse return false; @@ -181,7 +183,7 @@ pub fn curTail(pane: *Pane) []const u8 { pub fn defaultTail(pane: *const Pane) []const u8 { if (comptime pdf_enabled) if (pane.pdf != null) return pdf_pane_tail; - if (pane.query) |q| return if (q.frozen) query_frozen_tail else query_live_tail; + if (pane.query) |q| return if (q.frozen) query_frozen_tail else if (q.capped) query_capped_tail else query_live_tail; if (pane.file) |file| if (file.listing != null) return dir_pane_tail; if (pane.file) |file| if (file.pager) return if (file.pager_diff) pager_diff_pane_tail else pager_pane_tail; if (pane.file != null and pane.colorAlgo() == .diff) return diff_pane_tail; diff --git a/src/tty/tty.zig b/src/tty/tty.zig index 8f5f102a..572a6f2a 100644 --- a/src/tty/tty.zig +++ b/src/tty/tty.zig @@ -94,6 +94,8 @@ pub const Command = struct { lsp_done: struct { id: u32, rows: ?[]u8 }, lsp_status: []u8, pipe_done: selection_pipe.Response, + /// A streaming search's rows (pardes.Stream). + search_rows: pardes.Stream.Batch, files_changed, fs_ready, } = .nop; @@ -1354,6 +1356,8 @@ fn localSession( sh.lsp_task = null; } sh.pipe_tasks.cancelAll(io); + // Its worker may be waiting on the loop's queue for room. + core.stream.cancelTask(); file_watch.stop(sh.inotify_fd); if (sh.watch_task) |*t| { t.cancel(io) catch {}; @@ -1372,6 +1376,7 @@ fn localSession( var response = response_value; response.deinit(gpa); }, + .search_rows => |b| b.deinit(gpa), else => {}, }; } @@ -1557,6 +1562,7 @@ const Shell = struct { .open_link = openLink, .lsp = lsp, .pipe = pipe, + .search = search, }; fn clockNow(ctx: ?*anyopaque) u64 { @@ -1719,6 +1725,10 @@ const Shell = struct { response.deinit(s.gpa); s.pipe_tasks.finish(s.io, response_value.id); }, + .search_rows => |b| { + core.update(.{ .search = b }); + b.deinit(s.gpa); + }, } return false; } @@ -2027,6 +2037,22 @@ const Shell = struct { selection_pipe.start(s.io, s.gpa, s.core, &s.pipe_tasks, id, s, deliverPipe); } + fn search(ctx: ?*anyopaque, gen: u32) void { + const s = of(ctx); + if (s.threads_ok) return pardes.Stream.start(s.io, s.gpa, s.core, gen, s, deliverSearch); + pardes.Stream.runHere(s.core, gen); + } + + /// The loop's queue waits for room (a wait the worker's cancel ends). + fn deliverSearch(ctx: ?*anyopaque, batch: pardes.Stream.Batch) bool { + const s = of(ctx); + s.loop.postEvent(.{ .search_rows = batch }) catch { + batch.deinit(s.gpa); + return false; + }; + return true; + } + fn deliverPipe(ctx: ?*anyopaque, response: selection_pipe.Response) void { const s = of(ctx); var owned = response; @@ -2876,7 +2902,7 @@ const Attach = struct { fn apply(a: *Attach, event: @TypeOf(Command.value)) ?AttachEnd { switch (event) { - .nop, .wake, .fs_ready, .lsp_done, .lsp_status, .pipe_done, .pty_read, .pty_eof, .files_changed => {}, + .nop, .wake, .fs_ready, .lsp_done, .lsp_status, .pipe_done, .search_rows, .pty_read, .pty_eof, .files_changed => {}, .quit => return .none, .focus_in => return a.send(.{ .focus = true }), .focus_out => { |
