summaryrefslogtreecommitdiff
path: root/src/fs.zig
diff options
context:
space:
mode:
Diffstat (limited to 'src/fs.zig')
-rw-r--r--src/fs.zig285
1 files changed, 2 insertions, 283 deletions
diff --git a/src/fs.zig b/src/fs.zig
index cb813262..08379697 100644
--- a/src/fs.zig
+++ b/src/fs.zig
@@ -1115,297 +1115,16 @@ pub const WalkNotes = struct {
}
};
const find_max_depth = 16;
-const find_max_steps = 100_000;
-/// One root's Find or Grep rows at most: its hits, each a path and a line.
-pub const search_root_output_bytes = find_max_hits * (4096 + 320);
pub const find_skip = [_][]const u8{
".git", ".jj", "target", "node_modules",
".venv", "__pycache__", ".zig-cache", "zig-out",
};
-/// A directory the walk may not open or read is passed over and counted in
-/// `notes.skipped`, its siblings' hits kept: one locked directory is not
-/// every search's failure. A cap the walk hit is noted too.
-pub fn find(arena: std.mem.Allocator, dir: []const u8, pat: []const u8, out: []u8, notes: *WalkNotes) !usize {
- var hits: [find_max_hits][]const u8 = undefined;
- var hits_len: usize = 0;
- if (platform_has_fs) {
- pardes.turn.yield();
- defer pardes.turn.back();
- const io = std.Io.Threaded.global_single_threaded.io();
- var root = try std.Io.Dir.cwd().openDir(io, dir, .{ .iterate = true });
- defer root.close(io);
- var w = try root.walkSelectively(arena);
- defer w.deinit();
- var steps: usize = 0;
- walk: while (steps < find_max_steps and hits_len < hits.len) {
- steps += 1; // an unreadable dir burns a step too, so it cannot spin
- const e = (w.next(io) catch {
- notes.skipped += 1;
- continue;
- }) orelse break;
- if (std.ascii.findIgnoreCase(e.basename, pat) != null) {
- hits[hits_len] = try arena.dupe(u8, e.path);
- hits_len += 1;
- }
- if (e.kind != .directory or e.depth() >= find_max_depth) continue;
- for (find_skip) |s| if (std.mem.eql(u8, e.basename, s)) continue :walk;
- w.enter(io, e) catch {
- notes.skipped += 1;
- };
- }
- if (hits_len >= hits.len) notes.hits_cut = true;
- if (steps >= find_max_steps) notes.walk_cut = find_max_steps;
- } else {
- for (sources) |s| {
- if (hits_len >= hits.len) break;
- if (std.ascii.findIgnoreCase(std.fs.path.basename(s.path), pat) != null) {
- hits[hits_len] = s.path;
- hits_len += 1;
- }
- }
- }
- std.mem.sort([]const u8, hits[0..hits_len], {}, struct {
- fn lt(_: void, a: []const u8, b: []const u8) bool {
- return std.mem.lessThan(u8, a, b);
- }
- }.lt);
- var written: usize = 0;
- for (hits[0..hits_len]) |h| {
- if (h.len + 1 > out.len - written) break;
- @memcpy(out[written..][0..h.len], h);
- written += h.len;
- out[written] = '\n';
- written += 1;
- }
- return written;
-}
-
-const grep_max_bytes = 256 * 1024;
+// Find and Grep walk on the disk engine's worker now (Stream.zig); what is
+// left here is what their notes and skips were.
const grep_max_files = 20_000;
-const GrepResult = struct { bytes: usize, hits: usize };
-
-/// `text` starts at line `first_line` + 1 of its file.
-fn grepText(path: []const u8, text: []const u8, pat: []const u8, out: []u8, budget: usize, first_line: usize) GrepResult {
- var result: GrepResult = .{ .bytes = 0, .hits = 0 };
- var line: usize = first_line;
- var it = std.mem.splitScalar(u8, text, '\n');
- while (it.next()) |raw| {
- line += 1;
- if (result.hits >= budget) break;
- const at = std.ascii.findIgnoreCase(raw, pat) orelse continue;
- const ln = std.mem.trimEnd(u8, raw, " \t\r");
- var cut = @min(ln.len, 200);
- while (cut > 0 and cut < ln.len and ln[cut] & 0xc0 == 0x80) cut -= 1;
- const row = std.fmt.bufPrint(out[result.bytes..], "{s}:{d}:{d}{c}{d} {s}\n", .{
- path, line, at + 1, config.range_sep, at + pat.len, ln[0..cut],
- }) catch break;
- result.bytes += row.len;
- result.hits += 1;
- }
- return result;
-}
-
-pub fn grep(arena: std.mem.Allocator, gpa: std.mem.Allocator, dir: []const u8, base: []const u8, pat: []const u8, out: []u8, notes: *WalkNotes) !usize {
- var hits: usize = 0;
- var written: usize = 0;
- if (!platform_has_fs) {
- for (sources) |s| {
- if (hits >= find_max_hits or written == out.len) break;
- const result = grepText(s.path, s.contents, pat, out[written..], find_max_hits - hits, 0);
- hits += result.hits;
- written += result.bytes;
- }
- return written;
- }
- const root_path = std.mem.trimEnd(u8, dir, "/");
- const home = std.mem.trimEnd(u8, base, "/");
- const files = try arena.alloc([]const u8, grep_max_files);
- var files_len: usize = 0;
- // A walk and thousands of reads: the turn goes out for all of it.
- pardes.turn.yield();
- defer pardes.turn.back();
- {
- const io = std.Io.Threaded.global_single_threaded.io();
- var root = try std.Io.Dir.cwd().openDir(io, dir, .{ .iterate = true });
- defer root.close(io);
- var w = try root.walkSelectively(arena);
- defer w.deinit();
- var steps: usize = 0;
- walk: while (steps < find_max_steps and files_len < files.len) {
- steps += 1;
- const e = (w.next(io) catch {
- notes.skipped += 1;
- continue;
- }) orelse break;
- if (e.kind == .directory) {
- if (e.depth() >= find_max_depth) continue;
- for (find_skip) |s| if (std.mem.eql(u8, e.basename, s)) continue :walk;
- w.enter(io, e) catch {
- notes.skipped += 1;
- };
- continue;
- }
- if (e.kind != .file) continue;
- files[files_len] = try std.fmt.allocPrint(arena, "{s}/{s}", .{ root_path, e.path });
- files_len += 1;
- }
- if (files_len >= files.len) notes.walk_cut = grep_max_files else if (steps >= find_max_steps) notes.walk_cut = find_max_steps;
- }
- std.mem.sort([]const u8, files[0..files_len], {}, struct {
- fn lt(_: void, a: []const u8, b: []const u8) bool {
- return std.mem.lessThan(u8, a, b);
- }
- }.lt);
- const buf = try gpa.alloc(u8, grep_max_bytes);
- defer gpa.free(buf);
- for (files[0..files_len]) |path| {
- if (hits >= find_max_hits or written == out.len) {
- notes.hits_cut = true;
- break;
- }
- var pathbuf: [4096]u8 = undefined;
- const path_z = std.fmt.bufPrintSentinel(&pathbuf, "{s}", .{path}, 0) catch return error.PathTooLong;
- const fd = libc.open(path_z, .{ .ACCMODE = .RDONLY, .CLOEXEC = true, .NONBLOCK = true });
- if (fd < 0) continue;
- defer _ = libc.close(fd);
- // The whole file, a buffer at a time: each searched through its last
- // whole line, the rest carried into the next.
- const shown = lsp.rel(home, path);
- var len: usize = 0;
- var lines_before: usize = 0;
- var first = true;
- file: while (true) {
- var eof = false;
- while (len < buf.len) {
- const n = libc.read(fd, buf[len..].ptr, buf.len - len);
- if (n < 0) {
- if (libc.errno(n) == .INTR) continue;
- break :file;
- }
- if (n == 0) {
- eof = true;
- break;
- }
- len += @intCast(n);
- }
- if (first and std.mem.indexOfScalar(u8, buf[0..@min(len, 1024)], 0) != null) break;
- first = false;
- // A line longer than the buffer is searched in buffer-sized parts.
- const end = if (eof) len else if (std.mem.lastIndexOfScalar(u8, buf[0..len], '\n')) |nl| nl + 1 else len;
- const result = grepText(shown, buf[0..end], pat, out[written..], find_max_hits - hits, lines_before);
- hits += result.hits;
- written += result.bytes;
- if (eof or hits >= find_max_hits or written == out.len) break;
- lines_before += std.mem.count(u8, buf[0..end], "\n");
- std.mem.copyForwards(u8, buf[0 .. len - end], buf[end..len]);
- len -= end;
- }
- if (hits >= find_max_hits) notes.hits_cut = true;
- }
- return written;
-}
-
-test "grep skips a file it cannot read instead of abandoning the search" {
- if (!platform_has_fs) return;
- const gpa = std.testing.allocator;
- var tmp = std.testing.tmpDir(.{});
- defer tmp.cleanup();
- var base_buf: [std.fs.max_path_bytes]u8 = undefined;
- const dir = base_buf[0..try tmp.dir.realPath(std.testing.io, &base_buf)];
-
- try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "a-locked.txt", .data = "needle here\n" });
- try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "b-open.txt", .data = "needle here\n" });
- var locked_buf: [std.fs.max_path_bytes]u8 = undefined;
- const locked = try std.fmt.bufPrintSentinel(&locked_buf, "{s}/a-locked.txt", .{dir}, 0);
- if (libc.chmod(locked, 0) != 0) return;
- const probe = libc.open(locked, .{ .ACCMODE = .RDONLY });
- if (probe >= 0) {
- _ = libc.close(probe);
- _ = libc.chmod(locked, 0o644);
- return error.SkipZigTest;
- }
-
- var arena: std.heap.ArenaAllocator = .init(gpa);
- defer arena.deinit();
- const out = try gpa.alloc(u8, 64 * 1024);
- defer gpa.free(out);
- var notes: WalkNotes = .{};
- const n = try grep(arena.allocator(), gpa, dir, dir, "needle", out, &notes);
- _ = libc.chmod(locked, 0o644); // so `tmp.cleanup` can remove it
-
- try std.testing.expect(std.mem.indexOf(u8, out[0..n], "b-open.txt") != null);
- try std.testing.expect(std.mem.indexOf(u8, out[0..n], "a-locked.txt") == null);
-}
-
-test "a directory the walk may not open is skipped and counted: Grep and Find keep every other hit" {
- if (!platform_has_fs) return;
- const gpa = std.testing.allocator;
- var tmp = std.testing.tmpDir(.{});
- defer tmp.cleanup();
- var base_buf: [std.fs.max_path_bytes]u8 = undefined;
- const dir = base_buf[0..try tmp.dir.realPath(std.testing.io, &base_buf)];
- try tmp.dir.createDirPath(std.testing.io, "locked");
- try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "locked/needle-in.txt", .data = "needle\n" });
- try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "needle-out.txt", .data = "needle\n" });
- var locked_buf: [std.fs.max_path_bytes]u8 = undefined;
- const locked = try std.fmt.bufPrintSentinel(&locked_buf, "{s}/locked", .{dir}, 0);
- if (libc.chmod(locked, 0) != 0) return;
- defer _ = libc.chmod(locked, 0o755); // so `tmp.cleanup` can remove it
- if (isDir(locked)) return error.SkipZigTest; // root reads it anyway
- var arena: std.heap.ArenaAllocator = .init(gpa);
- defer arena.deinit();
- const out = try gpa.alloc(u8, 64 * 1024);
- defer gpa.free(out);
- var notes: WalkNotes = .{};
- var n = try grep(arena.allocator(), gpa, dir, dir, "needle", out, &notes);
- try std.testing.expect(std.mem.indexOf(u8, out[0..n], "needle-out.txt:1:") != null);
- try std.testing.expectEqual(@as(usize, 1), notes.skipped);
- notes = .{};
- n = try find(arena.allocator(), dir, "needle", out, &notes);
- try std.testing.expect(std.mem.indexOf(u8, out[0..n], "needle-out.txt") != null);
- try std.testing.expectEqual(@as(usize, 1), notes.skipped);
-}
-
-test "Grep reads a long file to its end, and says when its hits were cut" {
- if (!platform_has_fs) return;
- const gpa = std.testing.allocator;
- var tmp = std.testing.tmpDir(.{});
- defer tmp.cleanup();
- var base_buf: [std.fs.max_path_bytes]u8 = undefined;
- const dir = base_buf[0..try tmp.dir.realPath(std.testing.io, &base_buf)];
- // A match past the first buffer of a long file, on its own line number.
- const long = try gpa.alloc(u8, grep_max_bytes + 64);
- defer gpa.free(long);
- @memset(long, 'x');
- for (0..long.len / 64) |i| long[i * 64] = '\n';
- @memcpy(long[long.len - 8 ..][0..7], "needle\n");
- try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "long.txt", .data = long });
- var arena: std.heap.ArenaAllocator = .init(gpa);
- defer arena.deinit();
- const out = try gpa.alloc(u8, 256 * 1024);
- defer gpa.free(out);
- var notes: WalkNotes = .{};
- var n = try grep(arena.allocator(), gpa, dir, dir, "needle", out, &notes);
- var want_buf: [64]u8 = undefined;
- const want = try std.fmt.bufPrint(&want_buf, "long.txt:{d}:", .{std.mem.count(u8, long, "\n")});
- try std.testing.expect(std.mem.startsWith(u8, out[0..n], want));
- // More hits than the cap.
- var many: std.ArrayList(u8) = .empty;
- defer many.deinit(gpa);
- for (0..find_max_hits + 10) |_| try many.appendSlice(gpa, "needle\n");
- try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "many.txt", .data = many.items });
- notes = .{};
- n = try grep(arena.allocator(), gpa, dir, dir, "needle", out, &notes);
- try std.testing.expect(notes.hits_cut);
- var said: [256]u8 = undefined;
- const lines = said[0..notes.write(&said)];
- try std.testing.expectEqualStrings("cut at 512 hits\n", lines);
- try std.testing.expect(WalkNotes.isNote("cut at 512 hits") and WalkNotes.isNote("2 directories skipped: permission denied") and !WalkNotes.isNote("src/2 directories.txt"));
-}
-
fn isDir(path: [*:0]const u8) bool {
const fd = libc.open(path, .{ .ACCMODE = .RDONLY, .DIRECTORY = true, .CLOEXEC = true });
if (fd < 0) return false;