Make parser use AnyReader.

2 files changed, 59 insertions, 15 deletions

diff --git a/src/libzisp.zig b/src/libzisp.zig
index d610f4e..f67f568 100644
--- a/src/libzisp.zig
+++ b/src/libzisp.zig
@@ -246,8 +246,13 @@ test "pair" {
     try std.testing.expectEqual(4, value.fixnum.unpack(cdr2));
 }
 
+fn parseString(str: []const u8) Value {
+    var fbs = std.io.fixedBufferStream(str);
+    return io.parser.parse(fbs.reader().any());
+}
+
 test "parse" {
-    const val = io.parser.parse("\"foo\"");
+    const val = parseString("\"foo\"");
 
     try std.testing.expect(value.sstr.check(val));
 
@@ -256,7 +261,7 @@ test "parse" {
 }
 
 test "parse2" {
-    const val = io.parser.parse(
+    const val = parseString(
         \\ ;; Testing some crazy datum comments
         \\ #;"bar"#;([x #"y"]{##`,'z}) #"foo"
         \\ ;; end
@@ -273,7 +278,7 @@ test "parse2" {
 }
 
 test "parse3" {
-    const val = io.parser.parse(
+    const val = parseString(
         \\(foo #;x #;(x y) #;x #bar [#x #"baz"] 'bat)
     );
 
@@ -289,7 +294,7 @@ test "parse3" {
 }
 
 test "parse4" {
-    const val = io.parser.parse("(foo . #;x bar #;y)");
+    const val = parseString("(foo . #;x bar #;y)");
 
     const s = value.sstr.unpack(value.pair.car(val));
     try std.testing.expectEqualStrings("foo", s.slice());
@@ -308,7 +313,7 @@ test "parse bench" {
     std.mem.doNotOptimizeAway(timer.lap());
     for (0..iters) |i| {
         _ = i;
-        std.mem.doNotOptimizeAway(io.parser.parse(
+        std.mem.doNotOptimizeAway(parseString(
             \\(a b c (x y z (a b c (x y z (a b c (x y z (a b c (x y z (a b c
             \\(x y z (a b c (x y z (a b c (x y z) d e f) d e f) d e f) d e f)
             \\d e f) d e f) d e f) d e f) d e f) d e f) d e f) 1 2 3))
@@ -326,7 +331,7 @@ test "unparse" {
     var out: std.ArrayList(u8) = .init(gpa.allocator());
 
     const w = out.writer();
-    const v = io.parser.parse("#foo");
+    const v = parseString("#foo");
     try unparse(w, v);
     try std.testing.expectEqualStrings("#foo", try out.toOwnedSlice());
 }
@@ -334,6 +339,6 @@ test "unparse" {
 test "unparse2" {
     try io.unparser.unparse(
         std.io.getStdErr().writer(),
-        io.parser.parse("#{foo bar['x]}"),
+        parseString("#{foo bar['x]}"),
     );
 }
diff --git a/src/libzisp/io/parser.zig b/src/libzisp/io/parser.zig
index 773e246..839b046 100644
--- a/src/libzisp/io/parser.zig
+++ b/src/libzisp/io/parser.zig
@@ -240,15 +240,28 @@ const Context = struct {
     char: u8 = undefined,
 };
 
+// Buffer size is 4096, so index type is u12 (0..4095).
+const BUF_SIZE = 4096;
+const IDX_TYPE = u12;
+
+const debug_mode = @import("builtin").mode == .Debug;
+
 const State = struct {
-    input: []const u8,
-    pos: usize = 0,
+    input: std.io.AnyReader,
+    is_eof: bool = false,
+
+    buf: [BUF_SIZE]u8 = .{0} ** BUF_SIZE,
+    pos: IDX_TYPE = 0,
+    write_pos: IDX_TYPE = 0,
 
     context: Context = .{},
     stack: std.ArrayList(Context),
     retval: Value = undefined,
 
-    fn init(input: []const u8, alloc: std.mem.Allocator) State {
+    // For debugging.
+    checked_eof: bool = false,
+
+    fn init(input: std.io.AnyReader, alloc: std.mem.Allocator) State {
         return .{ .input = input, .stack = .init(alloc) };
     }
 
@@ -274,17 +287,43 @@ const State = struct {
         }
     }
 
+    fn fillBuffer(s: *State) !void {
+        if (s.pos != s.write_pos) {
+            return;
+        }
+        // Make sure *not* to overwrite the entire buffer with newly read data,
+        // because we use it as a circular buffer so as to be able to reset to
+        // previous points in it.  We overwrite at most BUF_SIZE / 2 bytes, so
+        // we can go back by up to BUF_SIZE / 2 bytes.
+        const avail = BUF_SIZE - @as(u64, s.write_pos);
+        const write_max = std.math.clamp(avail, 0, BUF_SIZE / 2);
+        const write_to = s.write_pos + write_max;
+        const count = try s.input.read(s.buf[s.write_pos..write_to]);
+        s.is_eof = count == 0;
+        s.write_pos +%= @intCast(count);
+    }
+
     fn eof(s: *State) bool {
-        return s.pos >= s.input.len;
+        if (debug_mode) {
+            s.checked_eof = true;
+        }
+        fillBuffer(s) catch @panic("reader error");
+        return s.is_eof;
     }
 
     fn peek(s: *State) u8 {
-        return s.input[s.pos];
+        return s.buf[s.pos];
     }
 
     fn skip(s: *State) void {
-        // std.debug.print("{c}\n", .{s.input[s.pos]});
-        s.pos += 1;
+        if (debug_mode) {
+            if (!s.checked_eof) {
+                @panic("Didn't check EOF before calling skip()!");
+            }
+            s.checked_eof = false;
+        }
+        // std.debug.print("{c}\n", .{s.buf[s.pos]});
+        s.pos +%= 1;
     }
 
     fn getc(s: *State) u8 {
@@ -352,7 +391,7 @@ const Fn = enum {
     done,
 };
 
-pub fn parse(input: []const u8) Value {
+pub fn parse(input: std.io.AnyReader) Value {
     var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init;
     defer if (gpa.deinit() == .leak) @panic("leak");
     var s: State = .init(input, gpa.allocator());