Skip to content

Commit

Permalink
std.json: Unify stringify and writeStream (#16405)
Browse files Browse the repository at this point in the history
  • Loading branch information
@thejoshwolfe
thejoshwolfe authored Jul 21, 2023
1 parent a2d81c5 commit 8924f81
Show file tree
Hide file tree
Showing 16 changed files with 1,096 additions and 971 deletions.
3 changes: 2 additions & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ set(ZIG_STAGE2_SOURCES
"${CMAKE_SOURCE_DIR}/lib/std/atomic/queue.zig"
"${CMAKE_SOURCE_DIR}/lib/std/atomic/stack.zig"
"${CMAKE_SOURCE_DIR}/lib/std/base64.zig"
"${CMAKE_SOURCE_DIR}/lib/std/BitStack.zig"
"${CMAKE_SOURCE_DIR}/lib/std/buf_map.zig"
"${CMAKE_SOURCE_DIR}/lib/std/Build.zig"
"${CMAKE_SOURCE_DIR}/lib/std/Build/Cache.zig"
Expand Down Expand Up @@ -260,7 +261,7 @@ set(ZIG_STAGE2_SOURCES
"${CMAKE_SOURCE_DIR}/lib/std/io/seekable_stream.zig"
"${CMAKE_SOURCE_DIR}/lib/std/io/writer.zig"
"${CMAKE_SOURCE_DIR}/lib/std/json.zig"
"${CMAKE_SOURCE_DIR}/lib/std/json/write_stream.zig"
"${CMAKE_SOURCE_DIR}/lib/std/json/stringify.zig"
"${CMAKE_SOURCE_DIR}/lib/std/leb128.zig"
"${CMAKE_SOURCE_DIR}/lib/std/linked_list.zig"
"${CMAKE_SOURCE_DIR}/lib/std/log.zig"
Expand Down
86 changes: 86 additions & 0 deletions lib/std/BitStack.zig
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
//! Effectively a stack of u1 values implemented using ArrayList(u8).

const BitStack = @This();

const std = @import("std");
const Allocator = std.mem.Allocator;
const ArrayList = std.ArrayList;

bytes: std.ArrayList(u8),
bit_len: usize = 0,

pub fn init(allocator: Allocator) @This() {
return .{
.bytes = std.ArrayList(u8).init(allocator),
};
}

pub fn deinit(self: *@This()) void {
self.bytes.deinit();
self.* = undefined;
}

pub fn ensureTotalCapacity(self: *@This(), bit_capcity: usize) Allocator.Error!void {
const byte_capacity = (bit_capcity + 7) >> 3;
try self.bytes.ensureTotalCapacity(byte_capacity);
}

pub fn push(self: *@This(), b: u1) Allocator.Error!void {
const byte_index = self.bit_len >> 3;
if (self.bytes.items.len <= byte_index) {
try self.bytes.append(0);
}

pushWithStateAssumeCapacity(self.bytes.items, &self.bit_len, b);
}

pub fn peek(self: *const @This()) u1 {
return peekWithState(self.bytes.items, self.bit_len);
}

pub fn pop(self: *@This()) u1 {
return popWithState(self.bytes.items, &self.bit_len);
}

/// Standalone function for working with a fixed-size buffer.
pub fn pushWithStateAssumeCapacity(buf: []u8, bit_len: *usize, b: u1) void {
const byte_index = bit_len.* >> 3;
const bit_index = @as(u3, @intCast(bit_len.* & 7));

buf[byte_index] &= ~(@as(u8, 1) << bit_index);
buf[byte_index] |= @as(u8, b) << bit_index;

bit_len.* += 1;
}

/// Standalone function for working with a fixed-size buffer.
pub fn peekWithState(buf: []const u8, bit_len: usize) u1 {
const byte_index = (bit_len - 1) >> 3;
const bit_index = @as(u3, @intCast((bit_len - 1) & 7));
return @as(u1, @intCast((buf[byte_index] >> bit_index) & 1));
}

/// Standalone function for working with a fixed-size buffer.
pub fn popWithState(buf: []const u8, bit_len: *usize) u1 {
const b = peekWithState(buf, bit_len.*);
bit_len.* -= 1;
return b;
}

const testing = std.testing;
test BitStack {
var stack = BitStack.init(testing.allocator);
defer stack.deinit();

try stack.push(1);
try stack.push(0);
try stack.push(0);
try stack.push(1);

try testing.expectEqual(@as(u1, 1), stack.peek());
try testing.expectEqual(@as(u1, 1), stack.pop());
try testing.expectEqual(@as(u1, 0), stack.peek());
try testing.expectEqual(@as(u1, 0), stack.pop());
try testing.expectEqual(@as(u1, 0), stack.pop());
try testing.expectEqual(@as(u1, 1), stack.pop());
}
23 changes: 13 additions & 10 deletions lib/std/json.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,15 @@ test Value {
test writeStream {
var out = ArrayList(u8).init(testing.allocator);
defer out.deinit();
var write_stream = writeStream(out.writer(), 99);
var write_stream = writeStream(out.writer(), .{ .whitespace = .indent_2 });
defer write_stream.deinit();
try write_stream.beginObject();
try write_stream.objectField("foo");
try write_stream.emitNumber(123);
try write_stream.write(123);
try write_stream.endObject();
const expected =
\\{
\\ "foo": 123
\\ "foo": 123
\\}
;
try testing.expectEqualSlices(u8, expected, out.items);
Expand Down Expand Up @@ -98,13 +99,16 @@ pub const ParseError = @import("json/static.zig").ParseError;
pub const ParseFromValueError = @import("json/static.zig").ParseFromValueError;

pub const StringifyOptions = @import("json/stringify.zig").StringifyOptions;
pub const encodeJsonString = @import("json/stringify.zig").encodeJsonString;
pub const encodeJsonStringChars = @import("json/stringify.zig").encodeJsonStringChars;
pub const stringify = @import("json/stringify.zig").stringify;
pub const stringifyMaxDepth = @import("json/stringify.zig").stringifyMaxDepth;
pub const stringifyArbitraryDepth = @import("json/stringify.zig").stringifyArbitraryDepth;
pub const stringifyAlloc = @import("json/stringify.zig").stringifyAlloc;

pub const WriteStream = @import("json/write_stream.zig").WriteStream;
pub const writeStream = @import("json/write_stream.zig").writeStream;
pub const writeStream = @import("json/stringify.zig").writeStream;
pub const writeStreamMaxDepth = @import("json/stringify.zig").writeStreamMaxDepth;
pub const writeStreamArbitraryDepth = @import("json/stringify.zig").writeStreamArbitraryDepth;
pub const WriteStream = @import("json/stringify.zig").WriteStream;
pub const encodeJsonString = @import("json/stringify.zig").encodeJsonString;
pub const encodeJsonStringChars = @import("json/stringify.zig").encodeJsonStringChars;

// Deprecations
pub const parse = @compileError("Deprecated; use parseFromSlice() or parseFromTokenSource() instead.");
Expand All @@ -117,9 +121,8 @@ pub const TokenStream = @compileError("Deprecated; use json.Scanner or json.Read
test {
_ = @import("json/test.zig");
_ = @import("json/scanner.zig");
_ = @import("json/write_stream.zig");
_ = @import("json/dynamic.zig");
_ = @import("json/hashmap_test.zig");
_ = @import("json/hashmap.zig");
_ = @import("json/static.zig");
_ = @import("json/stringify.zig");
_ = @import("json/JSONTestSuite_test.zig");
Expand Down
45 changes: 12 additions & 33 deletions lib/std/json/dynamic.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,44 +59,23 @@ pub const Value = union(enum) {
stringify(self, .{}, stderr) catch return;
}

pub fn jsonStringify(
value: @This(),
options: StringifyOptions,
out_stream: anytype,
) @TypeOf(out_stream).Error!void {
pub fn jsonStringify(value: @This(), jws: anytype) !void {
switch (value) {
.null => try stringify(null, options, out_stream),
.bool => |inner| try stringify(inner, options, out_stream),
.integer => |inner| try stringify(inner, options, out_stream),
.float => |inner| try stringify(inner, options, out_stream),
.number_string => |inner| try out_stream.writeAll(inner),
.string => |inner| try stringify(inner, options, out_stream),
.array => |inner| try stringify(inner.items, options, out_stream),
.null => try jws.write(null),
.bool => |inner| try jws.write(inner),
.integer => |inner| try jws.write(inner),
.float => |inner| try jws.write(inner),
.number_string => |inner| try jws.writePreformatted(inner),
.string => |inner| try jws.write(inner),
.array => |inner| try jws.write(inner.items),
.object => |inner| {
try out_stream.writeByte('{');
var field_output = false;
var child_options = options;
child_options.whitespace.indent_level += 1;
try jws.beginObject();
var it = inner.iterator();
while (it.next()) |entry| {
if (!field_output) {
field_output = true;
} else {
try out_stream.writeByte(',');
}
try child_options.whitespace.outputIndent(out_stream);

try stringify(entry.key_ptr.*, options, out_stream);
try out_stream.writeByte(':');
if (child_options.whitespace.separator) {
try out_stream.writeByte(' ');
}
try stringify(entry.value_ptr.*, child_options, out_stream);
}
if (field_output) {
try options.whitespace.outputIndent(out_stream);
try jws.objectField(entry.key_ptr.*);
try jws.write(entry.value_ptr.*);
}
try out_stream.writeByte('}');
try jws.endObject();
},
}
}
Expand Down
124 changes: 53 additions & 71 deletions lib/std/json/dynamic_test.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,38 +69,34 @@ test "json.parser.dynamic" {
try testing.expect(mem.eql(u8, large_int.number_string, "18446744073709551615"));
}

const writeStream = @import("./write_stream.zig").writeStream;
const writeStream = @import("./stringify.zig").writeStream;
test "write json then parse it" {
var out_buffer: [1000]u8 = undefined;

var fixed_buffer_stream = std.io.fixedBufferStream(&out_buffer);
const out_stream = fixed_buffer_stream.writer();
var jw = writeStream(out_stream, 4);
var jw = writeStream(out_stream, .{});
defer jw.deinit();

try jw.beginObject();

try jw.objectField("f");
try jw.emitBool(false);
try jw.write(false);

try jw.objectField("t");
try jw.emitBool(true);
try jw.write(true);

try jw.objectField("int");
try jw.emitNumber(1234);
try jw.write(1234);

try jw.objectField("array");
try jw.beginArray();

try jw.arrayElem();
try jw.emitNull();

try jw.arrayElem();
try jw.emitNumber(12.34);

try jw.write(null);
try jw.write(12.34);
try jw.endArray();

try jw.objectField("str");
try jw.emitString("hello");
try jw.write("hello");

try jw.endObject();

Expand Down Expand Up @@ -185,64 +181,50 @@ test "escaped characters" {
}

test "Value.jsonStringify" {
{
var buffer: [10]u8 = undefined;
var fbs = std.io.fixedBufferStream(&buffer);
try @as(Value, .null).jsonStringify(.{}, fbs.writer());
try testing.expectEqualSlices(u8, fbs.getWritten(), "null");
}
{
var buffer: [10]u8 = undefined;
var fbs = std.io.fixedBufferStream(&buffer);
try (Value{ .bool = true }).jsonStringify(.{}, fbs.writer());
try testing.expectEqualSlices(u8, fbs.getWritten(), "true");
}
{
var buffer: [10]u8 = undefined;
var fbs = std.io.fixedBufferStream(&buffer);
try (Value{ .integer = 42 }).jsonStringify(.{}, fbs.writer());
try testing.expectEqualSlices(u8, fbs.getWritten(), "42");
}
{
var buffer: [10]u8 = undefined;
var fbs = std.io.fixedBufferStream(&buffer);
try (Value{ .number_string = "43" }).jsonStringify(.{}, fbs.writer());
try testing.expectEqualSlices(u8, fbs.getWritten(), "43");
}
{
var buffer: [10]u8 = undefined;
var fbs = std.io.fixedBufferStream(&buffer);
try (Value{ .float = 42 }).jsonStringify(.{}, fbs.writer());
try testing.expectEqualSlices(u8, fbs.getWritten(), "4.2e+01");
}
{
var buffer: [10]u8 = undefined;
var fbs = std.io.fixedBufferStream(&buffer);
try (Value{ .string = "weeee" }).jsonStringify(.{}, fbs.writer());
try testing.expectEqualSlices(u8, fbs.getWritten(), "\"weeee\"");
}
{
var buffer: [10]u8 = undefined;
var fbs = std.io.fixedBufferStream(&buffer);
var vals = [_]Value{
.{ .integer = 1 },
.{ .integer = 2 },
.{ .number_string = "3" },
};
try (Value{
.array = Array.fromOwnedSlice(undefined, &vals),
}).jsonStringify(.{}, fbs.writer());
try testing.expectEqualSlices(u8, fbs.getWritten(), "[1,2,3]");
}
{
var buffer: [10]u8 = undefined;
var fbs = std.io.fixedBufferStream(&buffer);
var obj = ObjectMap.init(testing.allocator);
defer obj.deinit();
try obj.putNoClobber("a", .{ .string = "b" });
try (Value{ .object = obj }).jsonStringify(.{}, fbs.writer());
try testing.expectEqualSlices(u8, fbs.getWritten(), "{\"a\":\"b\"}");
}
var vals = [_]Value{
.{ .integer = 1 },
.{ .integer = 2 },
.{ .number_string = "3" },
};
var obj = ObjectMap.init(testing.allocator);
defer obj.deinit();
try obj.putNoClobber("a", .{ .string = "b" });
var array = [_]Value{
Value.null,
Value{ .bool = true },
Value{ .integer = 42 },
Value{ .number_string = "43" },
Value{ .float = 42 },
Value{ .string = "weeee" },
Value{ .array = Array.fromOwnedSlice(undefined, &vals) },
Value{ .object = obj },
};
var buffer: [0x1000]u8 = undefined;
var fbs = std.io.fixedBufferStream(&buffer);

var jw = writeStream(fbs.writer(), .{ .whitespace = .indent_1 });
defer jw.deinit();
try jw.write(array);

const expected =
\\[
\\ null,
\\ true,
\\ 42,
\\ 43,
\\ 4.2e+01,
\\ "weeee",
\\ [
\\ 1,
\\ 2,
\\ 3
\\ ],
\\ {
\\ "a": "b"
\\ }
\\]
;
try testing.expectEqualSlices(u8, expected, fbs.getWritten());
}

test "parseFromValue(std.json.Value,...)" {
Expand Down
Loading

0 comments on commit 8924f81

Please sign in to comment.