test harness improvements
* `-Dskip-compile-errors` is removed; `-Dskip-stage1` is added. * Use `std.testing.allocator` instead of a new instance of GPA. - Fix the memory leaks this revealed. * Show the file name when it is not parsed correctly such as when the manifest is missing. - Better error messages when test files are not parsed correctly. * Ignore unknown files such as swap files. * Move logic from declarative file to the test harness implementation. * Move stage1 tests to stage2 tests where appropriate.
This commit is contained in:
parent
df1ba38a88
commit
243afdcdf5
|
@ -54,7 +54,7 @@ pub fn build(b: *Builder) !void {
|
|||
const skip_release_safe = b.option(bool, "skip-release-safe", "Main test suite skips release-safe builds") orelse skip_release;
|
||||
const skip_non_native = b.option(bool, "skip-non-native", "Main test suite skips non-native builds") orelse false;
|
||||
const skip_libc = b.option(bool, "skip-libc", "Main test suite skips tests that link libc") orelse false;
|
||||
const skip_compile_errors = b.option(bool, "skip-compile-errors", "Main test suite skips compile error tests") orelse false;
|
||||
const skip_stage1 = b.option(bool, "skip-stage1", "Main test suite skips stage1 compile error tests") orelse false;
|
||||
const skip_run_translated_c = b.option(bool, "skip-run-translated-c", "Main test suite skips run-translated-c tests") orelse false;
|
||||
const skip_stage2_tests = b.option(bool, "skip-stage2-tests", "Main test suite skips self-hosted compiler tests") orelse false;
|
||||
const skip_install_lib_files = b.option(bool, "skip-install-lib-files", "Do not copy lib/ files to installation prefix") orelse false;
|
||||
|
@ -386,7 +386,7 @@ pub fn build(b: *Builder) !void {
|
|||
test_stage2_options.addOption(bool, "enable_logging", enable_logging);
|
||||
test_stage2_options.addOption(bool, "enable_link_snapshots", enable_link_snapshots);
|
||||
test_stage2_options.addOption(bool, "skip_non_native", skip_non_native);
|
||||
test_stage2_options.addOption(bool, "skip_compile_errors", skip_compile_errors);
|
||||
test_stage2_options.addOption(bool, "skip_stage1", skip_stage1);
|
||||
test_stage2_options.addOption(bool, "is_stage1", is_stage1);
|
||||
test_stage2_options.addOption(bool, "omit_stage2", omit_stage2);
|
||||
test_stage2_options.addOption(bool, "have_llvm", enable_llvm);
|
||||
|
|
159
src/test.zig
159
src/test.zig
|
@ -12,7 +12,7 @@ const enable_wasmtime: bool = build_options.enable_wasmtime;
|
|||
const enable_darling: bool = build_options.enable_darling;
|
||||
const enable_rosetta: bool = build_options.enable_rosetta;
|
||||
const glibc_runtimes_dir: ?[]const u8 = build_options.glibc_runtimes_dir;
|
||||
const skip_compile_errors = build_options.skip_compile_errors;
|
||||
const skip_stage1 = build_options.skip_stage1;
|
||||
const ThreadPool = @import("ThreadPool.zig");
|
||||
const CrossTarget = std.zig.CrossTarget;
|
||||
const print = std.debug.print;
|
||||
|
@ -20,7 +20,6 @@ const assert = std.debug.assert;
|
|||
|
||||
const zig_h = link.File.C.zig_h;
|
||||
|
||||
var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{}){};
|
||||
const hr = "=" ** 80;
|
||||
|
||||
test {
|
||||
|
@ -28,9 +27,50 @@ test {
|
|||
@import("stage1.zig").os_init();
|
||||
}
|
||||
|
||||
var ctx = TestContext.init();
|
||||
var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena_allocator.deinit();
|
||||
const arena = arena_allocator.allocator();
|
||||
|
||||
var ctx = TestContext.init(std.testing.allocator, arena);
|
||||
defer ctx.deinit();
|
||||
|
||||
const compile_errors_dir_path = try std.fs.path.join(arena, &.{
|
||||
std.fs.path.dirname(@src().file).?, "..", "test", "compile_errors",
|
||||
});
|
||||
|
||||
var compile_errors_dir = try std.fs.cwd().openDir(compile_errors_dir_path, .{});
|
||||
defer compile_errors_dir.close();
|
||||
|
||||
{
|
||||
var stage2_dir = try compile_errors_dir.openDir("stage2", .{ .iterate = true });
|
||||
defer stage2_dir.close();
|
||||
|
||||
// TODO make this incremental once the bug is solved that it triggers
|
||||
ctx.addErrorCasesFromDir("stage2", stage2_dir, .stage2, .Obj, false, .independent);
|
||||
}
|
||||
|
||||
if (!skip_stage1) {
|
||||
var stage1_dir = try compile_errors_dir.openDir("stage1", .{});
|
||||
defer stage1_dir.close();
|
||||
|
||||
const Config = struct {
|
||||
name: []const u8,
|
||||
is_test: bool,
|
||||
output_mode: std.builtin.OutputMode,
|
||||
};
|
||||
|
||||
for ([_]Config{
|
||||
.{ .name = "obj", .is_test = false, .output_mode = .Obj },
|
||||
.{ .name = "exe", .is_test = false, .output_mode = .Exe },
|
||||
.{ .name = "test", .is_test = true, .output_mode = .Exe },
|
||||
}) |config| {
|
||||
var dir = try stage1_dir.openDir(config.name, .{ .iterate = true });
|
||||
defer dir.close();
|
||||
|
||||
ctx.addErrorCasesFromDir("stage1", dir, .stage1, config.output_mode, config.is_test, .independent);
|
||||
}
|
||||
}
|
||||
|
||||
try @import("test_cases").addCases(&ctx);
|
||||
|
||||
try ctx.run();
|
||||
|
@ -114,6 +154,7 @@ const ErrorMsg = union(enum) {
|
|||
};
|
||||
|
||||
pub const TestContext = struct {
|
||||
arena: Allocator,
|
||||
cases: std.ArrayList(Case),
|
||||
|
||||
pub const Update = struct {
|
||||
|
@ -316,7 +357,7 @@ pub const TestContext = struct {
|
|||
.target = target,
|
||||
.updates = std.ArrayList(Update).init(ctx.cases.allocator),
|
||||
.output_mode = .Exe,
|
||||
.files = std.ArrayList(File).init(ctx.cases.allocator),
|
||||
.files = std.ArrayList(File).init(ctx.arena),
|
||||
}) catch @panic("out of memory");
|
||||
return &ctx.cases.items[ctx.cases.items.len - 1];
|
||||
}
|
||||
|
@ -327,7 +368,7 @@ pub const TestContext = struct {
|
|||
}
|
||||
|
||||
pub fn exeFromCompiledC(ctx: *TestContext, name: []const u8, target: CrossTarget) *Case {
|
||||
const prefixed_name = std.fmt.allocPrint(ctx.cases.allocator, "CBE: {s}", .{name}) catch
|
||||
const prefixed_name = std.fmt.allocPrint(ctx.arena, "CBE: {s}", .{name}) catch
|
||||
@panic("out of memory");
|
||||
ctx.cases.append(Case{
|
||||
.name = prefixed_name,
|
||||
|
@ -335,7 +376,7 @@ pub const TestContext = struct {
|
|||
.updates = std.ArrayList(Update).init(ctx.cases.allocator),
|
||||
.output_mode = .Exe,
|
||||
.object_format = .c,
|
||||
.files = std.ArrayList(File).init(ctx.cases.allocator),
|
||||
.files = std.ArrayList(File).init(ctx.arena),
|
||||
}) catch @panic("out of memory");
|
||||
return &ctx.cases.items[ctx.cases.items.len - 1];
|
||||
}
|
||||
|
@ -348,7 +389,7 @@ pub const TestContext = struct {
|
|||
.target = target,
|
||||
.updates = std.ArrayList(Update).init(ctx.cases.allocator),
|
||||
.output_mode = .Exe,
|
||||
.files = std.ArrayList(File).init(ctx.cases.allocator),
|
||||
.files = std.ArrayList(File).init(ctx.arena),
|
||||
.backend = .llvm,
|
||||
.link_libc = true,
|
||||
}) catch @panic("out of memory");
|
||||
|
@ -365,7 +406,7 @@ pub const TestContext = struct {
|
|||
.target = target,
|
||||
.updates = std.ArrayList(Update).init(ctx.cases.allocator),
|
||||
.output_mode = .Obj,
|
||||
.files = std.ArrayList(File).init(ctx.cases.allocator),
|
||||
.files = std.ArrayList(File).init(ctx.arena),
|
||||
}) catch @panic("out of memory");
|
||||
return &ctx.cases.items[ctx.cases.items.len - 1];
|
||||
}
|
||||
|
@ -381,7 +422,7 @@ pub const TestContext = struct {
|
|||
.updates = std.ArrayList(Update).init(ctx.cases.allocator),
|
||||
.output_mode = .Exe,
|
||||
.is_test = true,
|
||||
.files = std.ArrayList(File).init(ctx.cases.allocator),
|
||||
.files = std.ArrayList(File).init(ctx.arena),
|
||||
}) catch @panic("out of memory");
|
||||
return &ctx.cases.items[ctx.cases.items.len - 1];
|
||||
}
|
||||
|
@ -404,7 +445,7 @@ pub const TestContext = struct {
|
|||
.updates = std.ArrayList(Update).init(ctx.cases.allocator),
|
||||
.output_mode = .Obj,
|
||||
.object_format = .c,
|
||||
.files = std.ArrayList(File).init(ctx.cases.allocator),
|
||||
.files = std.ArrayList(File).init(ctx.arena),
|
||||
}) catch @panic("out of memory");
|
||||
return &ctx.cases.items[ctx.cases.items.len - 1];
|
||||
}
|
||||
|
@ -423,7 +464,7 @@ pub const TestContext = struct {
|
|||
src: [:0]const u8,
|
||||
expected_errors: []const []const u8,
|
||||
) void {
|
||||
if (skip_compile_errors) return;
|
||||
if (skip_stage1) return;
|
||||
|
||||
const case = ctx.addObj(name, .{});
|
||||
case.backend = .stage1;
|
||||
|
@ -436,7 +477,7 @@ pub const TestContext = struct {
|
|||
src: [:0]const u8,
|
||||
expected_errors: []const []const u8,
|
||||
) void {
|
||||
if (skip_compile_errors) return;
|
||||
if (skip_stage1) return;
|
||||
|
||||
const case = ctx.addTest(name, .{});
|
||||
case.backend = .stage1;
|
||||
|
@ -449,7 +490,7 @@ pub const TestContext = struct {
|
|||
src: [:0]const u8,
|
||||
expected_errors: []const []const u8,
|
||||
) void {
|
||||
if (skip_compile_errors) return;
|
||||
if (skip_stage1) return;
|
||||
|
||||
const case = ctx.addExe(name, .{});
|
||||
case.backend = .stage1;
|
||||
|
@ -612,6 +653,8 @@ pub const TestContext = struct {
|
|||
case.compiles(fixed_src);
|
||||
}
|
||||
|
||||
const Strategy = enum { incremental, independent };
|
||||
|
||||
/// Adds a compile-error test for each file in the provided directory, using the
|
||||
/// selected backend and output mode. If `one_test_case_per_file` is true, a new
|
||||
/// test case is created for each file. Otherwise, a single test case is used for
|
||||
|
@ -628,33 +671,58 @@ pub const TestContext = struct {
|
|||
backend: Backend,
|
||||
output_mode: std.builtin.OutputMode,
|
||||
is_test: bool,
|
||||
one_test_case_per_file: bool,
|
||||
) !void {
|
||||
if (skip_compile_errors) return;
|
||||
strategy: Strategy,
|
||||
) void {
|
||||
var current_file: []const u8 = "none";
|
||||
addErrorCasesFromDirInner(ctx, name, dir, backend, output_mode, is_test, strategy, ¤t_file) catch |err| {
|
||||
std.debug.panic("test harness failed to process file '{s}': {s}\n", .{
|
||||
current_file, @errorName(err),
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
const gpa = general_purpose_allocator.allocator();
|
||||
fn addErrorCasesFromDirInner(
|
||||
ctx: *TestContext,
|
||||
name: []const u8,
|
||||
dir: std.fs.Dir,
|
||||
backend: Backend,
|
||||
output_mode: std.builtin.OutputMode,
|
||||
is_test: bool,
|
||||
strategy: Strategy,
|
||||
/// This is kept up to date with the currently being processed file so
|
||||
/// that if any errors occur the caller knows it happened during this file.
|
||||
current_file: *[]const u8,
|
||||
) !void {
|
||||
var opt_case: ?*Case = null;
|
||||
|
||||
var it = dir.iterate();
|
||||
while (try it.next()) |entry| {
|
||||
if (entry.kind != .File) continue;
|
||||
|
||||
var contents = try dir.readFileAlloc(gpa, entry.name, std.math.maxInt(u32));
|
||||
defer gpa.free(contents);
|
||||
// Ignore stuff such as .swp files
|
||||
switch (Compilation.classifyFileExt(entry.name)) {
|
||||
.unknown => continue,
|
||||
else => {},
|
||||
}
|
||||
|
||||
current_file.* = try ctx.arena.dupe(u8, entry.name);
|
||||
|
||||
const max_file_size = 10 * 1024 * 1024;
|
||||
const src = try dir.readFileAllocOptions(ctx.arena, entry.name, max_file_size, null, 1, 0);
|
||||
|
||||
// The manifest is the last contiguous block of comments in the file
|
||||
// We scan for the beginning by searching backward for the first non-empty line that does not start with "//"
|
||||
var manifest_start: ?usize = null;
|
||||
var manifest_end: usize = contents.len;
|
||||
if (contents.len > 0) {
|
||||
var cursor: usize = contents.len - 1;
|
||||
var manifest_end: usize = src.len;
|
||||
if (src.len > 0) {
|
||||
var cursor: usize = src.len - 1;
|
||||
while (true) {
|
||||
// Move to beginning of line
|
||||
while (cursor > 0 and contents[cursor - 1] != '\n') cursor -= 1;
|
||||
while (cursor > 0 and src[cursor - 1] != '\n') cursor -= 1;
|
||||
|
||||
// Check if line is non-empty and does not start with "//"
|
||||
if (cursor + 1 < contents.len and contents[cursor + 1] != '\n' and contents[cursor + 1] != '\r') {
|
||||
if (std.mem.startsWith(u8, contents[cursor..], "//")) {
|
||||
if (cursor + 1 < src.len and src[cursor + 1] != '\n' and src[cursor + 1] != '\r') {
|
||||
if (std.mem.startsWith(u8, src[cursor..], "//")) {
|
||||
manifest_start = cursor;
|
||||
} else {
|
||||
break;
|
||||
|
@ -666,27 +734,23 @@ pub const TestContext = struct {
|
|||
}
|
||||
}
|
||||
|
||||
var errors = std.ArrayList([]const u8).init(gpa);
|
||||
defer errors.deinit();
|
||||
var errors = std.ArrayList([]const u8).init(ctx.arena);
|
||||
|
||||
if (manifest_start) |start| {
|
||||
// Due to the above processing, we know that this is a contiguous block of comments
|
||||
var manifest_it = std.mem.tokenize(u8, contents[start..manifest_end], "\r\n");
|
||||
var manifest_it = std.mem.tokenize(u8, src[start..manifest_end], "\r\n");
|
||||
|
||||
// First line is the test case name
|
||||
const first_line = manifest_it.next() orelse return error.InvalidFile;
|
||||
const case_name = try std.mem.concat(gpa, u8, &.{ name, ": ", std.mem.trim(u8, first_line[2..], " \t") });
|
||||
const first_line = manifest_it.next() orelse return error.MissingTestCaseName;
|
||||
const case_name = try std.mem.concat(ctx.arena, u8, &.{ name, ": ", std.mem.trim(u8, first_line[2..], " \t") });
|
||||
|
||||
// If the second line is present, it should be blank
|
||||
if (manifest_it.next()) |second_line| {
|
||||
if (std.mem.trim(u8, second_line[2..], " \t").len != 0) return error.InvalidFile;
|
||||
if (std.mem.trim(u8, second_line[2..], " \t").len != 0) return error.SecondLineNotBlank;
|
||||
}
|
||||
|
||||
// All following lines are expected error messages
|
||||
while (manifest_it.next()) |line| try errors.append(try gpa.dupe(u8, std.mem.trim(u8, line[2..], " \t")));
|
||||
|
||||
// The entire file contents is the source, including the manifest
|
||||
const src = try gpa.dupeZ(u8, contents);
|
||||
while (manifest_it.next()) |line| try errors.append(try ctx.arena.dupe(u8, std.mem.trim(u8, line[2..], " \t")));
|
||||
|
||||
const case = opt_case orelse case: {
|
||||
ctx.cases.append(TestContext.Case{
|
||||
|
@ -702,22 +766,27 @@ pub const TestContext = struct {
|
|||
opt_case = case;
|
||||
break :case case;
|
||||
};
|
||||
if (one_test_case_per_file) {
|
||||
case.name = case_name;
|
||||
case.addError(src, errors.items);
|
||||
opt_case = null;
|
||||
} else {
|
||||
case.addErrorNamed(case_name, src, errors.items);
|
||||
switch (strategy) {
|
||||
.independent => {
|
||||
case.name = case_name;
|
||||
case.addError(src, errors.items);
|
||||
opt_case = null;
|
||||
},
|
||||
.incremental => {
|
||||
case.addErrorNamed(case_name, src, errors.items);
|
||||
},
|
||||
}
|
||||
} else {
|
||||
return error.InvalidFile; // Manifests are currently mandatory
|
||||
return error.MissingManifest;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn init() TestContext {
|
||||
const allocator = std.heap.page_allocator;
|
||||
return .{ .cases = std.ArrayList(Case).init(allocator) };
|
||||
fn init(gpa: Allocator, arena: Allocator) TestContext {
|
||||
return .{
|
||||
.cases = std.ArrayList(Case).init(gpa),
|
||||
.arena = arena,
|
||||
};
|
||||
}
|
||||
|
||||
fn deinit(self: *TestContext) void {
|
||||
|
|
|
@ -3,44 +3,6 @@ const builtin = @import("builtin");
|
|||
const TestContext = @import("../src/test.zig").TestContext;
|
||||
|
||||
pub fn addCases(ctx: *TestContext) !void {
|
||||
var parent_dir = try std.fs.cwd().openDir(std.fs.path.dirname(@src().file).?, .{ .no_follow = true });
|
||||
defer parent_dir.close();
|
||||
|
||||
var compile_errors_dir = try parent_dir.openDir("compile_errors", .{ .no_follow = true });
|
||||
defer compile_errors_dir.close();
|
||||
|
||||
{
|
||||
var stage2_dir = try compile_errors_dir.openDir("stage2", .{ .iterate = true, .no_follow = true });
|
||||
defer stage2_dir.close();
|
||||
|
||||
// TODO make this false once the bug is solved that it triggers
|
||||
const one_test_case_per_file = true;
|
||||
try ctx.addErrorCasesFromDir("stage2", stage2_dir, .stage2, .Obj, false, one_test_case_per_file);
|
||||
}
|
||||
|
||||
{
|
||||
var stage1_dir = try compile_errors_dir.openDir("stage1", .{ .no_follow = true });
|
||||
defer stage1_dir.close();
|
||||
{
|
||||
const one_test_case_per_file = true;
|
||||
|
||||
var obj_dir = try stage1_dir.openDir("obj", .{ .iterate = true, .no_follow = true });
|
||||
defer obj_dir.close();
|
||||
|
||||
try ctx.addErrorCasesFromDir("stage1", obj_dir, .stage1, .Obj, false, one_test_case_per_file);
|
||||
|
||||
var exe_dir = try stage1_dir.openDir("exe", .{ .iterate = true, .no_follow = true });
|
||||
defer exe_dir.close();
|
||||
|
||||
try ctx.addErrorCasesFromDir("stage1", exe_dir, .stage1, .Exe, false, one_test_case_per_file);
|
||||
|
||||
var test_dir = try stage1_dir.openDir("test", .{ .iterate = true, .no_follow = true });
|
||||
defer test_dir.close();
|
||||
|
||||
try ctx.addErrorCasesFromDir("stage1", test_dir, .stage1, .Exe, true, one_test_case_per_file);
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
const case = ctx.obj("callconv(.Interrupt) on unsupported platform", .{
|
||||
.cpu_arch = .aarch64,
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
pub fn main() !void {
|
||||
export fn entry() void {
|
||||
@import("std").debug.print("{d} {d} {d} {d} {d}", .{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15});
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
comptime {
|
||||
blk: { blk: while (false) {} }
|
||||
}
|
||||
comptime {
|
||||
blk: while (false) { blk: for (@as([0]void, undefined)) |_| {} }
|
||||
}
|
||||
comptime {
|
||||
blk: for (@as([0]void, undefined)) |_| { blk: {} }
|
||||
}
|
||||
comptime {
|
||||
blk: {}
|
||||
}
|
||||
comptime {
|
||||
blk: while(false) {}
|
||||
}
|
||||
comptime {
|
||||
blk: for(@as([0]void, undefined)) |_| {}
|
||||
}
|
||||
|
||||
// duplicate/unused labels
|
||||
//
|
||||
// tmp.zig:2:12: error: redefinition of label 'blk'
|
||||
// tmp.zig:2:5: note: previous definition here
|
||||
// tmp.zig:5:26: error: redefinition of label 'blk'
|
||||
// tmp.zig:5:5: note: previous definition here
|
||||
// tmp.zig:8:46: error: redefinition of label 'blk'
|
||||
// tmp.zig:8:5: note: previous definition here
|
||||
// tmp.zig:11:5: error: unused block label
|
||||
// tmp.zig:14:5: error: unused while loop label
|
||||
// tmp.zig:17:5: error: unused for loop label
|
|
@ -2,6 +2,7 @@ const ContextAllocator = MemoryPool(usize);
|
|||
|
||||
pub fn MemoryPool(comptime T: type) type {
|
||||
const free_list_t = @compileError("aoeu",);
|
||||
_ = T;
|
||||
|
||||
return struct {
|
||||
free_list: free_list_t,
|
||||
|
@ -10,10 +11,10 @@ pub fn MemoryPool(comptime T: type) type {
|
|||
|
||||
export fn entry() void {
|
||||
var allocator: ContextAllocator = undefined;
|
||||
_ = allocator;
|
||||
}
|
||||
|
||||
// constant inside comptime function has compile error
|
||||
//
|
||||
// tmp.zig:4:5: error: unreachable code
|
||||
// tmp.zig:4:25: note: control flow is diverted here
|
||||
// tmp.zig:12:9: error: unused local variable
|
||||
// :4:5: error: unreachable code
|
||||
// :4:25: note: control flow is diverted here
|
30
test/compile_errors/stage2/duplicate-unused_labels.zig
Normal file
30
test/compile_errors/stage2/duplicate-unused_labels.zig
Normal file
|
@ -0,0 +1,30 @@
|
|||
comptime {
|
||||
blk: { blk: while (false) {} }
|
||||
}
|
||||
comptime {
|
||||
blk: while (false) { blk: for (@as([0]void, undefined)) |_| {} }
|
||||
}
|
||||
comptime {
|
||||
blk: for (@as([0]void, undefined)) |_| { blk: {} }
|
||||
}
|
||||
comptime {
|
||||
blk: {}
|
||||
}
|
||||
comptime {
|
||||
blk: while(false) {}
|
||||
}
|
||||
comptime {
|
||||
blk: for(@as([0]void, undefined)) |_| {}
|
||||
}
|
||||
|
||||
// duplicate/unused labels
|
||||
//
|
||||
// :2:12: error: redefinition of label 'blk'
|
||||
// :2:5: note: previous definition here
|
||||
// :5:26: error: redefinition of label 'blk'
|
||||
// :5:5: note: previous definition here
|
||||
// :8:46: error: redefinition of label 'blk'
|
||||
// :8:5: note: previous definition here
|
||||
// :11:5: error: unused block label
|
||||
// :14:5: error: unused while loop label
|
||||
// :17:5: error: unused for loop label
|
Loading…
Reference in New Issue
Block a user