From 0d4db8828a9efc05b5c3622098a8337de0b62d1e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 Feb 2019 14:03:36 -0500 Subject: [PATCH] `@cImport` works with `--cache on` We pass -MD -MF args to clang when doing `@cImport`, which gives us a complete list of files that the C code read from. Then we add these to the cache. So even when using `@cImport` Zig's caching system remains perfect. This is a proof of concept for the mechanism that the self-hosted compiler will use to watch and rebuild files. --- src/cache_hash.cpp | 34 ++++++++++++++++++++++++++++++++++ src/cache_hash.hpp | 2 ++ src/codegen.cpp | 31 ++++++------------------------- src/error.cpp | 1 + src/error.hpp | 1 + src/ir.cpp | 6 ------ src/os.cpp | 12 ++++++++++++ src/os.hpp | 1 + src/translate_c.cpp | 20 ++++++++++++++++++++ 9 files changed, 77 insertions(+), 31 deletions(-) diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index 4526a83c2..85bad3dd2 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -414,6 +414,39 @@ Error cache_add_file(CacheHash *ch, Buf *path) { return cache_add_file_fetch(ch, resolved_path, nullptr); } +Error cache_add_dep_file(CacheHash *ch, Buf *dep_file_path, bool verbose) { + Error err; + Buf *contents = buf_alloc(); + if ((err = os_fetch_file_path(dep_file_path, contents, false))) { + if (verbose) { + fprintf(stderr, "unable to read .d file: %s\n", err_str(err)); + } + return ErrorReadingDepFile; + } + SplitIterator it = memSplit(buf_to_slice(contents), str("\n")); + // skip first line + SplitIterator_next(&it); + for (;;) { + Optional> opt_line = SplitIterator_next(&it); + if (!opt_line.is_some) + break; + if (opt_line.value.len == 0) + continue; + SplitIterator line_it = memSplit(opt_line.value, str(" \t")); + Slice filename; + if (!SplitIterator_next(&line_it).unwrap(&filename)) + continue; + Buf *filename_buf = buf_create_from_slice(filename); + if ((err = cache_add_file(ch, filename_buf))) { + if (verbose) { + fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(filename_buf), err_str(err)); + } + return err; + } + } + return ErrorNone; +} + static Error write_manifest_file(CacheHash *ch) { Error err; Buf contents = BUF_INIT; @@ -464,3 +497,4 @@ void cache_release(CacheHash *ch) { os_file_close(ch->manifest_file); } + diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp index db1c42ec0..d74c8623c 100644 --- a/src/cache_hash.hpp +++ b/src/cache_hash.hpp @@ -56,6 +56,8 @@ Error ATTRIBUTE_MUST_USE cache_hit(CacheHash *ch, Buf *out_b64_digest); // If you did not get a cache hit, call this function for every file // that is depended on, and then finish with cache_final. Error ATTRIBUTE_MUST_USE cache_add_file(CacheHash *ch, Buf *path); +// This opens a file created by -MD -MF args to Clang +Error ATTRIBUTE_MUST_USE cache_add_dep_file(CacheHash *ch, Buf *path, bool verbose); // This variant of cache_add_file returns the file contents. // Also the file path argument must be already resolved. diff --git a/src/codegen.cpp b/src/codegen.cpp index 3edff93be..4a8b376e2 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -8218,8 +8218,9 @@ static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) { } if (g->verbose_cc) { + fprintf(stderr, "zig"); for (size_t arg_i = 0; arg_i < args.length; arg_i += 1) { - fprintf(stderr, "%s ", args.at(arg_i)); + fprintf(stderr, " %s", args.at(arg_i)); } fprintf(stderr, "\n"); } @@ -8232,35 +8233,15 @@ static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) { g->link_objects.append(out_obj_path); - // add the files depended on to the cache system if (g->enable_cache) { - Buf *contents = buf_alloc(); - if ((err = os_fetch_file_path(out_dep_path, contents, false))) { - fprintf(stderr, "unable to read .d file: %s\n", err_str(err)); - exit(1); - } + // add the files depended on to the cache system if ((err = cache_add_file(&g->cache_hash, c_source_file))) { fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(c_source_file), err_str(err)); exit(1); } - SplitIterator it = memSplit(buf_to_slice(contents), str("\n")); - // skip first line - SplitIterator_next(&it); - for (;;) { - Optional> opt_line = SplitIterator_next(&it); - if (!opt_line.is_some) - break; - if (opt_line.value.len == 0) - continue; - SplitIterator line_it = memSplit(opt_line.value, str(" \t")); - Slice filename; - if (!SplitIterator_next(&line_it).unwrap(&filename)) - continue; - Buf *filename_buf = buf_create_from_slice(filename); - if ((err = cache_add_file(&g->cache_hash, filename_buf))) { - fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(c_source_file), err_str(err)); - exit(1); - } + if ((err = cache_add_dep_file(&g->cache_hash, out_dep_path, true))) { + fprintf(stderr, "failed to add C source dependencies to cache: %s\n", err_str(err)); + exit(1); } } } diff --git a/src/error.cpp b/src/error.cpp index 9e01a86d8..c81cfd168 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -36,6 +36,7 @@ const char *err_str(Error err) { case ErrorCacheUnavailable: return "cache unavailable"; case ErrorPathTooLong: return "path too long"; case ErrorCCompilerCannotFindFile: return "C compiler cannot find file"; + case ErrorReadingDepFile: return "failed to read .d file"; } return "(invalid error)"; } diff --git a/src/error.hpp b/src/error.hpp index 0e14f37a6..a580da547 100644 --- a/src/error.hpp +++ b/src/error.hpp @@ -38,6 +38,7 @@ enum Error { ErrorCacheUnavailable, ErrorPathTooLong, ErrorCCompilerCannotFindFile, + ErrorReadingDepFile, }; const char *err_str(Error err); diff --git a/src/ir.cpp b/src/ir.cpp index ea2fcb289..929f19558 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -18670,12 +18670,6 @@ static IrInstruction *ir_analyze_instruction_type_name(IrAnalyze *ira, IrInstruc } static IrInstruction *ir_analyze_instruction_c_import(IrAnalyze *ira, IrInstructionCImport *instruction) { - if (ira->codegen->enable_cache) { - ir_add_error(ira, &instruction->base, - buf_sprintf("TODO @cImport is incompatible with --cache on. The cache system currently is unable to detect subsequent changes in .h files.")); - return ira->codegen->invalid_instruction; - } - AstNode *node = instruction->base.source_node; assert(node->type == NodeTypeFnCallExpr); AstNode *block_node = node->data.fn_call_expr.params.at(0); diff --git a/src/os.cpp b/src/os.cpp index 95febca9b..f52325af0 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -1232,6 +1232,18 @@ static Error os_buf_to_tmp_file_posix(Buf *contents, Buf *suffix, Buf *out_tmp_p } #endif +Buf *os_tmp_filename(Buf *prefix, Buf *suffix) { + Buf *result = buf_create_from_buf(prefix); + + const char base64[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"; + assert(array_length(base64) == 64 + 1); + for (size_t i = 0; i < 12; i += 1) { + buf_append_char(result, base64[rand() % 64]); + } + buf_append_buf(result, suffix); + return result; +} + #if defined(ZIG_OS_WINDOWS) static Error os_buf_to_tmp_file_windows(Buf *contents, Buf *suffix, Buf *out_tmp_path) { char tmp_dir[MAX_PATH + 1]; diff --git a/src/os.hpp b/src/os.hpp index db357c22a..c4fae1d62 100644 --- a/src/os.hpp +++ b/src/os.hpp @@ -121,6 +121,7 @@ Error ATTRIBUTE_MUST_USE os_get_cwd(Buf *out_cwd); bool os_stderr_tty(void); void os_stderr_set_color(TermColor color); +Buf *os_tmp_filename(Buf *prefix, Buf *suffix); Error os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path); Error os_delete_file(Buf *path); diff --git a/src/translate_c.cpp b/src/translate_c.cpp index 3fcdf139f..c80bc6d72 100644 --- a/src/translate_c.cpp +++ b/src/translate_c.cpp @@ -4776,6 +4776,15 @@ Error parse_h_file(ImportTableEntry *import, ZigList *errors, const clang_argv.append("-x"); clang_argv.append("c"); + Buf *out_dep_path = nullptr; + if (codegen->enable_cache) { + Buf *prefix = buf_sprintf("%s" OS_SEP, buf_ptr(&codegen->cache_dir)); + out_dep_path = os_tmp_filename(prefix, buf_create_from_str(".d")); + clang_argv.append("-MD"); + clang_argv.append("-MF"); + clang_argv.append(buf_ptr(out_dep_path)); + } + if (c->codegen->zig_target->is_native) { char *ZIG_PARSEC_CFLAGS = getenv("ZIG_NATIVE_PARSEC_CFLAGS"); if (ZIG_PARSEC_CFLAGS) { @@ -4912,6 +4921,17 @@ Error parse_h_file(ImportTableEntry *import, ZigList *errors, const return ErrorCCompileErrors; } + if (codegen->enable_cache) { + Error err; + assert(out_dep_path != nullptr); + if ((err = cache_add_dep_file(&codegen->cache_hash, out_dep_path, codegen->verbose_cimport))) { + if (codegen->verbose_cimport) { + fprintf(stderr, "translate-c: aborting due to failed cache operation: %s\n", err_str(err)); + } + return err; + } + } + c->ctx = ZigClangASTUnit_getASTContext(ast_unit); c->source_manager = ZigClangASTUnit_getSourceManager(ast_unit); c->root = trans_create_node(c, NodeTypeContainerDecl);