diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index d0cef58..9b5eda3 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -22,14 +22,14 @@ jobs: run: | apt update -y DEBIAN_FRONTEND=noninteractive apt install -y \ - gcc g++-12 \ + gcc \ libcapstone-dev \ libcapstone4 \ make - name: make run: make TARGET=x86_64 all - name: test - run: make CXX=g++-12 TARGET=x86_64 test-ci + run: make TARGET=x86_64 test-ci Linux_i386: name: 'Linux i386' runs-on: ubuntu-22.04 @@ -44,7 +44,7 @@ jobs: dpkg --add-architecture i386 apt update -y DEBIAN_FRONTEND=noninteractive apt install -y \ - gcc g++-12 \ + gcc \ gcc-multilib \ libc6:i386 \ make \ @@ -62,7 +62,7 @@ jobs: - name: make run: make TARGET=x86 all - name: test - run: make CXX=g++-12 TARGET=x86 test-ci + run: make TARGET=x86 test-ci Linux_aarch64: name: 'Linux aarch64' runs-on: ubuntu-20.04 @@ -78,7 +78,7 @@ jobs: install: | apt update -y DEBIAN_FRONTEND=noninteractive apt install -y \ - gcc g++-12 \ + gcc \ libc6 \ libcapstone-dev \ libcapstone4 \ @@ -86,7 +86,7 @@ jobs: run: | make CC=gcc TARGET=aarch64 all # FIXME: For some reason some of the object files are rebuilt - make CC=gcc CXX=g++-12 TARGET=aarch64 test-ci + make CC=gcc TARGET=aarch64 test-ci MACOS_x86_64: runs-on: macos-11 @@ -99,7 +99,7 @@ jobs: - name: make run: make TARGET=x86_64 all - name: test - run: make CXX=g++-12 TARGET=x86_64 test-ci + run: make TARGET=x86_64 test-ci Windows: strategy: diff --git a/.gitignore b/.gitignore index 78704a1..7a37a4f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ ir_emit_aarch64.h minilua gen_ir_fold_hash ir_test -ir-test +tester ir b.c diff --git a/Makefile b/Makefile index f609f8d..26563ac 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,6 @@ EXAMPLES_SRC_DIR = $(SRC_DIR)/examples EXAMPLES_BUILD_DIR = $(BUILD_DIR)/examples CC = gcc -CXX = g++ BUILD_CC = gcc override CFLAGS += -Wall -Wextra -Wno-unused-parameter LDFLAGS = -lm -ldl @@ -94,17 +93,19 @@ $(OBJS_COMMON) $(OBJS_IR) $(OBJS_IR_TEST): $(BUILD_DIR)/$(notdir %.o): $(SRC_DIR $(EXAMPLE_EXES): $(EXAMPLES_BUILD_DIR)/$(notdir %): $(EXAMPLES_SRC_DIR)/$(notdir %.c) $(CC) $(CFLAGS) -I$(SRC_DIR) $< -o $@ $(OBJS_COMMON) $(LDFLAGS) -lcapstone -$(BUILD_DIR)/ir-test: $(SRC_DIR)/ir-test.cxx - $(CXX) -O3 -std=c++17 $(SRC_DIR)/ir-test.cxx -o $(BUILD_DIR)/ir-test +$(BUILD_DIR)/tester: $(SRC_DIR)/tools/tester.c + $(CC) $(CFLAGS) -o $@ $< -test: $(BUILD_DIR)/ir $(BUILD_DIR)/ir-test +test: $(BUILD_DIR)/ir $(BUILD_DIR)/tester $(BUILD_DIR)/ir $(SRC_DIR)/test.ir --dump --save 2>$(BUILD_DIR)/test.log $(BUILD_DIR)/ir $(SRC_DIR)/test.ir --dot $(BUILD_DIR)/ir.dot dot -Tpdf $(BUILD_DIR)/ir.dot -o $(BUILD_DIR)/ir.pdf - $(BUILD_DIR)/ir-test $(SRC_DIR)/tests + $(BUILD_DIR)/tester --test-cmd $(BUILD_DIR)/ir --target $(TARGET) --default-args "--save" \ + --test-extension ".irt" --code-extension ".ir" $(SRC_DIR)/tests -test-ci: $(BUILD_DIR)/ir $(BUILD_DIR)/ir-test - $(BUILD_DIR)/ir-test --show-diff $(SRC_DIR)/tests +test-ci: $(BUILD_DIR)/ir $(BUILD_DIR)/tester + $(BUILD_DIR)/tester --test-cmd $(BUILD_DIR)/ir --target $(TARGET) --default-args "--save" \ + --test-extension ".irt" --code-extension ".ir" --show-diff $(SRC_DIR)/tests examples: $(OBJS_COMMON) $(EXAMPLES_BUILD_DIR) $(EXAMPLE_EXES) @@ -113,7 +114,7 @@ clean: $(BUILD_DIR)/minilua $(BUILD_DIR)/ir_emit_$(DASM_ARCH).h \ $(BUILD_DIR)/ir_fold_hash.h $(BUILD_DIR)/gen_ir_fold_hash \ $(BUILD_DIR)/ir.dot $(BUILD_DIR)/ir.pdf $(BUILD_DIR)/test.log \ - $(BUILD_DIR)/ir-test + $(BUILD_DIR)/tester find $(SRC_DIR)/tests -type f -name '*.diff' -delete find $(SRC_DIR)/tests -type f -name '*.out' -delete find $(SRC_DIR)/tests -type f -name '*.exp' -delete diff --git a/ir-test.cxx b/ir-test.cxx deleted file mode 100644 index 74390d7..0000000 --- a/ir-test.cxx +++ /dev/null @@ -1,391 +0,0 @@ -/* - * IR - Lightweight JIT Compilation Framework - * (Test runner implementation) - * Copyright (C) 2023 by IR project. - * Authors: Dmitry Stogov , Anatol Belski - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -# include -# define popen _popen -# define pclose _pclose -# define PATH_SEP "\\" -# define EXE_SUF ".exe" -# define REGEX_FLAGS std::regex::extended -# define DIFF_CMD "fc.exe" -# define DIFF_ARGS "/n" -# define IR_ARGS "--no-abort-fault" -#else -# define PATH_SEP "/" -# define EXE_SUF "" -# define REGEX_FLAGS std::regex::ECMAScript | std::regex::multiline -# define DIFF_CMD "diff" -# define DIFF_ARGS "--strip-trailing-cr -u" -# define IR_ARGS "" -#endif - -uint32_t skipped = 0; -std::vector bad_list; -std::vector> failed; -std::vector> xfailed_list; -bool show_diff = false, colorize = true; -std::string ir_exe, ir_target; -std::string this_exe, this_exe_path; - -namespace ir { - decltype(auto) trim(std::string s) { - const char* ws = " \t\n\r\f\v"; - s.erase(0, s.find_first_not_of(ws)); - s.erase(s.find_last_not_of(ws) + 1); - return s; - } - - decltype(auto) exec(const std::string& cmd) { - std::array buffer; - std::string result; - std::unique_ptr pipe(popen(cmd.c_str(), "r"), pclose); - if (!pipe) { - throw std::runtime_error("popen() failed!"); - } - while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) { - result += buffer.data(); - } - return result; - } - - enum color { GREEN, YELLOW, RED }; - - decltype(auto) colorize(const std::string& s, enum color c) { - if (::colorize) - switch (c) { - case GREEN: - return "\x1b[1;32m" + s + "\x1b[0m"; - case YELLOW: - return "\x1b[1;33m" + s + "\x1b[0m"; - case RED: - return "\x1b[1;31m" + s + "\x1b[0m"; - } - return "" + s; - } - - void init_console() { -#ifdef _WIN32 - if (::colorize) { - DWORD mode, new_mode = ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN; - HANDLE h = GetStdHandle(STD_OUTPUT_HANDLE); - GetConsoleMode(h, &mode); - SetConsoleMode(h, mode | new_mode); - } -#endif - } - - class broken_test_exception : public std::exception {} broken_test; - - struct it_sect { - std::size_t name_pos; - std::string name; - std::size_t content_pos; - std::string content; - }; - - struct test { - // Section contents - std::string name, code, expect, target, xfail, args; - // Test file path and contents and output filenames - std::string irt_file, irt_file_content, ir_file, out_file, exp_file, diff_file; - test(const std::string& test_fl) : irt_file{test_fl} { - std::ifstream f(test_fl); - if (!f) throw "Couldn't read '" + test_fl + "'"; - std::ostringstream oss; - oss << f.rdbuf(); - irt_file_content = oss.str(); - - auto make_fn = [&test_fl](const std::string& s) { - return test_fl.substr(0, test_fl.length() - 4) + s; - }; - ir_file = make_fn(".ir"); - out_file = make_fn(".out"); - exp_file = make_fn(".exp"); - diff_file = make_fn(".diff"); - - parse(); - } - void parse() { - std::regex sect_reg("^--[A-Z]+--$", REGEX_FLAGS); - auto sect_begin = std::sregex_iterator(irt_file_content.begin(), irt_file_content.end(), sect_reg); - auto sect_end = std::sregex_iterator(); - std::vector test_data; - - // PASS 0: Find all the test sections - auto found_sects = std::distance(sect_begin, sect_end); - for (auto i = sect_begin; i != sect_end; i++) { - it_sect iti; - std::smatch m = *i; - iti.name_pos = m.position(); - iti.name = m.str(); - test_data.push_back(std::move(iti)); - } - // PASS 1: Read section contents - auto it = test_data.begin(); - auto it_next = it; - it_next++; - for (; it != test_data.end(); it++) { - it->content_pos = it->name_pos + it->name.length(); - - if (test_data.end() == it_next) { - it->content = trim(irt_file_content.substr(it->content_pos, irt_file_content.length() - it->content_pos)); - } else { - it->content = trim(irt_file_content.substr(it->content_pos, it_next->name_pos - it->content_pos)); - it_next++; - } - } - - // PASS 2: Save necessary data into properties -#define NO_SECT test_data.end() - auto get_sect = [&test_data](std::string s) -> decltype(auto) { - for (auto test_it = test_data.begin(); test_it != test_data.end(); test_it++) - if (!test_it->name.compare(s)) return test_it; - return NO_SECT; - }; - auto test_sect = get_sect("--TEST--"); - auto code_sect = get_sect("--CODE--"); - auto exp_sect = get_sect("--EXPECT--"); - if (NO_SECT == test_sect || NO_SECT == code_sect || NO_SECT == exp_sect) { - throw broken_test; - } - name = test_sect->content; - code = code_sect->content; - expect = exp_sect->content; - auto args_sect = get_sect("--ARGS--"); - args = (NO_SECT == args_sect) ? "--save": args_sect->content; - auto tgt_sect = get_sect("--TARGET--"); - if (NO_SECT != tgt_sect) target = tgt_sect->content; - auto xfail_sect = get_sect("--XFAIL--"); - if (NO_SECT != xfail_sect) xfail = xfail_sect->content; - } - bool skip() { - return target.length() > 0 && target.compare(::ir_target); - } - bool run() { - std::ofstream in_os(ir_file); - in_os << code; - in_os.close(); - - auto test_cmd = ::ir_exe + " " + ir_file + " " + args + " " + IR_ARGS + " >" + out_file + " 2>&1"; - auto diff_cmd = std::string(DIFF_CMD) + " " + DIFF_ARGS + " " + exp_file + " " + out_file + " > " + diff_file + " 2>&1"; - - int ret_code = std::system(test_cmd.c_str()) >> 0x8; - - std::stringstream out_buf; - out_buf << std::ifstream(out_file).rdbuf(); - std::string out = trim(out_buf.str()); - out.erase(std::remove(out.begin(), out.end(), '\r'), out.end()); - - if (ret_code || out.compare(expect)) { - std::ofstream exp_os(exp_file); - exp_os << (expect + "\n"); - exp_os.close(); - - // XXX move away from diff tool dependency - ret_code = std::system(diff_cmd.c_str()) >> 0x8; - if (0 != ret_code && 1 != ret_code) { // 1 stands for "there is a diff" - std::cout << diff_cmd << std::endl; - std::cout << ret_code << std::endl; - throw "Couldn't compare output vs. expected result for '" + irt_file + "'"; - } - return false; - } - - std::remove(out_file.c_str()); - std::remove(exp_file.c_str()); - std::remove(diff_file.c_str()); - std::remove(ir_file.c_str()); - - return true; - } - }; - - void find_tests_in_dir(std::set& user_files, std::vector& irt_files) { - assert(user_files.size()>0); - - for (const auto& f : user_files) { - if (std::filesystem::is_directory(f)) { - for (const std::filesystem::directory_entry& ent : - std::filesystem::recursive_directory_iterator(f)) { - std::string fl(ent.path().string()); - if (fl.length() < 4 || ent.path().extension() != ".irt") - continue; - irt_files.push_back(fl); - } - } else { - irt_files.push_back(f); - } - } - - std::sort(irt_files.begin(), - irt_files.end(), - [&](const std::string& a, const std::string& b) { - return 1 <= b.compare(a); - }); - } -} - -static void print_help(void) -{ - std::cout << "Run IR uint tests\n"; - std::cout << "Usage:\n "; - std::cout << ::this_exe - << " [--show-diff] [--no-color] [test folders or files...]\n"; - std::cout << " Run all tests if no test folders/files are specified\n"; -} - -int main(int argc, char **argv) { - // Save this executable path to infer ir executable path - ::this_exe = argv[0]; - ::this_exe_path = std::filesystem::path{::this_exe}.parent_path().string(); - - // Store test folders/files specified by user - std::set user_files; - bool bad_opt = false; // unsupported option given - bool bad_files = false; // bad file or folder given - - for (int i = 1; i < argc; i++) { - // XXX use some cleaner arg parsing solution - if (!std::string(argv[i]).compare("--show-diff")) { - ::show_diff = true; - continue; - } else if (!std::string(argv[i]).compare("--no-color")) { - ::colorize = false; - continue; - } else if (!std::string(argv[i]).compare("--help")) { - print_help(); - return 0; - } else if (std::string(argv[i]).find_first_of("-") == 0) { - // Unsupported options - bad_opt = true; - std::cerr << ir::colorize("ERROR", ir::RED) - << ": Unsupported Option [" << argv[i] << "]\n"; - } else { - // User specified test folders/files - std::string file = argv[i]; - if (std::filesystem::exists(file) - && (std::filesystem::is_directory(file) - || std::filesystem::is_regular_file(file))) { - user_files.insert(argv[i]); - } else { - bad_files = true; - std::cerr << ir::colorize("ERROR", ir::RED) - << ": Bad File or Folder [" << file << "] \n"; - } - } - } /* for: arguments iteration */ - - if (bad_opt || bad_files) { - if (bad_opt) - print_help(); - return 1; - } - - ir::init_console(); - - ::ir_exe = ::this_exe_path + PATH_SEP + "ir" + EXE_SUF; - ::ir_target = ir::trim(ir::exec(::ir_exe + " --target")); - - // Get test files, either specified by user or all tests by default - std::vector irt_files; - if (user_files.empty()) { - // Pretend user specified all test - std::string tests_dir = ::this_exe_path + PATH_SEP + "tests"; - user_files.insert(tests_dir); - } - ir::find_tests_in_dir(user_files, irt_files); - - // Run each test - for (const std::string& test_fl : irt_files) { - try { - auto test = ir::test(test_fl); - - size_t idx = &test_fl - &irt_files[0] + 1; - std::string test_ln_0 = "TEST: " + std::to_string(idx) + PATH_SEP + std::to_string(irt_files.size()) + " " + test.name + "[" + test_fl + "]\r"; - std::cout << test_ln_0 << std::flush; - - if (test.skip()) { - std::cout << std::string(test_ln_0.length(), ' '); - std::cout << "\r" << ir::colorize("SKIP", ir::YELLOW) << ": " << test.name << " [" << test_fl << "]\n"; - ::skipped++; - continue; - } - - auto ret = test.run(); - std::cout << std::string(test_ln_0.length(), ' '); - - if (ret) { - std::cout << "\r" << ir::colorize("PASS", ir::GREEN) << ": " << test.name << " [" << test_fl << "]\n"; - } else if (test.xfail.length() > 0) { - std::cout << "\r" << ir::colorize("XFAIL", ir::RED) << ": " << test.name << " [" << test_fl << "] XFAIL REASON: " << test.xfail << "\n"; - ::xfailed_list.push_back({test.name, test.irt_file, test.xfail}); - } else { - std::cout << "\r" << ir::colorize("FAIL", ir::RED) << ": " << test.name << " [" << test_fl << "]\n"; - ::failed.push_back({test.name, test.irt_file}); - if (::show_diff) { - std::ifstream f(test.diff_file); - if (!f) { - throw "Couldn't read '" + test.diff_file + "'"; - } - std::cout << f.rdbuf() << std::endl; - f.close(); - } - } - } catch (ir::broken_test_exception& e) { - std::cout << "\r" << ir::colorize("BROK", ir::RED) << ": [" << test_fl << "]\n"; - ::bad_list.push_back(test_fl); - } catch (std::string& s) { - std::cout << "\r" << ir::colorize("ERROR", ir::RED) << ": " << s << '\n'; - } catch (std::exception& e) { - std::cout << "\r" << ir::colorize("ERROR", ir::RED) << ": " << e.what() << '\n'; - } - - // XXX parallelize - } - - // Produce the summary -#define SEPARATOR() std::cout << std::string(32, '-') << std::endl -#define WRAP_OUT(expr) SEPARATOR(); expr; SEPARATOR() - WRAP_OUT(std::cout << "Test Summary" << std::endl); - if (::bad_list.size() > 0) { - std::cout << "Bad tests: " << ::bad_list.size() << std::endl; - WRAP_OUT(for (const std::string& fname : ::bad_list) std::cout << fname << std::endl); - } - std::cout << "Total: " << irt_files.size() << std::endl; - std::cout << "Passed: " << (irt_files.size() - ::failed.size() - ::skipped) << std::endl; - std::cout << "Expected fail: " << ::xfailed_list.size() << std::endl; - std::cout << "Failed: " << ::failed.size() << std::endl; - std::cout << "Skipped: " << ::skipped << std::endl; - if (::xfailed_list.size() > 0) { - WRAP_OUT(std::cout << "EXPECTED FAILED TESTS" << std::endl); - for (const auto& t : ::xfailed_list) { - std::cout << std::get<0>(t) << " [" << std::get<1>(t) << "] XFAIL REASON: " << std::get<2>(t) << std::endl; - } - } - if (::failed.size() > 0) { - WRAP_OUT(std::cout << "FAILED TESTS" << std::endl); - for (const auto& t : ::failed) { - std::cout << std::get<0>(t) << " [" << std::get<1>(t) << "]" << std::endl; - } - } - SEPARATOR(); - - return ::failed.size() > 0 ? 1 : 0; -} diff --git a/ir-test.php b/ir-test.php deleted file mode 100644 index 1f40a39..0000000 --- a/ir-test.php +++ /dev/null @@ -1,231 +0,0 @@ - - */ - -function parse_test($test, &$name, &$code, &$expect, &$args, &$target, &$xfail) { - $sections = array(); - $text = @file_get_contents($test); - if (!$text || !preg_match_all("/^--[A-Z]+--$/m", $text, $r, PREG_SET_ORDER | PREG_OFFSET_CAPTURE)) { - return false; - } - foreach($r as $sect) { - if (isset($sect_name)) { - $sections[$sect_name] = trim(substr($text, $sect_offset, $sect[0][1] - $sect_offset)); - } - $sect_name = $sect[0][0]; - $sect_offset = $sect[0][1] + strlen($sect_name); - } - if (isset($sect_name)) { - $sections[$sect_name] = trim(substr($text, $sect_offset)); - } - if (!isset($sections['--TEST--']) || !isset($sections['--CODE--']) || !isset($sections['--EXPECT--'])) { - return false; - } - $name = $sections['--TEST--']; - $code = $sections['--CODE--']; - $expect = $sections['--EXPECT--']; - $args = isset($sections['--ARGS--']) ? $sections['--ARGS--'] : "--save"; - $target = isset($sections['--TARGET--']) ? $sections['--TARGET--'] : null; - $xfail = isset($sections['--XFAIL--']) ? $sections['--XFAIL--'] : null; - return true; -} - -function run_test($build_dir, $test, $name, $code, $expect, $args) { - $base = substr($test, 0, -4); - $input = $base . ".ir"; - $output = $base . ".out"; - @unlink($input); - @unlink($output); - @unlink("$base.exp"); - @unlink("$base.diff"); - if (!@file_put_contents($input, $code)) { - return false; - } - if (PHP_OS_FAMILY != "Windows") { - $cmd = "$build_dir/ir $input $args >$output 2>&1"; - } else { - $cmd = "$build_dir\\ir $input $args --no-abort-fault >$output 2>&1"; - } - $ret = @system($cmd, $result_code); - if ($ret === false) { - return false; - } - if ($result_code) { - } - $out = @file_get_contents($output); - if ($out === false) { - return false; - } - if ($result_code) { - $out = "\nExit Code = $result_code\n"; - file_put_contents($output, $out); - } - $out = trim($out); - $out = str_replace("\r", "", $out); - if ($out !== $expect) { - if (!@file_put_contents("$base.exp", "$expect\n")) { - return false; - } - if (PHP_OS_FAMILY != "Windows") { - $cmd = "diff -u $base.exp $output > $base.diff"; - } else { - /* Diff somehow resets terminal and breaks "cooring" */ - $cmd = "diff --strip-trailing-cr -u $base.exp $output > $base.diff 2>&1"; - } - if (@system($cmd) != 0) { - return false; - } - return false; - } - @unlink($input); - @unlink($output); - return true; -} - -function find_tests_in_dir($dir, &$tests) { - $d = opendir($dir); - if ($d !== false) { - while (($name = readdir($d)) !== false) { - if ($name === '.' || $name === '..') continue; - $fn = "$dir/$name"; - if (is_dir($fn)) { - find_tests_in_dir($fn, $tests); - } else if (substr($name, -4) === '.irt') { - $tests[] = $fn; - } - } - closedir($d); - } -} - -function find_tests($dir) { - $tests = []; - find_tests_in_dir($dir, $tests); - sort($tests); - return $tests; -} - -function run_tests() { - global $show_diff, $colorize; - - $build_dir = getenv("BUILD_DIR") ?? "."; - $src_dir = getenv("SRC_DIR") ?? "."; - $skiped = 0; - if (PHP_OS_FAMILY != "Windows") { - $cmd = "$build_dir/ir --target"; - } else { - $cmd = "$build_dir\\ir --target"; - } - $target = @system($cmd); - if ($target === false) { - echo "Cannot run '$cmd'\n"; - return 1; - } - $tests = find_tests("$src_dir/tests"); - $bad = array(); - $failed = array(); - $xfailed = array(); - $total = count($tests); - $count = 0; - foreach($tests as $test) { - $count++; - if (parse_test($test, $name, $code, $expect, $opt, $test_target, $xfail)) { - if ($test_target !== null && $target != $test_target) { - if ($colorize) { - echo "\r\e[1;33mSKIP\e[0m: $name [$test]\n"; - } else { - echo "\rSKIP: $name [$test]\n"; - } - $skiped++; - continue; - } - $str = "TEST: $count/$total $name [$test]\r"; - $len = strlen($str); - echo $str; - flush(); - $ret = run_test($build_dir, $test, $name, $code, $expect, $opt); - echo str_repeat(" ", $len); - if ($ret) { - if ($colorize) { - echo "\r\e[1;32mPASS\e[0m: $name [$test]\n"; - } else { - echo "\rPASS: $name [$test]\n"; - } - } else if (isset($xfail)) { - if ($colorize) { - echo "\r\e[1;31mXFAIL\e[0m: $name [$test] XFAIL REASON: $xfail\n"; - } else { - echo "\rXFAIL: $name [$test] XFAIL REASON: $xfail\n"; - } - $xfailed[$test] = array($name, $xfail); - } else { - if ($colorize) { - echo "\r\e[1;31mFAIL\e[0m: $name [$test]\n"; - } else { - echo "\rFAIL: $name [$test]\n"; - } - $failed[$test] = $name; - if ($show_diff) { - $base = substr($test, 0, -4); - echo file_get_contents("$base.diff"); - } - } - } else { - if ($colorize) { - echo "\r\e[1;31mBROK\e[0m: $name [$test]\n"; - } else { - echo "\rBROK: $name [$test]\n"; - } - $bad[] = $test; - } - } - echo "-------------------------------\n"; - echo "Test Summary\n"; - echo "-------------------------------\n"; - if (count($bad) > 0) { - echo "Bad tests: " . count($bad) . "\n"; - echo "-------------------------------\n"; - foreach ($bad as $test) { - echo "$test\n"; - } - echo "-------------------------------\n"; - } - echo "Total: " . count($tests) . "\n"; - echo "Passed: " . (count($tests) - count($failed) - $skiped) . "\n"; - echo "Expected fail: " . count($xfailed) . "\n"; - echo "Failed: " . count($failed) . "\n"; - echo "Skiped: " . $skiped . "\n"; - if (count($xfailed) > 0) { - echo "-------------------------------\n"; - echo "EXPECTED FAILED TESTS\n"; - echo "-------------------------------\n"; - foreach ($xfailed as $test => list($name, $xfail)) { - echo "$name [$test] XFAIL REASON: $xfail\n"; - } - } - if (count($failed) > 0) { - echo "-------------------------------\n"; - echo "FAILED TESTS\n"; - echo "-------------------------------\n"; - foreach ($failed as $test => $name) { - echo "$name [$test]\n"; - } - } - echo "-------------------------------\n"; - - return count($failed) > 0 ? 1 : 0; -} - -$colorize = true; -if (function_exists('sapi_windows_vt100_support') && !sapi_windows_vt100_support(STDOUT, true)) { - $colorize = false; -} -if (in_array('--no-color', $argv, true)) { - $colorize = false; -} -$show_diff = in_array('--show-diff', $argv, true); -exit(run_tests()); diff --git a/tools/tester.c b/tools/tester.c new file mode 100644 index 0000000..11fb438 --- /dev/null +++ b/tools/tester.c @@ -0,0 +1,660 @@ +/* + * IR - Lightweight JIT Compilation Framework + * (Test runner implementation) + * Copyright (C) 2023 by IR project. + * Authors: Dmitry Stogov , Anatol Belski + */ + +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# include +# include +# pragma warning(disable : 4996) +# define PATH_SEP '\\' +# define DEFAULT_DIFF_CMD "fc" +# define S_IRUSR _S_IREAD +# define S_IWUSR _S_IWRITE +#else +# include +# include +# include +# define PATH_SEP '/' +# define DEFAULT_DIFF_CMD "diff --strip-trailing-cr -u" +# define O_BINARY 0 +#endif + +typedef enum _color {GREEN, YELLOW, RED} color; + +typedef struct _test { + int id; + char *name; + char *target; + char *args; + char *code; + char *expect; + char *xfail; +} test; + +static int colorize = 1; + +static const char *test_cmd = NULL; +static const char *target = NULL; +static const char *default_args = NULL; +static const char *additional_args = NULL; +static const char *diff_cmd = NULL; + +static const char *test_extension = NULL; +static int test_extension_len = 0; + +static const char *code_extension = NULL; +static int code_extension_len = 0; + +static char **files = NULL; +static int files_count = 0; +static int files_limit = 0; + +static void print_color(const char *s, color c) +{ + if (colorize) { + switch (c) { + case GREEN: + printf("\x1b[1;32m%s\x1b[0m", s); + return; + case YELLOW: + printf("\x1b[1;33m%s\x1b[0m", s); + return; + case RED: + printf("\x1b[1;31m%s\x1b[0m", s); + return; + } + } + printf("%s", s); +} + +static void init_console() +{ +#ifdef _WIN32 + if (colorize) { + DWORD mode, new_mode = ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN; + HANDLE h = GetStdHandle(STD_OUTPUT_HANDLE); + GetConsoleMode(h, &mode); + SetConsoleMode(h, mode | new_mode); + } +#endif +} + +static test *parse_file(const char *filename, int id) +{ + test *t; + char *buf, **section; + int fd; + size_t start, i, size; + struct stat stat_buf; + + if (stat(filename, &stat_buf) != 0) { + return NULL; + } + size = stat_buf.st_size; + fd = open(filename, O_RDONLY | O_BINARY, 0); + if (fd < 0) { + return NULL; + } + t = malloc(sizeof(test) + size + 1); + if (!t) { + return NULL; + } + buf = (char*)t + sizeof(test); + if ((size_t)read(fd, buf, size) != size) { + free(t); + return NULL; + } + close(fd); + memset(t, 0, sizeof(test)); + buf[size] = 0; + i = 0; + while (i < size) { + start = i; + while (i < size && buf[i] != '\r' && buf[i] != '\n') i++; + if (i - start == strlen("--TEST--") && memcmp(buf + start, "--TEST--", strlen("--TEST--")) == 0) { + section = &t->name; + } else if (i - start == strlen("--ARGS--") && memcmp(buf + start, "--ARGS--", strlen("--ARGS--")) == 0) { + section = &t->args; + } else if (i - start == strlen("--CODE--") && memcmp(buf + start, "--CODE--", strlen("--CODE--")) == 0) { + section = &t->code; + } else if (i - start == strlen("--EXPECT--") && memcmp(buf + start, "--EXPECT--", strlen("--EXPECT--")) == 0) { + section = &t->expect; + } else if (i - start == strlen("--XFAIL--") && memcmp(buf + start, "--XFAIL--", strlen("--XFAIL--")) == 0) { + section = &t->xfail; + } else if (i - start == strlen("--TARGET--") && memcmp(buf + start, "--TARGET--", strlen("--TARGET--")) == 0) { + section = &t->target; + } else { + section = NULL; + while (i < size && (buf[i] == '\r' || buf[i] == '\n')) i++; + } + if (section) { + while (start > 0 && (buf[start - 1] == '\r' || buf[start - 1] == '\n')) start--; + buf[start] = 0; + if (*section) { + free(t); + return NULL; + } + while (i < size && (buf[i] == '\r' || buf[i] == '\n')) i++; + *section = buf + i; + } + } + if (!t->name || !t->code || !t->expect) { + free(t); + return NULL; + } + t->id = id; + return t; +} + +static int skip_test(test *t) +{ + if (target && (t->target && strcmp(t->target, target) != 0)) { + return 1; + } + return 0; +} + +static int same_text(const char *exp, const char *out) +{ + int i, j; + + while (*exp == '\r' || *exp == '\n') exp++; + while (*out == '\r' || *out == '\n') out++; + while (*exp != 0 && *out != 0) { + i = j = 0; + while (exp[i] != 0 && exp[i] != '\r' && exp[i] != '\n') i++; + while (out[j] != 0 && out[j] != '\r' && out[j] != '\n') j++; + if (i != j || memcmp(exp, out, i) != 0) { + return 0; + } + exp += i; + out += j; + if (*exp == '\r') exp++; + if (*exp == '\n') exp++; + if (*out == '\r') out++; + if (*out == '\n') out++; + } + while (*exp == '\r' || *exp == '\n') exp++; + while (*out == '\r' || *out == '\n') out++; + return *exp == 0 && *out == 0; +} + +static char *read_file(const char *filename) +{ + struct stat stat_buf; + size_t size; + char *buf; + int fd; + + if (stat(filename, &stat_buf) != 0) { + return NULL; + } + size = stat_buf.st_size; + buf = malloc(size + 1); + if (!buf) { + return NULL; + } + fd = open(filename, O_RDONLY | O_BINARY, 0); + if (fd < 0) { + free(buf); + return NULL; + } + if ((size_t)read(fd, buf, size) != size) { + free(buf); + return NULL; + } + close(fd); + buf[size] = 0; + return buf; +} + +static int write_file(const char *filename, const char *buf, size_t size) +{ + int fd = open(filename, O_CREAT | O_TRUNC | O_WRONLY | O_BINARY, S_IRUSR | S_IWUSR); + if (fd < 0) { + return 0; + } + if ((size_t)write(fd, buf, size) != size) { + return 0; + } + close(fd); + return 1; +} + +static char *replace_extension(const char *filename, size_t len, const char *ext, size_t ext_len) +{ + char *ret; + + if (test_extension) { + ret = malloc(len - test_extension_len + ext_len + 1); + memcpy(ret, filename, len - test_extension_len); + memcpy(ret + len - test_extension_len, ext, ext_len + 1); + } else { + ret = malloc(len + ext_len + 1); + memcpy(ret, filename, len); + memcpy(ret + len, ext, ext_len + 1); + } + return ret; +} + +static int run_test(const char *filename, test *t, int show_diff) +{ + size_t len; + int ret; + char cmd[4096]; + char *code_filename, *out_filename, *exp_filename, *diff_filename, *out; + + len = strlen(filename); + + code_filename = replace_extension(filename, len, code_extension, code_extension_len); + out_filename = replace_extension(filename, len, ".out", strlen(".out")); + exp_filename = replace_extension(filename, len, ".exp", strlen(".exp")); + diff_filename = replace_extension(filename, len, ".diff", strlen(".diff")); + + unlink(code_filename); + unlink(out_filename); + unlink(exp_filename); + unlink(diff_filename); + + if (!write_file(code_filename, t->code, strlen(t->code))) { + free(code_filename); + free(code_filename); + free(out_filename); + free(exp_filename); + free(diff_filename); + return 0; + } + + if ((size_t)snprintf(cmd, sizeof(cmd), "%s %s %s %s > %s 2>&1", + test_cmd, code_filename, + t->args ? t->args : default_args, additional_args, + out_filename) > sizeof(cmd)) { + free(code_filename); + free(out_filename); + free(exp_filename); + free(diff_filename); + return 0; + } + + ret = system(cmd) >> 8; + + out = read_file(out_filename); + if (!out) { + out = malloc(1); + out[0] = 0; + } + + ret = (same_text(t->expect, out) && ret == 0); + if (ret) { + unlink(code_filename); + unlink(out_filename); + } else { + if (write_file(exp_filename, t->expect, strlen(t->expect))) { + if ((size_t)snprintf(cmd, sizeof(cmd), "%s %s %s > %s", + diff_cmd, exp_filename, out_filename, diff_filename) < sizeof(cmd)) { + system(cmd); + if (show_diff && !t->xfail) { + char *diff = read_file(diff_filename); + printf("\n"); + printf("%s", diff); + free(diff); + } + } + } + } + + free(out); + free(code_filename); + free(out_filename); + free(exp_filename); + free(diff_filename); + + return ret; +} + +static void add_file(char *name) +{ + if (files_count >= files_limit) { + files_limit += 1024; + files = realloc(files, sizeof(char*) * files_limit); + } + files[files_count++] = name; +} + +static void find_files_in_dir(const char *dir_name, size_t dir_name_len) +{ + char *name; + size_t len; +#ifdef _WIN32 + char buf[MAX_PATH]; + HANDLE dir; + WIN32_FIND_DATA info; + + memcpy(buf, dir_name, dir_name_len); + buf[dir_name_len] = '\\'; + buf[dir_name_len + 1] = '*'; + buf[dir_name_len + 2] = 0; + + dir = FindFirstFile(buf, &info); + if (dir == INVALID_HANDLE_VALUE) { + fprintf(stderr, "ERROR: Cannot read directory [%s]\n", dir_name); + return; + } + do { + len = strlen(info.cFileName); + if (info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + if ((len == 1 && info.cFileName[0] == '.') + || (len == 2 && info.cFileName[0] == '.' && info.cFileName[1] == '.')) { + /* skip */ + } else { + name = malloc(dir_name_len + len + 2); + memcpy(name, dir_name, dir_name_len); + name[dir_name_len] = PATH_SEP; + memcpy(name + dir_name_len + 1, info.cFileName, len + 1); + find_files_in_dir(name, dir_name_len + len + 1); + free(name); + } + } else /*if (info.dwFileAttributes & FILE_ATTRIBUTE_NORMAL)*/ { + if (!test_extension + || stricmp(info.cFileName + len - test_extension_len, test_extension) == 0) { + name = malloc(dir_name_len + len + 2); + memcpy(name, dir_name, dir_name_len); + name[dir_name_len] = PATH_SEP; + memcpy(name + dir_name_len + 1, info.cFileName, len + 1); + add_file(name); + } + } + } while (FindNextFile(dir, &info)); + FindClose(dir); +#else + DIR *dir = opendir(dir_name); + struct dirent *info; + + if (!dir) { + fprintf(stderr, "ERROR: Cannot read directory [%s]\n", dir_name); + return; + } + while ((info = readdir(dir)) != 0) { + len = strlen(info->d_name); + if (info->d_type == DT_DIR) { + if ((len == 1 && info->d_name[0] == '.') + || (len == 2 && info->d_name[0] == '.' && info->d_name[1] == '.')) { + /* skip */ + } else { + name = malloc(dir_name_len + len + 2); + memcpy(name, dir_name, dir_name_len); + name[dir_name_len] = PATH_SEP; + memcpy(name + dir_name_len + 1, info->d_name, len + 1); + find_files_in_dir(name, dir_name_len + len + 1); + free(name); + } + } else if (info->d_type == DT_REG) { + if (!test_extension + || strcasecmp(info->d_name + len - test_extension_len, test_extension) == 0) { + name = malloc(dir_name_len + len + 2); + memcpy(name, dir_name, dir_name_len); + name[dir_name_len] = PATH_SEP; + memcpy(name + dir_name_len + 1, info->d_name, len + 1); + add_file(name); + } + } + } + closedir(dir); +#endif +} + +static int cmp_files(const void *s1, const void *s2) +{ + return strcmp(*(const char**)s1, *(const char**)s2); +} + +static void find_files(char **tests, int tests_count) +{ + int i; + struct stat stat_buf; + + for (i = 0; i < tests_count; i++) { + if (stat(tests[i], &stat_buf) != 0) { + fprintf(stderr, "ERROR: Bad File or Folder [%s]\n", tests[i]); + } + if ((stat_buf.st_mode & S_IFMT) == S_IFDIR) { + find_files_in_dir(tests[i], strlen(tests[i])); + } else { + add_file(strdup(tests[i])); + } + } + qsort(files, files_count, sizeof(char*), cmp_files); +} + +static void print_help(const char *exe_name) +{ + printf( + "Run IR uint tests\n" + "Usage:\n" + " %s --test-cmd {options} \n" + " Run the \"--CODE--\" section of specified test files using \n" + "Options:\n" + " --target - skip tests that specifies different --TARGET--\n" + " --default-args - default arguments (if --ARGS-- is missed)\n" + " --additional-args - additional arguments (always added at the end)\n" + " --diff-cmd - diff command\n" + " --test-extension - search test files with the given extension\n" + " --code-extension - produce code files with the given extension\n" + " --show-diff - show diff of the failed tests\n" + " --no-color - disable color output\n" + , exe_name); +} + +static int check_arg(const char *name, const char **value, int argc, char **argv, int *pos, int *bad_opt) +{ + if (strcmp(argv[*pos], name) == 0) { + if (*value) { + *bad_opt = 1; + fprintf(stderr, "ERROR: Duplicate %s\n", name); + } else if (*pos + 1 == argc) { + *bad_opt = 1; + fprintf(stderr, "ERROR: Missing %s value\n", name); + } else { + *value = argv[++(*pos)]; + } + return 1; + } + return 0; +} + +int main(int argc, char **argv) +{ + int bad_opt = 0; // unsupported option given + int bad_files = 0; // bad file or folder given + int show_diff = 0; + struct stat stat_buf; +#ifdef _WIN32 + char **tests = _alloca(sizeof(char*) * argc); +#else + char **tests = alloca(sizeof(char*) * argc); +#endif + int i, tests_count = 0; + int skipped = 0; + int passed = 0; + int xfailed = 0, xfailed_limit = 0; + int failed = 0, failed_limit = 0; + int broken = 0, broken_limit = 0; + test *t, **xfailed_tests = NULL, **failed_tests = NULL; + char **broken_tests = NULL; + + for (i = 1; i < argc; i++) { + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) { + print_help(argv[0]); + return 0; + } else if (check_arg("--test-cmd", &test_cmd, argc, argv, &i, &bad_opt)) { + } else if (check_arg("--target", &target, argc, argv, &i, &bad_opt)) { + } else if (check_arg("--default-args", &default_args, argc, argv, &i, &bad_opt)) { + } else if (check_arg("--additional-args", &additional_args, argc, argv, &i, &bad_opt)) { + } else if (check_arg("--diff-cmd", &diff_cmd, argc, argv, &i, &bad_opt)) { + } else if (check_arg("--test-extension", &test_extension, argc, argv, &i, &bad_opt)) { + } else if (check_arg("--code-extension", &code_extension, argc, argv, &i, &bad_opt)) { + } else if (strcmp(argv[i], "--show-diff") == 0) { + show_diff = 1; + } else if (strcmp(argv[i], "--no-color") == 0) { + colorize = 0; + } else if (argv[i][0] == '-') { + bad_opt = 1; + fprintf(stderr, "ERROR: Unsupported Option [%s]\n", argv[i]); + } else { + // User specified test folders/files + if (stat(argv[i], &stat_buf) == 0 + && ((stat_buf.st_mode & S_IFMT) == S_IFDIR + || (stat_buf.st_mode & S_IFMT) == S_IFREG)) { + tests[tests_count++] = argv[i]; + } else { + bad_files = 1; + fprintf(stderr, "ERROR: Bad File or Folder [%s]\n", argv[i]); + } + } + } + + if (bad_opt || bad_files || !tests_count || !test_cmd) { + if (bad_opt || !tests_count || !test_cmd) { + print_help(argv[0]); + } + return 1; + } + + if (!default_args) { + default_args = ""; + } + if (!additional_args) { + additional_args = ""; + } + + if (!code_extension) { + code_extension = ".code"; + } + code_extension_len = (int)strlen(code_extension); + if (test_extension) { + test_extension_len = (int)strlen(test_extension); + if (strcmp(test_extension, code_extension) == 0) { + fprintf(stderr, "ERROR: --test-extension and --code-extension can't be the same\n"); + return 1; + } + } + + if (!diff_cmd) { + diff_cmd = DEFAULT_DIFF_CMD; + } + + init_console(); + + find_files(tests, tests_count); + + // Run each test + for (i = 0; i < files_count; i++) { + t = parse_file(files[i], i); + if (!t) { + printf("\r"); + print_color("BROK", RED); + printf(": [%s]\n", files[i]); + if (broken >= broken_limit) { + broken_limit += 1024; + broken_tests = realloc(broken_tests, sizeof(char*) * broken_limit); + } + broken_tests[broken++] = files[i]; + continue; + } + printf("TEST: %s [%s]", t->name, files[i]); + fflush(stdout); + if (skip_test(t)) { + printf("\r"); + print_color("SKIP", YELLOW); + printf(": %s [%s]\n", t->name, files[i]); + skipped++; + free(t); + } else if (run_test(files[i], t, show_diff)) { + printf("\r"); + print_color("PASS", GREEN); + printf(": %s [%s]\n", t->name, files[i]); + passed++; + free(t); + } else if (t->xfail) { + printf("\r"); + print_color("XFAIL", RED); + printf(": %s [%s]\n", t->name, files[i]); + if (xfailed >= xfailed_limit) { + xfailed_limit += 1024; + xfailed_tests = realloc(xfailed_tests, sizeof(test*) * xfailed_limit); + } + xfailed_tests[xfailed++] = t; + } else { + printf("\r"); + print_color("FAIL", RED); + printf(": %s [%s]\n", t->name, files[i]); + if (failed >= failed_limit) { + failed_limit += 1024; + failed_tests = realloc(failed_tests, sizeof(test*) * failed_limit); + } + failed_tests[failed++] = t; + } + } + + // Produce the summary + printf("-------------------------------\n"); + printf("Test Sumary\n"); + printf("-------------------------------\n"); + printf("Total: %d\n", files_count); + printf("Passed: %d\n", passed); + printf("Skipped: %d\n", skipped); + printf("Expected fail: %d\n", xfailed); + printf("Failed: %d\n", failed); + if (broken) { + printf("Broken: %d\n", broken); + } + if (xfailed > 0) { + printf("-------------------------------\n"); + printf("EXPECTED FAILED TESTS\n"); + printf("-------------------------------\n"); + for (i = 0; i < xfailed; i++) { + t = xfailed_tests[i]; + printf("%s [%s] XFAIL REASON: %s\n", t->name, files[t->id], t->xfail); + free(t); + } + free(xfailed_tests); + } + if (failed > 0) { + printf("-------------------------------\n"); + printf("FAILED TESTS\n"); + printf("-------------------------------\n"); + for (i = 0; i < failed; i++) { + t = failed_tests[i]; + printf("%s [%s]\n", t->name, files[t->id]); + free(t); + } + free(failed_tests); + } + if (broken > 0) { + printf("-------------------------------\n"); + printf("BROKEN TESTS\n"); + printf("-------------------------------\n"); + for (i = 0; i < broken; i++) { + printf("%s\n", broken_tests[i]); + } + free(broken_tests); + } + printf("-------------------------------\n"); + + for (i = 0; i < files_count; i++) { + free(files[i]); + } + free(files); + + return failed > 0 ? 1 : 0; +} diff --git a/win32/Makefile b/win32/Makefile index f2003a8..5d21838 100644 --- a/win32/Makefile +++ b/win32/Makefile @@ -38,10 +38,6 @@ LDFLAGS=$(LDFLAGS) /DEBUG CC=cl.exe !endif -!if "$(CXX)" == "" -CXX=cl.exe -!endif - !if "$(LD)" == "" LD=link.exe !endif @@ -57,11 +53,13 @@ TARGET = $(VSCMD_ARG_TGT_ARCH) CFLAGS=$(CFLAGS) /DIR_TARGET_X64 DASM_ARCH=x86 DASM_FLAGS=-D X64=1 +TEST_TARGET=Windows-x86_64 !endif !if "$(TARGET)" == "x86" CFLAGS=$(CFLAGS) /DIR_TARGET_X86 DASM_ARCH=x86 DASM_FLAGS= +TEST_TARGET=x86 !endif !if "$(VSCMD_ARG_TGT_ARCH)" == "x64" @@ -149,15 +147,19 @@ $(BUILD_DIR): $(EXAMPLES_BUILD_DIR): md "$(EXAMPLES_BUILD_DIR)" -$(BUILD_DIR)\ir-test.exe: $(SRC_DIR)/ir-test.cxx - "$(CXX)" /std:c++17 $(SRC_DIR)\ir-test.cxx /Fe:$@ +$(BUILD_DIR)\tester.exe: $(SRC_DIR)/tools/tester.c + "$(CC)" $(CFLAGS) $(SRC_DIR)\tools\tester.c /Fe:$@ -test: $(BUILD_DIR)\ir.exe $(BUILD_DIR)\ir-test.exe +test: $(BUILD_DIR)\ir.exe $(BUILD_DIR)\tester.exe $(BUILD_DIR)\ir.exe $(SRC_DIR)\test.ir --dump --save $(BUILD_DIR)\test.log - $(BUILD_DIR)\ir-test.exe $(SRC_DIR)\tests + $(BUILD_DIR)\tester.exe --test-cmd $(BUILD_DIR)\ir.exe --target $(TEST_TARGET) --default-args "--save" \ + --additional-args "--no-abort-fault" \ + --test-extension ".irt" --code-extension ".ir" $(SRC_DIR)\tests -test-ci: $(BUILD_DIR)\ir.exe $(BUILD_DIR)\ir-test.exe - $(BUILD_DIR)\ir-test.exe --show-diff $(SRC_DIR)\tests +test-ci: $(BUILD_DIR)\ir.exe $(BUILD_DIR)\tester.exe + $(BUILD_DIR)\tester.exe --test-cmd $(BUILD_DIR)\ir.exe --target $(TEST_TARGET) --default-args "--save" \ + --additional-args "--no-abort-fault" --diff-cmd "diff --strip-trailing-cr -u" \ + --test-extension ".irt" --code-extension ".ir" --show-diff $(SRC_DIR)\tests examples: $(OBJS_COMMON) $(EXAMPLES_BUILD_DIR) $(EXAMPLE_EXES)