/* * IR - Lightweight JIT Compilation Framework * (Test runner implementation) * Copyright (C) 2023 by IR project. * Authors: Dmitry Stogov , Anatol Belski */ #include #include #include #include #include #include #include #include #include #include #include #ifdef _WIN32 # include # define popen _popen # define pclose _pclose # define PATH_SEP "\\" # define EXE_SUF ".exe" # define REGEX_FLAGS std::regex::extended # define DIFF_CMD "fc.exe" # define DIFF_ARGS "/n" # define IR_ARGS "--no-abort-fault" #else # define PATH_SEP "/" # define EXE_SUF "" # define REGEX_FLAGS std::regex::ECMAScript | std::regex::multiline # define DIFF_CMD "diff" # define DIFF_ARGS "--strip-trailing-cr -u" # define IR_ARGS "" #endif uint32_t skipped = 0; std::vector bad_list; std::vector> failed; std::vector> xfailed_list; bool show_diff = false, colorize = true; std::string ir_exe, ir_target; std::string this_exe, this_exe_path; namespace ir { decltype(auto) trim(std::string s) { const char* ws = " \t\n\r\f\v"; s.erase(0, s.find_first_not_of(ws)); s.erase(s.find_last_not_of(ws) + 1); return s; } decltype(auto) exec(const std::string& cmd) { std::array buffer; std::string result; std::unique_ptr pipe(popen(cmd.c_str(), "r"), pclose); if (!pipe) { throw std::runtime_error("popen() failed!"); } while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) { result += buffer.data(); } return result; } enum color { GREEN, YELLOW, RED }; decltype(auto) colorize(const std::string& s, enum color c) { if (::colorize) switch (c) { case GREEN: return "\x1b[1;32m" + s + "\x1b[0m"; case YELLOW: return "\x1b[1;33m" + s + "\x1b[0m"; case RED: return "\x1b[1;31m" + s + "\x1b[0m"; } return "" + s; } void init_console() { #ifdef _WIN32 if (::colorize) { DWORD mode, new_mode = ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN; HANDLE h = GetStdHandle(STD_OUTPUT_HANDLE); GetConsoleMode(h, &mode); SetConsoleMode(h, mode | new_mode); } #endif } class broken_test_exception : public std::exception {} broken_test; struct it_sect { std::size_t name_pos; std::string name; std::size_t content_pos; std::string content; }; struct test { // Section contents std::string name, code, expect, target, xfail, args; // Test file path and contents and output filenames std::string irt_file, irt_file_content, ir_file, out_file, exp_file, diff_file; test(const std::string& test_fl) : irt_file{test_fl} { std::ifstream f(test_fl); if (!f) throw "Couldn't read '" + test_fl + "'"; std::ostringstream oss; oss << f.rdbuf(); irt_file_content = oss.str(); auto make_fn = [&test_fl](const std::string& s) { return test_fl.substr(0, test_fl.length() - 4) + s; }; ir_file = make_fn(".ir"); out_file = make_fn(".out"); exp_file = make_fn(".exp"); diff_file = make_fn(".diff"); parse(); } void parse() { std::regex sect_reg("^--[A-Z]+--$", REGEX_FLAGS); auto sect_begin = std::sregex_iterator(irt_file_content.begin(), irt_file_content.end(), sect_reg); auto sect_end = std::sregex_iterator(); std::vector test_data; // PASS 0: Find all the test sections auto found_sects = std::distance(sect_begin, sect_end); for (auto i = sect_begin; i != sect_end; i++) { it_sect iti; std::smatch m = *i; iti.name_pos = m.position(); iti.name = m.str(); test_data.push_back(std::move(iti)); } // PASS 1: Read section contents auto it = test_data.begin(); auto it_next = it; it_next++; for (; it != test_data.end(); it++) { it->content_pos = it->name_pos + it->name.length(); if (test_data.end() == it_next) { it->content = trim(irt_file_content.substr(it->content_pos, irt_file_content.length() - it->content_pos)); } else { it->content = trim(irt_file_content.substr(it->content_pos, it_next->name_pos - it->content_pos)); it_next++; } } // PASS 2: Save necessary data into properties #define NO_SECT test_data.end() auto get_sect = [&test_data](std::string s) -> decltype(auto) { for (auto test_it = test_data.begin(); test_it != test_data.end(); test_it++) if (!test_it->name.compare(s)) return test_it; return NO_SECT; }; auto test_sect = get_sect("--TEST--"); auto code_sect = get_sect("--CODE--"); auto exp_sect = get_sect("--EXPECT--"); if (NO_SECT == test_sect || NO_SECT == code_sect || NO_SECT == exp_sect) { throw broken_test; } name = test_sect->content; code = code_sect->content; expect = exp_sect->content; auto args_sect = get_sect("--ARGS--"); args = (NO_SECT == args_sect) ? "--save": args_sect->content; auto tgt_sect = get_sect("--TARGET--"); if (NO_SECT != tgt_sect) target = tgt_sect->content; auto xfail_sect = get_sect("--XFAIL--"); if (NO_SECT != xfail_sect) xfail = xfail_sect->content; } bool skip() { return target.length() > 0 && target.compare(::ir_target); } bool run() { std::ofstream in_os(ir_file); in_os << code; in_os.close(); auto test_cmd = ::ir_exe + " " + ir_file + " " + args + " " + IR_ARGS + " >" + out_file + " 2>&1"; auto diff_cmd = std::string(DIFF_CMD) + " " + DIFF_ARGS + " " + exp_file + " " + out_file + " > " + diff_file + " 2>&1"; int ret_code = std::system(test_cmd.c_str()) >> 0x8; std::stringstream out_buf; out_buf << std::ifstream(out_file).rdbuf(); std::string out = trim(out_buf.str()); out.erase(std::remove(out.begin(), out.end(), '\r'), out.end()); if (ret_code || out.compare(expect)) { std::ofstream exp_os(exp_file); exp_os << (expect + "\n"); exp_os.close(); // XXX move away from diff tool dependency ret_code = std::system(diff_cmd.c_str()) >> 0x8; if (0 != ret_code && 1 != ret_code) { // 1 stands for "there is a diff" std::cout << diff_cmd << std::endl; std::cout << ret_code << std::endl; throw "Couldn't compare output vs. expected result for '" + irt_file + "'"; } return false; } std::remove(out_file.c_str()); std::remove(exp_file.c_str()); std::remove(diff_file.c_str()); std::remove(ir_file.c_str()); return true; } }; void find_tests_in_dir(std::set& user_files, std::vector& irt_files) { assert(user_files.size()>0); for (const auto& f : user_files) { if (std::filesystem::is_directory(f)) { for (const std::filesystem::directory_entry& ent : std::filesystem::recursive_directory_iterator(f)) { std::string fl(ent.path().string()); if (fl.length() < 4 || ent.path().extension() != ".irt") continue; irt_files.push_back(fl); } } else { irt_files.push_back(f); } } std::sort(irt_files.begin(), irt_files.end(), [&](const std::string& a, const std::string& b) { return 1 <= b.compare(a); }); } } static void print_help(void) { std::cout << "Run IR uint tests\n"; std::cout << "Usage:\n "; std::cout << ::this_exe << " [--show-diff] [--no-color] [test folders or files...]\n"; std::cout << " Run all tests if no test folders/files are specified\n"; } int main(int argc, char **argv) { // Save this executable path to infer ir executable path ::this_exe = argv[0]; ::this_exe_path = std::filesystem::path{::this_exe}.parent_path().string(); // Store test folders/files specified by user std::set user_files; bool bad_opt = false; // unsupported option given bool bad_files = false; // bad file or folder given for (int i = 1; i < argc; i++) { // XXX use some cleaner arg parsing solution if (!std::string(argv[i]).compare("--show-diff")) { ::show_diff = true; continue; } else if (!std::string(argv[i]).compare("--no-color")) { ::colorize = false; continue; } else if (!std::string(argv[i]).compare("--help")) { print_help(); return 0; } else if (std::string(argv[i]).find_first_of("-") == 0) { // Unsupported options bad_opt = true; std::cerr << ir::colorize("ERROR", ir::RED) << ": Unsupported Option [" << argv[i] << "]\n"; } else { // User specified test folders/files std::string file = argv[i]; if (std::filesystem::exists(file) && (std::filesystem::is_directory(file) || std::filesystem::is_regular_file(file))) { user_files.insert(argv[i]); } else { bad_files = true; std::cerr << ir::colorize("ERROR", ir::RED) << ": Bad File or Folder [" << file << "] \n"; } } } /* for: arguments iteration */ if (bad_opt || bad_files) { if (bad_opt) print_help(); return 1; } ir::init_console(); ::ir_exe = ::this_exe_path + PATH_SEP + "ir" + EXE_SUF; ::ir_target = ir::trim(ir::exec(::ir_exe + " --target")); // Get test files, either specified by user or all tests by default std::vector irt_files; if (user_files.empty()) { // Pretend user specified all test std::string tests_dir = ::this_exe_path + PATH_SEP + "tests"; user_files.insert(tests_dir); } ir::find_tests_in_dir(user_files, irt_files); // Run each test for (const std::string& test_fl : irt_files) { try { auto test = ir::test(test_fl); size_t idx = &test_fl - &irt_files[0] + 1; std::string test_ln_0 = "TEST: " + std::to_string(idx) + PATH_SEP + std::to_string(irt_files.size()) + " " + test.name + "[" + test_fl + "]\r"; std::cout << test_ln_0 << std::flush; if (test.skip()) { std::cout << std::string(test_ln_0.length(), ' '); std::cout << "\r" << ir::colorize("SKIP", ir::YELLOW) << ": " << test.name << " [" << test_fl << "]\n"; ::skipped++; continue; } auto ret = test.run(); std::cout << std::string(test_ln_0.length(), ' '); if (ret) { std::cout << "\r" << ir::colorize("PASS", ir::GREEN) << ": " << test.name << " [" << test_fl << "]\n"; } else if (test.xfail.length() > 0) { std::cout << "\r" << ir::colorize("XFAIL", ir::RED) << ": " << test.name << " [" << test_fl << "] XFAIL REASON: " << test.xfail << "\n"; ::xfailed_list.push_back({test.name, test.irt_file, test.xfail}); } else { std::cout << "\r" << ir::colorize("FAIL", ir::RED) << ": " << test.name << " [" << test_fl << "]\n"; ::failed.push_back({test.name, test.irt_file}); if (::show_diff) { std::ifstream f(test.diff_file); if (!f) { throw "Couldn't read '" + test.diff_file + "'"; } std::cout << f.rdbuf() << std::endl; f.close(); } } } catch (ir::broken_test_exception& e) { std::cout << "\r" << ir::colorize("BROK", ir::RED) << ": [" << test_fl << "]\n"; ::bad_list.push_back(test_fl); } catch (std::string& s) { std::cout << "\r" << ir::colorize("ERROR", ir::RED) << ": " << s << '\n'; } catch (std::exception& e) { std::cout << "\r" << ir::colorize("ERROR", ir::RED) << ": " << e.what() << '\n'; } // XXX parallelize } // Produce the summary #define SEPARATOR() std::cout << std::string(32, '-') << std::endl #define WRAP_OUT(expr) SEPARATOR(); expr; SEPARATOR() WRAP_OUT(std::cout << "Test Summary" << std::endl); if (::bad_list.size() > 0) { std::cout << "Bad tests: " << ::bad_list.size() << std::endl; WRAP_OUT(for (const std::string& fname : ::bad_list) std::cout << fname << std::endl); } std::cout << "Total: " << irt_files.size() << std::endl; std::cout << "Passed: " << (irt_files.size() - ::failed.size() - ::skipped) << std::endl; std::cout << "Expected fail: " << ::xfailed_list.size() << std::endl; std::cout << "Failed: " << ::failed.size() << std::endl; std::cout << "Skipped: " << ::skipped << std::endl; if (::xfailed_list.size() > 0) { WRAP_OUT(std::cout << "EXPECTED FAILED TESTS" << std::endl); for (const auto& t : ::xfailed_list) { std::cout << std::get<0>(t) << " [" << std::get<1>(t) << "] XFAIL REASON: " << std::get<2>(t) << std::endl; } } if (::failed.size() > 0) { WRAP_OUT(std::cout << "FAILED TESTS" << std::endl); for (const auto& t : ::failed) { std::cout << std::get<0>(t) << " [" << std::get<1>(t) << "]" << std::endl; } } SEPARATOR(); return ::failed.size() > 0 ? 1 : 0; }