danog/ir
1
0
Fork 0
mirror of https://github.com/danog/ir.git synced 2024-11-30 04:39:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

Reimplement tester program in C and in IR independent way (#50)

Browse Source 
* Reimplement tester program in C and in IR independent way

* Fix Windows support
This commit is contained in:
Dmitry Stogov 2023-09-27 00:34:03 +03:00 committed by GitHub
parent 9a8cbdf28c
commit 4fef47b1a5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 689 additions and 648 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.github/workflows/push.yml vendored
View File

@ -22,14 +22,14 @@ jobs:
run: |
apt update -y
DEBIAN_FRONTEND=noninteractive apt install -y \
gcc g++-12 \
gcc \
libcapstone-dev \
libcapstone4 \
make
- name: make
run: make TARGET=x86_64 all
- name: test
run: make CXX=g++-12 TARGET=x86_64 test-ci
run: make TARGET=x86_64 test-ci
Linux_i386:
name: 'Linux i386'
runs-on: ubuntu-22.04
@ -44,7 +44,7 @@ jobs:
dpkg --add-architecture i386
apt update -y
DEBIAN_FRONTEND=noninteractive apt install -y \
gcc g++-12 \
gcc \
gcc-multilib \
libc6:i386 \
make \
@ -62,7 +62,7 @@ jobs:
- name: make
run: make TARGET=x86 all
- name: test
run: make CXX=g++-12 TARGET=x86 test-ci
run: make TARGET=x86 test-ci
Linux_aarch64:
name: 'Linux aarch64'
runs-on: ubuntu-20.04
@ -78,7 +78,7 @@ jobs:
install: |
apt update -y
DEBIAN_FRONTEND=noninteractive apt install -y \
gcc g++-12 \
gcc \
libc6 \
libcapstone-dev \
libcapstone4 \
@ -86,7 +86,7 @@ jobs:
run: |
make CC=gcc TARGET=aarch64 all
# FIXME: For some reason some of the object files are rebuilt
make CC=gcc CXX=g++-12 TARGET=aarch64 test-ci
make CC=gcc TARGET=aarch64 test-ci
MACOS_x86_64:
runs-on: macos-11
@ -99,7 +99,7 @@ jobs:
- name: make
run: make TARGET=x86_64 all
- name: test
run: make CXX=g++-12 TARGET=x86_64 test-ci
run: make TARGET=x86_64 test-ci
Windows:
strategy:

2
.gitignore vendored
View File

@ -8,7 +8,7 @@ ir_emit_aarch64.h
minilua
gen_ir_fold_hash
ir_test
ir-test
tester
ir
b.c

17
Makefile
View File

@ -8,7 +8,6 @@ EXAMPLES_SRC_DIR = $(SRC_DIR)/examples
EXAMPLES_BUILD_DIR = $(BUILD_DIR)/examples
CC = gcc
CXX = g++
BUILD_CC = gcc
override CFLAGS += -Wall -Wextra -Wno-unused-parameter
LDFLAGS = -lm -ldl
@ -94,17 +93,19 @@ $(OBJS_COMMON) $(OBJS_IR) $(OBJS_IR_TEST): $(BUILD_DIR)/$(notdir %.o): $(SRC_DIR
$(EXAMPLE_EXES): $(EXAMPLES_BUILD_DIR)/$(notdir %): $(EXAMPLES_SRC_DIR)/$(notdir %.c)
$(CC) $(CFLAGS) -I$(SRC_DIR) $< -o $@ $(OBJS_COMMON) $(LDFLAGS) -lcapstone
$(BUILD_DIR)/ir-test: $(SRC_DIR)/ir-test.cxx
$(CXX) -O3 -std=c++17 $(SRC_DIR)/ir-test.cxx -o $(BUILD_DIR)/ir-test
$(BUILD_DIR)/tester: $(SRC_DIR)/tools/tester.c
$(CC) $(CFLAGS) -o $@ $<
test: $(BUILD_DIR)/ir $(BUILD_DIR)/ir-test
test: $(BUILD_DIR)/ir $(BUILD_DIR)/tester
$(BUILD_DIR)/ir $(SRC_DIR)/test.ir --dump --save 2>$(BUILD_DIR)/test.log
$(BUILD_DIR)/ir $(SRC_DIR)/test.ir --dot $(BUILD_DIR)/ir.dot
dot -Tpdf $(BUILD_DIR)/ir.dot -o $(BUILD_DIR)/ir.pdf
$(BUILD_DIR)/ir-test $(SRC_DIR)/tests
$(BUILD_DIR)/tester --test-cmd $(BUILD_DIR)/ir --target $(TARGET) --default-args "--save" \
--test-extension ".irt" --code-extension ".ir" $(SRC_DIR)/tests
test-ci: $(BUILD_DIR)/ir $(BUILD_DIR)/ir-test
$(BUILD_DIR)/ir-test --show-diff $(SRC_DIR)/tests
test-ci: $(BUILD_DIR)/ir $(BUILD_DIR)/tester
$(BUILD_DIR)/tester --test-cmd $(BUILD_DIR)/ir --target $(TARGET) --default-args "--save" \
--test-extension ".irt" --code-extension ".ir" --show-diff $(SRC_DIR)/tests
examples: $(OBJS_COMMON) $(EXAMPLES_BUILD_DIR) $(EXAMPLE_EXES)
@ -113,7 +114,7 @@ clean:
$(BUILD_DIR)/minilua $(BUILD_DIR)/ir_emit_$(DASM_ARCH).h \
$(BUILD_DIR)/ir_fold_hash.h $(BUILD_DIR)/gen_ir_fold_hash \
$(BUILD_DIR)/ir.dot $(BUILD_DIR)/ir.pdf $(BUILD_DIR)/test.log \
$(BUILD_DIR)/ir-test
$(BUILD_DIR)/tester
find $(SRC_DIR)/tests -type f -name '*.diff' -delete
find $(SRC_DIR)/tests -type f -name '*.out' -delete
find $(SRC_DIR)/tests -type f -name '*.exp' -delete

391
ir-test.cxx
View File

@ -1,391 +0,0 @@
/*
* IR - Lightweight JIT Compilation Framework
* (Test runner implementation)
* Copyright (C) 2023 by IR project.
* Authors: Dmitry Stogov <dmitry@php.net>, Anatol Belski <anbelski@linux.microsoft.com>
*/
#include <string>
#include <regex>
#include <iostream>
#include <fstream>
#include <sstream>
#include <filesystem>
#include <array>
#include <set>
#include <exception>
#include <cstdlib>
#include <cassert>
#ifdef _WIN32
# include <windows.h>
# define popen _popen
# define pclose _pclose
# define PATH_SEP "\\"
# define EXE_SUF ".exe"
# define REGEX_FLAGS std::regex::extended
# define DIFF_CMD "fc.exe"
# define DIFF_ARGS "/n"
# define IR_ARGS "--no-abort-fault"
#else
# define PATH_SEP "/"
# define EXE_SUF ""
# define REGEX_FLAGS std::regex::ECMAScript | std::regex::multiline
# define DIFF_CMD "diff"
# define DIFF_ARGS "--strip-trailing-cr -u"
# define IR_ARGS ""
#endif
uint32_t skipped = 0;
std::vector<std::string> bad_list;
std::vector<std::tuple<std::string, std::string>> failed;
std::vector<std::tuple<std::string, std::string, std::string>> xfailed_list;
bool show_diff = false, colorize = true;
std::string ir_exe, ir_target;
std::string this_exe, this_exe_path;
namespace ir {
decltype(auto) trim(std::string s) {
const char* ws = " \t\n\r\f\v";
s.erase(0, s.find_first_not_of(ws));
s.erase(s.find_last_not_of(ws) + 1);
return s;
}
decltype(auto) exec(const std::string& cmd) {
std::array<char, 256> buffer;
std::string result;
std::unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd.c_str(), "r"), pclose);
if (!pipe) {
throw std::runtime_error("popen() failed!");
}
while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) {
result += buffer.data();
}
return result;
}
enum color { GREEN, YELLOW, RED };
decltype(auto) colorize(const std::string& s, enum color c) {
if (::colorize)
switch (c) {
case GREEN:
return "\x1b[1;32m" + s + "\x1b[0m";
case YELLOW:
return "\x1b[1;33m" + s + "\x1b[0m";
case RED:
return "\x1b[1;31m" + s + "\x1b[0m";
}
return "" + s;
}
void init_console() {
#ifdef _WIN32
if (::colorize) {
DWORD mode, new_mode = ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN;
HANDLE h = GetStdHandle(STD_OUTPUT_HANDLE);
GetConsoleMode(h, &mode);
SetConsoleMode(h, mode | new_mode);
}
#endif
}
class broken_test_exception : public std::exception {} broken_test;
struct it_sect {
std::size_t name_pos;
std::string name;
std::size_t content_pos;
std::string content;
};
struct test {
// Section contents
std::string name, code, expect, target, xfail, args;
// Test file path and contents and output filenames
std::string irt_file, irt_file_content, ir_file, out_file, exp_file, diff_file;
test(const std::string& test_fl) : irt_file{test_fl} {
std::ifstream f(test_fl);
if (!f) throw "Couldn't read '" + test_fl + "'";
std::ostringstream oss;
oss << f.rdbuf();
irt_file_content = oss.str();
auto make_fn = [&test_fl](const std::string& s) {
return test_fl.substr(0, test_fl.length() - 4) + s;
};
ir_file = make_fn(".ir");
out_file = make_fn(".out");
exp_file = make_fn(".exp");
diff_file = make_fn(".diff");
parse();
}
void parse() {
std::regex sect_reg("^--[A-Z]+--$", REGEX_FLAGS);
auto sect_begin = std::sregex_iterator(irt_file_content.begin(), irt_file_content.end(), sect_reg);
auto sect_end = std::sregex_iterator();
std::vector<it_sect> test_data;
// PASS 0: Find all the test sections
auto found_sects = std::distance(sect_begin, sect_end);
for (auto i = sect_begin; i != sect_end; i++) {
it_sect iti;
std::smatch m = *i;
iti.name_pos = m.position();
iti.name = m.str();
test_data.push_back(std::move(iti));
}
// PASS 1: Read section contents
auto it = test_data.begin();
auto it_next = it;
it_next++;
for (; it != test_data.end(); it++) {
it->content_pos = it->name_pos + it->name.length();
if (test_data.end() == it_next) {
it->content = trim(irt_file_content.substr(it->content_pos, irt_file_content.length() - it->content_pos));
} else {
it->content = trim(irt_file_content.substr(it->content_pos, it_next->name_pos - it->content_pos));
it_next++;
}
}
// PASS 2: Save necessary data into properties
#define NO_SECT test_data.end()
auto get_sect = [&test_data](std::string s) -> decltype(auto) {
for (auto test_it = test_data.begin(); test_it != test_data.end(); test_it++)
if (!test_it->name.compare(s)) return test_it;
return NO_SECT;
};
auto test_sect = get_sect("--TEST--");
auto code_sect = get_sect("--CODE--");
auto exp_sect = get_sect("--EXPECT--");
if (NO_SECT == test_sect || NO_SECT == code_sect || NO_SECT == exp_sect) {
throw broken_test;
}
name = test_sect->content;
code = code_sect->content;
expect = exp_sect->content;
auto args_sect = get_sect("--ARGS--");
args = (NO_SECT == args_sect) ? "--save": args_sect->content;
auto tgt_sect = get_sect("--TARGET--");
if (NO_SECT != tgt_sect) target = tgt_sect->content;
auto xfail_sect = get_sect("--XFAIL--");
if (NO_SECT != xfail_sect) xfail = xfail_sect->content;
}
bool skip() {
return target.length() > 0 && target.compare(::ir_target);
}
bool run() {
std::ofstream in_os(ir_file);
in_os << code;
in_os.close();
auto test_cmd = ::ir_exe + " " + ir_file + " " + args + " " + IR_ARGS + " >" + out_file + " 2>&1";
auto diff_cmd = std::string(DIFF_CMD) + " " + DIFF_ARGS + " " + exp_file + " " + out_file + " > " + diff_file + " 2>&1";
int ret_code = std::system(test_cmd.c_str()) >> 0x8;
std::stringstream out_buf;
out_buf << std::ifstream(out_file).rdbuf();
std::string out = trim(out_buf.str());
out.erase(std::remove(out.begin(), out.end(), '\r'), out.end());
if (ret_code || out.compare(expect)) {
std::ofstream exp_os(exp_file);
exp_os << (expect + "\n");
exp_os.close();
// XXX move away from diff tool dependency
ret_code = std::system(diff_cmd.c_str()) >> 0x8;
if (0 != ret_code && 1 != ret_code) { // 1 stands for "there is a diff"
std::cout << diff_cmd << std::endl;
std::cout << ret_code << std::endl;
throw "Couldn't compare output vs. expected result for '" + irt_file + "'";
}
return false;
}
std::remove(out_file.c_str());
std::remove(exp_file.c_str());
std::remove(diff_file.c_str());
std::remove(ir_file.c_str());
return true;
}
};
void find_tests_in_dir(std::set<std::string>& user_files, std::vector<std::string>& irt_files) {
assert(user_files.size()>0);
for (const auto& f : user_files) {
if (std::filesystem::is_directory(f)) {
for (const std::filesystem::directory_entry& ent :
std::filesystem::recursive_directory_iterator(f)) {
std::string fl(ent.path().string());
if (fl.length() < 4 || ent.path().extension() != ".irt")
continue;
irt_files.push_back(fl);
}
} else {
irt_files.push_back(f);
}
}
std::sort(irt_files.begin(),
irt_files.end(),
[&](const std::string& a, const std::string& b) {
return 1 <= b.compare(a);
});
}
}
static void print_help(void)
{
std::cout << "Run IR uint tests\n";
std::cout << "Usage:\n ";
std::cout << ::this_exe
<< " [--show-diff] [--no-color] [test folders or files...]\n";
std::cout << " Run all tests if no test folders/files are specified\n";
}
int main(int argc, char **argv) {
// Save this executable path to infer ir executable path
::this_exe = argv[0];
::this_exe_path = std::filesystem::path{::this_exe}.parent_path().string();
// Store test folders/files specified by user
std::set<std::string> user_files;
bool bad_opt = false; // unsupported option given
bool bad_files = false; // bad file or folder given
for (int i = 1; i < argc; i++) {
// XXX use some cleaner arg parsing solution
if (!std::string(argv[i]).compare("--show-diff")) {
::show_diff = true;
continue;
} else if (!std::string(argv[i]).compare("--no-color")) {
::colorize = false;
continue;
} else if (!std::string(argv[i]).compare("--help")) {
print_help();
return 0;
} else if (std::string(argv[i]).find_first_of("-") == 0) {
// Unsupported options
bad_opt = true;
std::cerr << ir::colorize("ERROR", ir::RED)
<< ": Unsupported Option [" << argv[i] << "]\n";
} else {
// User specified test folders/files
std::string file = argv[i];
if (std::filesystem::exists(file)
&& (std::filesystem::is_directory(file)
|| std::filesystem::is_regular_file(file))) {
user_files.insert(argv[i]);
} else {
bad_files = true;
std::cerr << ir::colorize("ERROR", ir::RED)
<< ": Bad File or Folder [" << file << "] \n";
}
}
} /* for: arguments iteration */
if (bad_opt || bad_files) {
if (bad_opt)
print_help();
return 1;
}
ir::init_console();
::ir_exe = ::this_exe_path + PATH_SEP + "ir" + EXE_SUF;
::ir_target = ir::trim(ir::exec(::ir_exe + " --target"));
// Get test files, either specified by user or all tests by default
std::vector<std::string> irt_files;
if (user_files.empty()) {
// Pretend user specified all test
std::string tests_dir = ::this_exe_path + PATH_SEP + "tests";
user_files.insert(tests_dir);
}
ir::find_tests_in_dir(user_files, irt_files);
// Run each test
for (const std::string& test_fl : irt_files) {
try {
auto test = ir::test(test_fl);
size_t idx = &test_fl - &irt_files[0] + 1;
std::string test_ln_0 = "TEST: " + std::to_string(idx) + PATH_SEP + std::to_string(irt_files.size()) + " " + test.name + "[" + test_fl + "]\r";
std::cout << test_ln_0 << std::flush;
if (test.skip()) {
std::cout << std::string(test_ln_0.length(), ' ');
std::cout << "\r" << ir::colorize("SKIP", ir::YELLOW) << ": " << test.name << " [" << test_fl << "]\n";
::skipped++;
continue;
}
auto ret = test.run();
std::cout << std::string(test_ln_0.length(), ' ');
if (ret) {
std::cout << "\r" << ir::colorize("PASS", ir::GREEN) << ": " << test.name << " [" << test_fl << "]\n";
} else if (test.xfail.length() > 0) {
std::cout << "\r" << ir::colorize("XFAIL", ir::RED) << ": " << test.name << " [" << test_fl << "] XFAIL REASON: " << test.xfail << "\n";
::xfailed_list.push_back({test.name, test.irt_file, test.xfail});
} else {
std::cout << "\r" << ir::colorize("FAIL", ir::RED) << ": " << test.name << " [" << test_fl << "]\n";
::failed.push_back({test.name, test.irt_file});
if (::show_diff) {
std::ifstream f(test.diff_file);
if (!f) {
throw "Couldn't read '" + test.diff_file + "'";
}
std::cout << f.rdbuf() << std::endl;
f.close();
}
}
} catch (ir::broken_test_exception& e) {
std::cout << "\r" << ir::colorize("BROK", ir::RED) << ": [" << test_fl << "]\n";
::bad_list.push_back(test_fl);
} catch (std::string& s) {
std::cout << "\r" << ir::colorize("ERROR", ir::RED) << ": " << s << '\n';
} catch (std::exception& e) {
std::cout << "\r" << ir::colorize("ERROR", ir::RED) << ": " << e.what() << '\n';
}
// XXX parallelize
}
// Produce the summary
#define SEPARATOR() std::cout << std::string(32, '-') << std::endl
#define WRAP_OUT(expr) SEPARATOR(); expr; SEPARATOR()
WRAP_OUT(std::cout << "Test Summary" << std::endl);
if (::bad_list.size() > 0) {
std::cout << "Bad tests: " << ::bad_list.size() << std::endl;
WRAP_OUT(for (const std::string& fname : ::bad_list) std::cout << fname << std::endl);
}
std::cout << "Total: " << irt_files.size() << std::endl;
std::cout << "Passed: " << (irt_files.size() - ::failed.size() - ::skipped) << std::endl;
std::cout << "Expected fail: " << ::xfailed_list.size() << std::endl;
std::cout << "Failed: " << ::failed.size() << std::endl;
std::cout << "Skipped: " << ::skipped << std::endl;
if (::xfailed_list.size() > 0) {
WRAP_OUT(std::cout << "EXPECTED FAILED TESTS" << std::endl);
for (const auto& t : ::xfailed_list) {
std::cout << std::get<0>(t) << " [" << std::get<1>(t) << "] XFAIL REASON: " << std::get<2>(t) << std::endl;
}
}
if (::failed.size() > 0) {
WRAP_OUT(std::cout << "FAILED TESTS" << std::endl);
for (const auto& t : ::failed) {
std::cout << std::get<0>(t) << " [" << std::get<1>(t) << "]" << std::endl;
}
}
SEPARATOR();
return ::failed.size() > 0 ? 1 : 0;
}

231
ir-test.php
View File

@ -1,231 +0,0 @@
<?php
/*
* IR - Lightweight JIT Compilation Framework
* (Test driver)
* Copyright (C) 2022 Zend by Perforce.
* Authors: Dmitry Stogov <dmitry@php.net>
*/
function parse_test($test, &$name, &$code, &$expect, &$args, &$target, &$xfail) {
$sections = array();
$text = @file_get_contents($test);
if (!$text || !preg_match_all("/^--[A-Z]+--$/m", $text, $r, PREG_SET_ORDER | PREG_OFFSET_CAPTURE)) {
return false;
}
foreach($r as $sect) {
if (isset($sect_name)) {
$sections[$sect_name] = trim(substr($text, $sect_offset, $sect[0][1] - $sect_offset));
}
$sect_name = $sect[0][0];
$sect_offset = $sect[0][1] + strlen($sect_name);
}
if (isset($sect_name)) {
$sections[$sect_name] = trim(substr($text, $sect_offset));
}
if (!isset($sections['--TEST--']) || !isset($sections['--CODE--']) || !isset($sections['--EXPECT--'])) {
return false;
}
$name = $sections['--TEST--'];
$code = $sections['--CODE--'];
$expect = $sections['--EXPECT--'];
$args = isset($sections['--ARGS--']) ? $sections['--ARGS--'] : "--save";
$target = isset($sections['--TARGET--']) ? $sections['--TARGET--'] : null;
$xfail = isset($sections['--XFAIL--']) ? $sections['--XFAIL--'] : null;
return true;
}
function run_test($build_dir, $test, $name, $code, $expect, $args) {
$base = substr($test, 0, -4);
$input = $base . ".ir";
$output = $base . ".out";
@unlink($input);
@unlink($output);
@unlink("$base.exp");
@unlink("$base.diff");
if (!@file_put_contents($input, $code)) {
return false;
}
if (PHP_OS_FAMILY != "Windows") {
$cmd = "$build_dir/ir $input $args >$output 2>&1";
} else {
$cmd = "$build_dir\\ir $input $args --no-abort-fault >$output 2>&1";
}
$ret = @system($cmd, $result_code);
if ($ret === false) {
return false;
}
if ($result_code) {
}
$out = @file_get_contents($output);
if ($out === false) {
return false;
}
if ($result_code) {
$out = "\nExit Code = $result_code\n";
file_put_contents($output, $out);
}
$out = trim($out);
$out = str_replace("\r", "", $out);
if ($out !== $expect) {
if (!@file_put_contents("$base.exp", "$expect\n")) {
return false;
}
if (PHP_OS_FAMILY != "Windows") {
$cmd = "diff -u $base.exp $output > $base.diff";
} else {
/* Diff somehow resets terminal and breaks "cooring" */
$cmd = "diff --strip-trailing-cr -u $base.exp $output > $base.diff 2>&1";
}
if (@system($cmd) != 0) {
return false;
}
return false;
}
@unlink($input);
@unlink($output);
return true;
}
function find_tests_in_dir($dir, &$tests) {
$d = opendir($dir);
if ($d !== false) {
while (($name = readdir($d)) !== false) {
if ($name === '.' || $name === '..') continue;
$fn = "$dir/$name";
if (is_dir($fn)) {
find_tests_in_dir($fn, $tests);
} else if (substr($name, -4) === '.irt') {
$tests[] = $fn;
}
}
closedir($d);
}
}
function find_tests($dir) {
$tests = [];
find_tests_in_dir($dir, $tests);
sort($tests);
return $tests;
}
function run_tests() {
global $show_diff, $colorize;
$build_dir = getenv("BUILD_DIR") ?? ".";
$src_dir = getenv("SRC_DIR") ?? ".";
$skiped = 0;
if (PHP_OS_FAMILY != "Windows") {
$cmd = "$build_dir/ir --target";
} else {
$cmd = "$build_dir\\ir --target";
}
$target = @system($cmd);
if ($target === false) {
echo "Cannot run '$cmd'\n";
return 1;
}
$tests = find_tests("$src_dir/tests");
$bad = array();
$failed = array();
$xfailed = array();
$total = count($tests);
$count = 0;
foreach($tests as $test) {
$count++;
if (parse_test($test, $name, $code, $expect, $opt, $test_target, $xfail)) {
if ($test_target !== null && $target != $test_target) {
if ($colorize) {
echo "\r\e[1;33mSKIP\e[0m: $name [$test]\n";
} else {
echo "\rSKIP: $name [$test]\n";
}
$skiped++;
continue;
}
$str = "TEST: $count/$total $name [$test]\r";
$len = strlen($str);
echo $str;
flush();
$ret = run_test($build_dir, $test, $name, $code, $expect, $opt);
echo str_repeat(" ", $len);
if ($ret) {
if ($colorize) {
echo "\r\e[1;32mPASS\e[0m: $name [$test]\n";
} else {
echo "\rPASS: $name [$test]\n";
}
} else if (isset($xfail)) {
if ($colorize) {
echo "\r\e[1;31mXFAIL\e[0m: $name [$test] XFAIL REASON: $xfail\n";
} else {
echo "\rXFAIL: $name [$test] XFAIL REASON: $xfail\n";
}
$xfailed[$test] = array($name, $xfail);
} else {
if ($colorize) {
echo "\r\e[1;31mFAIL\e[0m: $name [$test]\n";
} else {
echo "\rFAIL: $name [$test]\n";
}
$failed[$test] = $name;
if ($show_diff) {
$base = substr($test, 0, -4);
echo file_get_contents("$base.diff");
}
}
} else {
if ($colorize) {
echo "\r\e[1;31mBROK\e[0m: $name [$test]\n";
} else {
echo "\rBROK: $name [$test]\n";
}
$bad[] = $test;
}
}
echo "-------------------------------\n";
echo "Test Summary\n";
echo "-------------------------------\n";
if (count($bad) > 0) {
echo "Bad tests: " . count($bad) . "\n";
echo "-------------------------------\n";
foreach ($bad as $test) {
echo "$test\n";
}
echo "-------------------------------\n";
}
echo "Total: " . count($tests) . "\n";
echo "Passed: " . (count($tests) - count($failed) - $skiped) . "\n";
echo "Expected fail: " . count($xfailed) . "\n";
echo "Failed: " . count($failed) . "\n";
echo "Skiped: " . $skiped . "\n";
if (count($xfailed) > 0) {
echo "-------------------------------\n";
echo "EXPECTED FAILED TESTS\n";
echo "-------------------------------\n";
foreach ($xfailed as $test => list($name, $xfail)) {
echo "$name [$test] XFAIL REASON: $xfail\n";
}
}
if (count($failed) > 0) {
echo "-------------------------------\n";
echo "FAILED TESTS\n";
echo "-------------------------------\n";
foreach ($failed as $test => $name) {
echo "$name [$test]\n";
}
}
echo "-------------------------------\n";
return count($failed) > 0 ? 1 : 0;
}
$colorize = true;
if (function_exists('sapi_windows_vt100_support') && !sapi_windows_vt100_support(STDOUT, true)) {
$colorize = false;
}
if (in_array('--no-color', $argv, true)) {
$colorize = false;
}
$show_diff = in_array('--show-diff', $argv, true);
exit(run_tests());

660
tools/tester.c Normal file
View File

@ -0,0 +1,660 @@
/*
* IR - Lightweight JIT Compilation Framework
* (Test runner implementation)
* Copyright (C) 2023 by IR project.
* Authors: Dmitry Stogov <dmitry@php.net>, Anatol Belski <anbelski@linux.microsoft.com>
*/
#include <sys/types.h>
#include <sys/stat.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#ifdef _WIN32
# include <io.h>
# include <windows.h>
# pragma warning(disable : 4996)
# define PATH_SEP '\\'
# define DEFAULT_DIFF_CMD "fc"
# define S_IRUSR _S_IREAD
# define S_IWUSR _S_IWRITE
#else
# include <dirent.h>
# include <unistd.h>
# include <alloca.h>
# define PATH_SEP '/'
# define DEFAULT_DIFF_CMD "diff --strip-trailing-cr -u"
# define O_BINARY 0
#endif
typedef enum _color {GREEN, YELLOW, RED} color;
typedef struct _test {
int id;
char *name;
char *target;
char *args;
char *code;
char *expect;
char *xfail;
} test;
static int colorize = 1;
static const char *test_cmd = NULL;
static const char *target = NULL;
static const char *default_args = NULL;
static const char *additional_args = NULL;
static const char *diff_cmd = NULL;
static const char *test_extension = NULL;
static int test_extension_len = 0;
static const char *code_extension = NULL;
static int code_extension_len = 0;
static char **files = NULL;
static int files_count = 0;
static int files_limit = 0;
static void print_color(const char *s, color c)
{
if (colorize) {
switch (c) {
case GREEN:
printf("\x1b[1;32m%s\x1b[0m", s);
return;
case YELLOW:
printf("\x1b[1;33m%s\x1b[0m", s);
return;
case RED:
printf("\x1b[1;31m%s\x1b[0m", s);
return;
}
}
printf("%s", s);
}
static void init_console()
{
#ifdef _WIN32
if (colorize) {
DWORD mode, new_mode = ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN;
HANDLE h = GetStdHandle(STD_OUTPUT_HANDLE);
GetConsoleMode(h, &mode);
SetConsoleMode(h, mode | new_mode);
}
#endif
}
static test *parse_file(const char *filename, int id)
{
test *t;
char *buf, **section;
int fd;
size_t start, i, size;
struct stat stat_buf;
if (stat(filename, &stat_buf) != 0) {
return NULL;
}
size = stat_buf.st_size;
fd = open(filename, O_RDONLY | O_BINARY, 0);
if (fd < 0) {
return NULL;
}
t = malloc(sizeof(test) + size + 1);
if (!t) {
return NULL;
}
buf = (char*)t + sizeof(test);
if ((size_t)read(fd, buf, size) != size) {
free(t);
return NULL;
}
close(fd);
memset(t, 0, sizeof(test));
buf[size] = 0;
i = 0;
while (i < size) {
start = i;
while (i < size && buf[i] != '\r' && buf[i] != '\n') i++;
if (i - start == strlen("--TEST--") && memcmp(buf + start, "--TEST--", strlen("--TEST--")) == 0) {
section = &t->name;
} else if (i - start == strlen("--ARGS--") && memcmp(buf + start, "--ARGS--", strlen("--ARGS--")) == 0) {
section = &t->args;
} else if (i - start == strlen("--CODE--") && memcmp(buf + start, "--CODE--", strlen("--CODE--")) == 0) {
section = &t->code;
} else if (i - start == strlen("--EXPECT--") && memcmp(buf + start, "--EXPECT--", strlen("--EXPECT--")) == 0) {
section = &t->expect;
} else if (i - start == strlen("--XFAIL--") && memcmp(buf + start, "--XFAIL--", strlen("--XFAIL--")) == 0) {
section = &t->xfail;
} else if (i - start == strlen("--TARGET--") && memcmp(buf + start, "--TARGET--", strlen("--TARGET--")) == 0) {
section = &t->target;
} else {
section = NULL;
while (i < size && (buf[i] == '\r' || buf[i] == '\n')) i++;
}
if (section) {
while (start > 0 && (buf[start - 1] == '\r' || buf[start - 1] == '\n')) start--;
buf[start] = 0;
if (*section) {
free(t);
return NULL;
}
while (i < size && (buf[i] == '\r' || buf[i] == '\n')) i++;
*section = buf + i;
}
}
if (!t->name || !t->code || !t->expect) {
free(t);
return NULL;
}
t->id = id;
return t;
}
static int skip_test(test *t)
{
if (target && (t->target && strcmp(t->target, target) != 0)) {
return 1;
}
return 0;
}
static int same_text(const char *exp, const char *out)
{
int i, j;
while (*exp == '\r' || *exp == '\n') exp++;
while (*out == '\r' || *out == '\n') out++;
while (*exp != 0 && *out != 0) {
i = j = 0;
while (exp[i] != 0 && exp[i] != '\r' && exp[i] != '\n') i++;
while (out[j] != 0 && out[j] != '\r' && out[j] != '\n') j++;
if (i != j || memcmp(exp, out, i) != 0) {
return 0;
}
exp += i;
out += j;
if (*exp == '\r') exp++;
if (*exp == '\n') exp++;
if (*out == '\r') out++;
if (*out == '\n') out++;
}
while (*exp == '\r' || *exp == '\n') exp++;
while (*out == '\r' || *out == '\n') out++;
return *exp == 0 && *out == 0;
}
static char *read_file(const char *filename)
{
struct stat stat_buf;
size_t size;
char *buf;
int fd;
if (stat(filename, &stat_buf) != 0) {
return NULL;
}
size = stat_buf.st_size;
buf = malloc(size + 1);
if (!buf) {
return NULL;
}
fd = open(filename, O_RDONLY | O_BINARY, 0);
if (fd < 0) {
free(buf);
return NULL;
}
if ((size_t)read(fd, buf, size) != size) {
free(buf);
return NULL;
}
close(fd);
buf[size] = 0;
return buf;
}
static int write_file(const char *filename, const char *buf, size_t size)
{
int fd = open(filename, O_CREAT | O_TRUNC | O_WRONLY | O_BINARY, S_IRUSR | S_IWUSR);
if (fd < 0) {
return 0;
}
if ((size_t)write(fd, buf, size) != size) {
return 0;
}
close(fd);
return 1;
}
static char *replace_extension(const char *filename, size_t len, const char *ext, size_t ext_len)
{
char *ret;
if (test_extension) {
ret = malloc(len - test_extension_len + ext_len + 1);
memcpy(ret, filename, len - test_extension_len);
memcpy(ret + len - test_extension_len, ext, ext_len + 1);
} else {
ret = malloc(len + ext_len + 1);
memcpy(ret, filename, len);
memcpy(ret + len, ext, ext_len + 1);
}
return ret;
}
static int run_test(const char *filename, test *t, int show_diff)
{
size_t len;
int ret;
char cmd[4096];
char *code_filename, *out_filename, *exp_filename, *diff_filename, *out;
len = strlen(filename);
code_filename = replace_extension(filename, len, code_extension, code_extension_len);
out_filename = replace_extension(filename, len, ".out", strlen(".out"));
exp_filename = replace_extension(filename, len, ".exp", strlen(".exp"));
diff_filename = replace_extension(filename, len, ".diff", strlen(".diff"));
unlink(code_filename);
unlink(out_filename);
unlink(exp_filename);
unlink(diff_filename);
if (!write_file(code_filename, t->code, strlen(t->code))) {
free(code_filename);
free(code_filename);
free(out_filename);
free(exp_filename);
free(diff_filename);
return 0;
}
if ((size_t)snprintf(cmd, sizeof(cmd), "%s %s %s %s > %s 2>&1",
test_cmd, code_filename,
t->args ? t->args : default_args, additional_args,
out_filename) > sizeof(cmd)) {
free(code_filename);
free(out_filename);
free(exp_filename);
free(diff_filename);
return 0;
}
ret = system(cmd) >> 8;
out = read_file(out_filename);
if (!out) {
out = malloc(1);
out[0] = 0;
}
ret = (same_text(t->expect, out) && ret == 0);
if (ret) {
unlink(code_filename);
unlink(out_filename);
} else {
if (write_file(exp_filename, t->expect, strlen(t->expect))) {
if ((size_t)snprintf(cmd, sizeof(cmd), "%s %s %s > %s",
diff_cmd, exp_filename, out_filename, diff_filename) < sizeof(cmd)) {
system(cmd);
if (show_diff && !t->xfail) {
char *diff = read_file(diff_filename);
printf("\n");
printf("%s", diff);
free(diff);
}
}
}
}
free(out);
free(code_filename);
free(out_filename);
free(exp_filename);
free(diff_filename);
return ret;
}
static void add_file(char *name)
{
if (files_count >= files_limit) {
files_limit += 1024;
files = realloc(files, sizeof(char*) * files_limit);
}
files[files_count++] = name;
}
static void find_files_in_dir(const char *dir_name, size_t dir_name_len)
{
char *name;
size_t len;
#ifdef _WIN32
char buf[MAX_PATH];
HANDLE dir;
WIN32_FIND_DATA info;
memcpy(buf, dir_name, dir_name_len);
buf[dir_name_len] = '\\';
buf[dir_name_len + 1] = '*';
buf[dir_name_len + 2] = 0;
dir = FindFirstFile(buf, &info);
if (dir == INVALID_HANDLE_VALUE) {
fprintf(stderr, "ERROR: Cannot read directory [%s]\n", dir_name);
return;
}
do {
len = strlen(info.cFileName);
if (info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
if ((len == 1 && info.cFileName[0] == '.')
|| (len == 2 && info.cFileName[0] == '.' && info.cFileName[1] == '.')) {
/* skip */
} else {
name = malloc(dir_name_len + len + 2);
memcpy(name, dir_name, dir_name_len);
name[dir_name_len] = PATH_SEP;
memcpy(name + dir_name_len + 1, info.cFileName, len + 1);
find_files_in_dir(name, dir_name_len + len + 1);
free(name);
}
} else /*if (info.dwFileAttributes & FILE_ATTRIBUTE_NORMAL)*/ {
if (!test_extension
|| stricmp(info.cFileName + len - test_extension_len, test_extension) == 0) {
name = malloc(dir_name_len + len + 2);
memcpy(name, dir_name, dir_name_len);
name[dir_name_len] = PATH_SEP;
memcpy(name + dir_name_len + 1, info.cFileName, len + 1);
add_file(name);
}
}
} while (FindNextFile(dir, &info));
FindClose(dir);
#else
DIR *dir = opendir(dir_name);
struct dirent *info;
if (!dir) {
fprintf(stderr, "ERROR: Cannot read directory [%s]\n", dir_name);
return;
}
while ((info = readdir(dir)) != 0) {
len = strlen(info->d_name);
if (info->d_type == DT_DIR) {
if ((len == 1 && info->d_name[0] == '.')
|| (len == 2 && info->d_name[0] == '.' && info->d_name[1] == '.')) {
/* skip */
} else {
name = malloc(dir_name_len + len + 2);
memcpy(name, dir_name, dir_name_len);
name[dir_name_len] = PATH_SEP;
memcpy(name + dir_name_len + 1, info->d_name, len + 1);
find_files_in_dir(name, dir_name_len + len + 1);
free(name);
}
} else if (info->d_type == DT_REG) {
if (!test_extension
|| strcasecmp(info->d_name + len - test_extension_len, test_extension) == 0) {
name = malloc(dir_name_len + len + 2);
memcpy(name, dir_name, dir_name_len);
name[dir_name_len] = PATH_SEP;
memcpy(name + dir_name_len + 1, info->d_name, len + 1);
add_file(name);
}
}
}
closedir(dir);
#endif
}
static int cmp_files(const void *s1, const void *s2)
{
return strcmp(*(const char**)s1, *(const char**)s2);
}
static void find_files(char **tests, int tests_count)
{
int i;
struct stat stat_buf;
for (i = 0; i < tests_count; i++) {
if (stat(tests[i], &stat_buf) != 0) {
fprintf(stderr, "ERROR: Bad File or Folder [%s]\n", tests[i]);
}
if ((stat_buf.st_mode & S_IFMT) == S_IFDIR) {
find_files_in_dir(tests[i], strlen(tests[i]));
} else {
add_file(strdup(tests[i]));
}
}
qsort(files, files_count, sizeof(char*), cmp_files);
}
static void print_help(const char *exe_name)
{
printf(
"Run IR uint tests\n"
"Usage:\n"
" %s --test-cmd <cmd> {options} <test folders or files...>\n"
" Run the \"--CODE--\" section of specified test files using <cmd>\n"
"Options:\n"
" --target <target> - skip tests that specifies different --TARGET--\n"
" --default-args <args> - default <cmd> arguments (if --ARGS-- is missed)\n"
" --additional-args <args> - additional <cmd> arguments (always added at the end)\n"
" --diff-cmd <cmd> - diff command\n"
" --test-extension <ext> - search test files with the given extension\n"
" --code-extension <ext> - produce code files with the given extension\n"
" --show-diff - show diff of the failed tests\n"
" --no-color - disable color output\n"
, exe_name);
}
static int check_arg(const char *name, const char **value, int argc, char **argv, int *pos, int *bad_opt)
{
if (strcmp(argv[*pos], name) == 0) {
if (*value) {
*bad_opt = 1;
fprintf(stderr, "ERROR: Duplicate %s\n", name);
} else if (*pos + 1 == argc) {
*bad_opt = 1;
fprintf(stderr, "ERROR: Missing %s value\n", name);
} else {
*value = argv[++(*pos)];
}
return 1;
}
return 0;
}
int main(int argc, char **argv)
{
int bad_opt = 0; // unsupported option given
int bad_files = 0; // bad file or folder given
int show_diff = 0;
struct stat stat_buf;
#ifdef _WIN32
char **tests = _alloca(sizeof(char*) * argc);
#else
char **tests = alloca(sizeof(char*) * argc);
#endif
int i, tests_count = 0;
int skipped = 0;
int passed = 0;
int xfailed = 0, xfailed_limit = 0;
int failed = 0, failed_limit = 0;
int broken = 0, broken_limit = 0;
test *t, **xfailed_tests = NULL, **failed_tests = NULL;
char **broken_tests = NULL;
for (i = 1; i < argc; i++) {
if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) {
print_help(argv[0]);
return 0;
} else if (check_arg("--test-cmd", &test_cmd, argc, argv, &i, &bad_opt)) {
} else if (check_arg("--target", &target, argc, argv, &i, &bad_opt)) {
} else if (check_arg("--default-args", &default_args, argc, argv, &i, &bad_opt)) {
} else if (check_arg("--additional-args", &additional_args, argc, argv, &i, &bad_opt)) {
} else if (check_arg("--diff-cmd", &diff_cmd, argc, argv, &i, &bad_opt)) {
} else if (check_arg("--test-extension", &test_extension, argc, argv, &i, &bad_opt)) {
} else if (check_arg("--code-extension", &code_extension, argc, argv, &i, &bad_opt)) {
} else if (strcmp(argv[i], "--show-diff") == 0) {
show_diff = 1;
} else if (strcmp(argv[i], "--no-color") == 0) {
colorize = 0;
} else if (argv[i][0] == '-') {
bad_opt = 1;
fprintf(stderr, "ERROR: Unsupported Option [%s]\n", argv[i]);
} else {
// User specified test folders/files
if (stat(argv[i], &stat_buf) == 0
&& ((stat_buf.st_mode & S_IFMT) == S_IFDIR
|| (stat_buf.st_mode & S_IFMT) == S_IFREG)) {
tests[tests_count++] = argv[i];
} else {
bad_files = 1;
fprintf(stderr, "ERROR: Bad File or Folder [%s]\n", argv[i]);
}
}
}
if (bad_opt || bad_files || !tests_count || !test_cmd) {
if (bad_opt || !tests_count || !test_cmd) {
print_help(argv[0]);
}
return 1;
}
if (!default_args) {
default_args = "";
}
if (!additional_args) {
additional_args = "";
}
if (!code_extension) {
code_extension = ".code";
}
code_extension_len = (int)strlen(code_extension);
if (test_extension) {
test_extension_len = (int)strlen(test_extension);
if (strcmp(test_extension, code_extension) == 0) {
fprintf(stderr, "ERROR: --test-extension and --code-extension can't be the same\n");
return 1;
}
}
if (!diff_cmd) {
diff_cmd = DEFAULT_DIFF_CMD;
}
init_console();
find_files(tests, tests_count);
// Run each test
for (i = 0; i < files_count; i++) {
t = parse_file(files[i], i);
if (!t) {
printf("\r");
print_color("BROK", RED);
printf(": [%s]\n", files[i]);
if (broken >= broken_limit) {
broken_limit += 1024;
broken_tests = realloc(broken_tests, sizeof(char*) * broken_limit);
}
broken_tests[broken++] = files[i];
continue;
}
printf("TEST: %s [%s]", t->name, files[i]);
fflush(stdout);
if (skip_test(t)) {
printf("\r");
print_color("SKIP", YELLOW);
printf(": %s [%s]\n", t->name, files[i]);
skipped++;
free(t);
} else if (run_test(files[i], t, show_diff)) {
printf("\r");
print_color("PASS", GREEN);
printf(": %s [%s]\n", t->name, files[i]);
passed++;
free(t);
} else if (t->xfail) {
printf("\r");
print_color("XFAIL", RED);
printf(": %s [%s]\n", t->name, files[i]);
if (xfailed >= xfailed_limit) {
xfailed_limit += 1024;
xfailed_tests = realloc(xfailed_tests, sizeof(test*) * xfailed_limit);
}
xfailed_tests[xfailed++] = t;
} else {
printf("\r");
print_color("FAIL", RED);
printf(": %s [%s]\n", t->name, files[i]);
if (failed >= failed_limit) {
failed_limit += 1024;
failed_tests = realloc(failed_tests, sizeof(test*) * failed_limit);
}
failed_tests[failed++] = t;
}
}
// Produce the summary
printf("-------------------------------\n");
printf("Test Sumary\n");
printf("-------------------------------\n");
printf("Total: %d\n", files_count);
printf("Passed: %d\n", passed);
printf("Skipped: %d\n", skipped);
printf("Expected fail: %d\n", xfailed);
printf("Failed: %d\n", failed);
if (broken) {
printf("Broken: %d\n", broken);
}
if (xfailed > 0) {
printf("-------------------------------\n");
printf("EXPECTED FAILED TESTS\n");
printf("-------------------------------\n");
for (i = 0; i < xfailed; i++) {
t = xfailed_tests[i];
printf("%s [%s] XFAIL REASON: %s\n", t->name, files[t->id], t->xfail);
free(t);
}
free(xfailed_tests);
}
if (failed > 0) {
printf("-------------------------------\n");
printf("FAILED TESTS\n");
printf("-------------------------------\n");
for (i = 0; i < failed; i++) {
t = failed_tests[i];
printf("%s [%s]\n", t->name, files[t->id]);
free(t);
}
free(failed_tests);
}
if (broken > 0) {
printf("-------------------------------\n");
printf("BROKEN TESTS\n");
printf("-------------------------------\n");
for (i = 0; i < broken; i++) {
printf("%s\n", broken_tests[i]);
}
free(broken_tests);
}
printf("-------------------------------\n");
for (i = 0; i < files_count; i++) {
free(files[i]);
}
free(files);
return failed > 0 ? 1 : 0;
}

22
win32/Makefile
View File

@ -38,10 +38,6 @@ LDFLAGS=$(LDFLAGS) /DEBUG
CC=cl.exe
!endif
!if "$(CXX)" == ""
CXX=cl.exe
!endif
!if "$(LD)" == ""
LD=link.exe
!endif
@ -57,11 +53,13 @@ TARGET = $(VSCMD_ARG_TGT_ARCH)
CFLAGS=$(CFLAGS) /DIR_TARGET_X64
DASM_ARCH=x86
DASM_FLAGS=-D X64=1
TEST_TARGET=Windows-x86_64
!endif
!if "$(TARGET)" == "x86"
CFLAGS=$(CFLAGS) /DIR_TARGET_X86
DASM_ARCH=x86
DASM_FLAGS=
TEST_TARGET=x86
!endif
!if "$(VSCMD_ARG_TGT_ARCH)" == "x64"
@ -149,15 +147,19 @@ $(BUILD_DIR):
$(EXAMPLES_BUILD_DIR):
md "$(EXAMPLES_BUILD_DIR)"
$(BUILD_DIR)\ir-test.exe: $(SRC_DIR)/ir-test.cxx
"$(CXX)" /std:c++17 $(SRC_DIR)\ir-test.cxx /Fe:$@
$(BUILD_DIR)\tester.exe: $(SRC_DIR)/tools/tester.c
"$(CC)" $(CFLAGS) $(SRC_DIR)\tools\tester.c /Fe:$@
test: $(BUILD_DIR)\ir.exe $(BUILD_DIR)\ir-test.exe
test: $(BUILD_DIR)\ir.exe $(BUILD_DIR)\tester.exe
$(BUILD_DIR)\ir.exe $(SRC_DIR)\test.ir --dump --save $(BUILD_DIR)\test.log
$(BUILD_DIR)\ir-test.exe $(SRC_DIR)\tests
$(BUILD_DIR)\tester.exe --test-cmd $(BUILD_DIR)\ir.exe --target $(TEST_TARGET) --default-args "--save" \
--additional-args "--no-abort-fault" \
--test-extension ".irt" --code-extension ".ir" $(SRC_DIR)\tests
test-ci: $(BUILD_DIR)\ir.exe $(BUILD_DIR)\ir-test.exe
$(BUILD_DIR)\ir-test.exe --show-diff $(SRC_DIR)\tests
test-ci: $(BUILD_DIR)\ir.exe $(BUILD_DIR)\tester.exe
$(BUILD_DIR)\tester.exe --test-cmd $(BUILD_DIR)\ir.exe --target $(TEST_TARGET) --default-args "--save" \
--additional-args "--no-abort-fault" --diff-cmd "diff --strip-trailing-cr -u" \
--test-extension ".irt" --code-extension ".ir" --show-diff $(SRC_DIR)\tests
examples: $(OBJS_COMMON) $(EXAMPLES_BUILD_DIR) $(EXAMPLE_EXES)