Test runner now supports multiple tests per file.
This commit is contained in:
Ben Vanik
parent
9fcffcf866
commit
7a81a08486
|
|
@ -1,11 +1,13 @@
|
|||
BINUTILS=../../../third_party/binutils/bin/
|
||||
BINUTILS=../../../../../third_party/binutils/bin/
|
||||
PPC_AS=$(BINUTILS)/powerpc-none-elf-as
|
||||
PPC_LD=$(BINUTILS)/powerpc-none-elf-ld
|
||||
PPC_OBJDUMP=$(BINUTILS)/powerpc-none-elf-objdump
|
||||
PPC_NM=$(BINUTILS)/powerpc-none-elf-nm
|
||||
|
||||
SRCS=$(wildcard *.s)
|
||||
BINS=$(SRCS:.s=.bin)
|
||||
DISASMS=$(SRCS:.s=.dis)
|
||||
MAPS=$(SRCS:.s=.map)
|
||||
|
||||
%.o: %.s
|
||||
$(PPC_AS) \
|
||||
|
|
@ -34,5 +36,10 @@ DISASMS=$(SRCS:.s=.dis)
|
|||
-e 0x100000 \
|
||||
-o $@ \
|
||||
$<
|
||||
%.map: %.o
|
||||
$(PPC_NM) \
|
||||
--numeric-sort \
|
||||
$< \
|
||||
> $@
|
||||
|
||||
all: $(BINS) $(DISASMS)
|
||||
all: $(BINS) $(DISASMS) $(MAPS)
|
||||
|
|
|
|||
|
|
@ -82,30 +82,133 @@ class ThreadState : public alloy::runtime::ThreadState {
|
|||
PPCContext* context_;
|
||||
};
|
||||
|
||||
bool ReadAnnotations(std::wstring& src_file_path, AnnotationList& annotations) {
|
||||
// TODO(benvanik): use PAL instead of this
|
||||
FILE* f = fopen(poly::to_string(src_file_path).c_str(), "r");
|
||||
char line_buffer[BUFSIZ];
|
||||
while (fgets(line_buffer, sizeof(line_buffer), f)) {
|
||||
if (strlen(line_buffer) > 3 && line_buffer[0] == '#' &&
|
||||
line_buffer[1] == ' ') {
|
||||
// Comment - check if formed like an annotation.
|
||||
// We don't actually verify anything here.
|
||||
char* next_space = strchr(line_buffer + 3, ' ');
|
||||
if (next_space) {
|
||||
// Looks legit.
|
||||
std::string key(line_buffer + 2, next_space);
|
||||
std::string value(next_space + 1);
|
||||
while (value.find_last_of(" \t\n") == value.size() - 1) {
|
||||
value.erase(value.end() - 1);
|
||||
}
|
||||
annotations.emplace_back(key, value);
|
||||
struct TestCase {
|
||||
TestCase(uint64_t address, std::string& name)
|
||||
: address(address), name(name) {}
|
||||
uint64_t address;
|
||||
std::string name;
|
||||
AnnotationList annotations;
|
||||
};
|
||||
|
||||
class TestSuite {
|
||||
public:
|
||||
TestSuite(const std::wstring& src_file_path) : src_file_path(src_file_path) {
|
||||
name = src_file_path.substr(
|
||||
src_file_path.find_last_of(poly::path_separator) + 1);
|
||||
name.replace(name.end() - 2, name.end(), L"");
|
||||
map_file_path = src_file_path;
|
||||
map_file_path.replace(map_file_path.end() - 2, map_file_path.end(),
|
||||
L".map");
|
||||
bin_file_path = src_file_path;
|
||||
bin_file_path.replace(bin_file_path.end() - 2, bin_file_path.end(),
|
||||
L".bin");
|
||||
}
|
||||
|
||||
bool Load() {
|
||||
if (!ReadMap(map_file_path)) {
|
||||
PLOGE("Unable to read map for test %ls", src_file_path.c_str());
|
||||
return false;
|
||||
}
|
||||
if (!ReadAnnotations(src_file_path)) {
|
||||
PLOGE("Unable to read annotations for test %ls", src_file_path.c_str());
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
std::wstring name;
|
||||
std::wstring src_file_path;
|
||||
std::wstring map_file_path;
|
||||
std::wstring bin_file_path;
|
||||
std::vector<TestCase> test_cases;
|
||||
|
||||
private:
|
||||
TestCase* FindTestCase(const std::string& name) {
|
||||
for (auto& test_case : test_cases) {
|
||||
if (test_case.name == name) {
|
||||
return &test_case;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
fclose(f);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ReadMap(const std::wstring& map_file_path) {
|
||||
FILE* f = fopen(poly::to_string(map_file_path).c_str(), "r");
|
||||
if (!f) {
|
||||
return false;
|
||||
}
|
||||
char line_buffer[BUFSIZ];
|
||||
while (fgets(line_buffer, sizeof(line_buffer), f)) {
|
||||
if (!strlen(line_buffer)) {
|
||||
continue;
|
||||
}
|
||||
// 0000000000000000 t test_add1\n
|
||||
char* newline = strrchr(line_buffer, '\n');
|
||||
if (newline) {
|
||||
*newline = 0;
|
||||
}
|
||||
char* t_test_ = strstr(line_buffer, " t test_");
|
||||
if (!t_test_) {
|
||||
continue;
|
||||
}
|
||||
std::string address(line_buffer, t_test_ - line_buffer);
|
||||
std::string name(t_test_ + strlen(" t test_"));
|
||||
test_cases.emplace_back(START_ADDRESS + std::stoull(address, 0, 16),
|
||||
name);
|
||||
}
|
||||
fclose(f);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ReadAnnotations(const std::wstring& src_file_path) {
|
||||
TestCase* current_test_case = nullptr;
|
||||
FILE* f = fopen(poly::to_string(src_file_path).c_str(), "r");
|
||||
if (!f) {
|
||||
return false;
|
||||
}
|
||||
char line_buffer[BUFSIZ];
|
||||
while (fgets(line_buffer, sizeof(line_buffer), f)) {
|
||||
if (!strlen(line_buffer)) {
|
||||
continue;
|
||||
}
|
||||
// Eat leading whitespace.
|
||||
char* start = line_buffer;
|
||||
while (*start == ' ') {
|
||||
++start;
|
||||
}
|
||||
if (strncmp(start, "test_", strlen("test_")) == 0) {
|
||||
// Global test label.
|
||||
std::string label(start + strlen("test_"), strchr(start, ':'));
|
||||
current_test_case = FindTestCase(label);
|
||||
if (!current_test_case) {
|
||||
PLOGE("Test case %s not found in corresponding map for %ls",
|
||||
label.c_str(), src_file_path.c_str());
|
||||
return false;
|
||||
}
|
||||
} else if (strlen(start) > 3 && start[0] == '#' && start[1] == '_') {
|
||||
// Annotation.
|
||||
// We don't actually verify anything here.
|
||||
char* next_space = strchr(start + 3, ' ');
|
||||
if (next_space) {
|
||||
// Looks legit.
|
||||
std::string key(start + 3, next_space);
|
||||
std::string value(next_space + 1);
|
||||
while (value.find_last_of(" \t\n") == value.size() - 1) {
|
||||
value.erase(value.end() - 1);
|
||||
}
|
||||
if (!current_test_case) {
|
||||
PLOGE("Annotation outside of test case in %ls",
|
||||
src_file_path.c_str());
|
||||
return false;
|
||||
}
|
||||
current_test_case->annotations.emplace_back(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
fclose(f);
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
class TestRunner {
|
||||
public:
|
||||
|
|
@ -125,23 +228,11 @@ class TestRunner {
|
|||
memory.reset();
|
||||
}
|
||||
|
||||
bool Setup(std::wstring& src_file_path) {
|
||||
// test.s -> test.bin
|
||||
std::wstring bin_file_path;
|
||||
size_t dot = src_file_path.find_last_of(L".s");
|
||||
bin_file_path = src_file_path;
|
||||
bin_file_path.replace(dot - 1, 2, L".bin");
|
||||
|
||||
// Read annotations so we can setup state/etc.
|
||||
if (!ReadAnnotations(src_file_path, annotations)) {
|
||||
PLOGE("Unable to read annotations for test %ls", src_file_path.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
bool Setup(TestSuite& suite) {
|
||||
// Load the binary module.
|
||||
auto module = std::make_unique<alloy::runtime::RawModule>(runtime.get());
|
||||
if (module->LoadFile(START_ADDRESS, bin_file_path)) {
|
||||
PLOGE("Unable to load test binary %ls", bin_file_path.c_str());
|
||||
if (module->LoadFile(START_ADDRESS, suite.bin_file_path)) {
|
||||
PLOGE("Unable to load test binary %ls", suite.bin_file_path.c_str());
|
||||
return false;
|
||||
}
|
||||
runtime->AddModule(std::move(module));
|
||||
|
|
@ -156,16 +247,16 @@ class TestRunner {
|
|||
return true;
|
||||
}
|
||||
|
||||
bool Run() {
|
||||
bool Run(TestCase& test_case) {
|
||||
// Setup test state from annotations.
|
||||
if (!SetupTestState()) {
|
||||
if (!SetupTestState(test_case)) {
|
||||
PLOGE("Test setup failed");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Execute test.
|
||||
alloy::runtime::Function* fn;
|
||||
runtime->ResolveFunction(START_ADDRESS, &fn);
|
||||
runtime->ResolveFunction(test_case.address, &fn);
|
||||
if (!fn) {
|
||||
PLOGE("Entry function not found");
|
||||
return false;
|
||||
|
|
@ -176,38 +267,35 @@ class TestRunner {
|
|||
fn->Call(thread_state.get(), ctx->lr);
|
||||
|
||||
// Assert test state expectations.
|
||||
bool result = CheckTestResults();
|
||||
bool result = CheckTestResults(test_case);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
bool SetupTestState() {
|
||||
bool SetupTestState(TestCase& test_case) {
|
||||
auto ppc_state = thread_state->context();
|
||||
|
||||
for (AnnotationList::iterator it = annotations.begin();
|
||||
it != annotations.end(); ++it) {
|
||||
if (it->first == "REGISTER_IN") {
|
||||
size_t space_pos = it->second.find(" ");
|
||||
auto reg_name = it->second.substr(0, space_pos);
|
||||
auto reg_value = it->second.substr(space_pos + 1);
|
||||
for (auto& it : test_case.annotations) {
|
||||
if (it.first == "REGISTER_IN") {
|
||||
size_t space_pos = it.second.find(" ");
|
||||
auto reg_name = it.second.substr(0, space_pos);
|
||||
auto reg_value = it.second.substr(space_pos + 1);
|
||||
ppc_state->SetRegFromString(reg_name.c_str(), reg_value.c_str());
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool CheckTestResults() {
|
||||
bool CheckTestResults(TestCase& test_case) {
|
||||
auto ppc_state = thread_state->context();
|
||||
|
||||
char actual_value[2048];
|
||||
|
||||
bool any_failed = false;
|
||||
for (AnnotationList::iterator it = annotations.begin();
|
||||
it != annotations.end(); ++it) {
|
||||
if (it->first == "REGISTER_OUT") {
|
||||
size_t space_pos = it->second.find(" ");
|
||||
auto reg_name = it->second.substr(0, space_pos);
|
||||
auto reg_value = it->second.substr(space_pos + 1);
|
||||
for (auto& it : test_case.annotations) {
|
||||
if (it.first == "REGISTER_OUT") {
|
||||
size_t space_pos = it.second.find(" ");
|
||||
auto reg_name = it.second.substr(0, space_pos);
|
||||
auto reg_value = it.second.substr(space_pos + 1);
|
||||
if (!ppc_state->CompareRegWithString(reg_name.c_str(),
|
||||
reg_value.c_str(), actual_value,
|
||||
poly::countof(actual_value))) {
|
||||
|
|
@ -225,7 +313,6 @@ class TestRunner {
|
|||
std::unique_ptr<Memory> memory;
|
||||
std::unique_ptr<Runtime> runtime;
|
||||
std::unique_ptr<ThreadState> thread_state;
|
||||
AnnotationList annotations;
|
||||
};
|
||||
|
||||
bool DiscoverTests(std::wstring& test_path,
|
||||
|
|
@ -295,24 +382,44 @@ bool RunTests(const std::wstring& test_name) {
|
|||
PLOGI("%d tests discovered.", (int)test_files.size());
|
||||
PLOGI("");
|
||||
|
||||
std::vector<TestSuite> test_suites;
|
||||
bool load_failed = false;
|
||||
for (auto& test_path : test_files) {
|
||||
if (!test_name.empty() && test_path != test_name) {
|
||||
continue;
|
||||
}
|
||||
|
||||
PLOGI("Running %ls...", test_path.c_str());
|
||||
TestRunner runner;
|
||||
if (!runner.Setup(test_path)) {
|
||||
PLOGE("TEST FAILED SETUP");
|
||||
++failed_count;
|
||||
TestSuite test_suite(test_path);
|
||||
if (!test_suite.Load()) {
|
||||
PLOGE("TEST SUITE %ls FAILED TO LOAD", test_path.c_str());
|
||||
load_failed = true;
|
||||
continue;
|
||||
}
|
||||
if (runner.Run()) {
|
||||
PLOGI("Passed");
|
||||
++passed_count;
|
||||
} else {
|
||||
PLOGE("TEST FAILED");
|
||||
++failed_count;
|
||||
test_suites.push_back(std::move(test_suite));
|
||||
}
|
||||
if (load_failed) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (auto& test_suite : test_suites) {
|
||||
PLOGI("%ls.s:", test_suite.name.c_str());
|
||||
|
||||
for (auto& test_case : test_suite.test_cases) {
|
||||
PLOGI(" - %s", test_case.name.c_str());
|
||||
TestRunner runner;
|
||||
if (!runner.Setup(test_suite)) {
|
||||
PLOGE(" TEST FAILED SETUP");
|
||||
++failed_count;
|
||||
}
|
||||
if (runner.Run(test_case)) {
|
||||
++passed_count;
|
||||
} else {
|
||||
PLOGE(" TEST FAILED");
|
||||
++failed_count;
|
||||
}
|
||||
}
|
||||
|
||||
PLOGI("");
|
||||
}
|
||||
|
||||
PLOGI("");
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -4,6 +4,10 @@ instr_add.o: file format elf64-powerpc
|
|||
|
||||
Disassembly of section .text:
|
||||
|
||||
0000000000100000 <.text>:
|
||||
0000000000100000 <test_add1>:
|
||||
100000: 7d 65 ca 14 add r11,r5,r25
|
||||
100004: 4e 80 00 20 blr
|
||||
|
||||
0000000000100008 <test_add2>:
|
||||
100008: 7d 60 ca 14 add r11,r0,r25
|
||||
10000c: 4e 80 00 20 blr
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
0000000000000000 t test_add1
|
||||
0000000000000008 t test_add2
|
||||
Binary file not shown.
|
|
@ -1,9 +1,21 @@
|
|||
# REGISTER_IN r5 0x00100000
|
||||
# REGISTER_IN r25 0x0000FFFF
|
||||
test_add1:
|
||||
#_ REGISTER_IN r5 0x00100000
|
||||
#_ REGISTER_IN r25 0x0000FFFF
|
||||
|
||||
add r11, r5, r25
|
||||
add r11, r5, r25
|
||||
|
||||
blr
|
||||
# REGISTER_OUT r5 0x00100000
|
||||
# REGISTER_OUT r25 0x0000FFFF
|
||||
# REGISTER_OUT r11 0x0010FFFF
|
||||
blr
|
||||
#_ REGISTER_OUT r5 0x00100000
|
||||
#_ REGISTER_OUT r25 0x0000FFFF
|
||||
#_ REGISTER_OUT r11 0x0010FFFF
|
||||
|
||||
test_add2:
|
||||
#_ REGISTER_IN r0 0x00100000
|
||||
#_ REGISTER_IN r25 0x0000FFFF
|
||||
|
||||
add r11, r0, r25
|
||||
|
||||
blr
|
||||
#_ REGISTER_OUT r0 0x00100000
|
||||
#_ REGISTER_OUT r25 0x0000FFFF
|
||||
#_ REGISTER_OUT r11 0x0010FFFF
|
||||
|
|
|
|||
|
|
@ -4,6 +4,6 @@ instr_extrwi.o: file format elf64-powerpc
|
|||
|
||||
Disassembly of section .text:
|
||||
|
||||
0000000000100000 <.text>:
|
||||
0000000000100000 <test_extrwi>:
|
||||
100000: 54 a7 ef 3e rlwinm r7,r5,29,28,31
|
||||
100004: 4e 80 00 20 blr
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
0000000000000000 t test_extrwi
|
||||
|
|
@ -1,11 +1,12 @@
|
|||
# This is a variant of rlwinmx:
|
||||
# extrwi ra,rs,n,b (n > 0) == rlwinm ra,rs,b+n,32-n,31
|
||||
|
||||
# REGISTER_IN r5 0x30
|
||||
test_extrwi:
|
||||
#_ REGISTER_IN r5 0x30
|
||||
|
||||
# rlwinm r7, r5, 29, 28, 31
|
||||
extrwi r7, r5, 4, 25
|
||||
# rlwinm r7, r5, 29, 28, 31
|
||||
extrwi r7, r5, 4, 25
|
||||
|
||||
blr
|
||||
# REGISTER_OUT r5 0x30
|
||||
# REGISTER_OUT r7 0x06
|
||||
blr
|
||||
#_ REGISTER_OUT r5 0x30
|
||||
#_ REGISTER_OUT r7 0x06
|
||||
|
|
|
|||
|
|
@ -4,6 +4,6 @@ instr_ori.o: file format elf64-powerpc
|
|||
|
||||
Disassembly of section .text:
|
||||
|
||||
0000000000100000 <.text>:
|
||||
0000000000100000 <test_ori>:
|
||||
100000: 60 83 fe dc ori r3,r4,65244
|
||||
100004: 4e 80 00 20 blr
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
0000000000000000 t test_ori
|
||||
|
|
@ -1,7 +1,8 @@
|
|||
# REGISTER_IN r4 0xDEADBEEF00000000
|
||||
test_ori:
|
||||
#_ REGISTER_IN r4 0xDEADBEEF00000000
|
||||
|
||||
ori r3, r4, 0xFEDC
|
||||
ori r3, r4, 0xFEDC
|
||||
|
||||
blr
|
||||
# REGISTER_OUT r3 0xDEADBEEF0000FEDC
|
||||
# REGISTER_OUT r4 0xDEADBEEF00000000
|
||||
blr
|
||||
#_ REGISTER_OUT r3 0xDEADBEEF0000FEDC
|
||||
#_ REGISTER_OUT r4 0xDEADBEEF00000000
|
||||
|
|
|
|||
|
|
@ -4,6 +4,6 @@ instr_rlwimi.o: file format elf64-powerpc
|
|||
|
||||
Disassembly of section .text:
|
||||
|
||||
0000000000100000 <.text>:
|
||||
0000000000100000 <test_rlwimi>:
|
||||
100000: 50 86 10 3a rlwimi r6,r4,2,0,29
|
||||
100004: 4e 80 00 20 blr
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
0000000000000000 t test_rlwimi
|
||||
|
|
@ -1,8 +1,9 @@
|
|||
# REGISTER_IN r4 0xCAFEBABE90003000
|
||||
# REGISTER_IN r6 0xDEADBEEF00000003
|
||||
test_rlwimi:
|
||||
#_ REGISTER_IN r4 0xCAFEBABE90003000
|
||||
#_ REGISTER_IN r6 0xDEADBEEF00000003
|
||||
|
||||
rlwimi r6, r4, 2, 0, 0x1D
|
||||
rlwimi r6, r4, 2, 0, 0x1D
|
||||
|
||||
blr
|
||||
# REGISTER_OUT r4 0xCAFEBABE90003000
|
||||
# REGISTER_OUT r6 0xDEADBEEF4000C003
|
||||
blr
|
||||
#_ REGISTER_OUT r4 0xCAFEBABE90003000
|
||||
#_ REGISTER_OUT r6 0xDEADBEEF4000C003
|
||||
|
|
|
|||
|
|
@ -4,6 +4,6 @@ instr_subfe.o: file format elf64-powerpc
|
|||
|
||||
Disassembly of section .text:
|
||||
|
||||
0000000000100000 <.text>:
|
||||
0000000000100000 <test_subfe>:
|
||||
100000: 7c 6a 59 10 subfe r3,r10,r11
|
||||
100004: 4e 80 00 20 blr
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
0000000000000000 t test_subfe
|
||||
|
|
@ -1,9 +1,10 @@
|
|||
# REGISTER_IN r10 0x00000000000103BF
|
||||
# REGISTER_IN r11 0x00000000000103C0
|
||||
test_subfe:
|
||||
#_ REGISTER_IN r10 0x00000000000103BF
|
||||
#_ REGISTER_IN r11 0x00000000000103C0
|
||||
|
||||
subfe r3, r10, r11
|
||||
subfe r3, r10, r11
|
||||
|
||||
blr
|
||||
# REGISTER_OUT r10 0x00000000000103BF
|
||||
# REGISTER_OUT r11 0x00000000000103C0
|
||||
# REGISTER_OUT r3 0x1
|
||||
blr
|
||||
#_ REGISTER_OUT r10 0x00000000000103BF
|
||||
#_ REGISTER_OUT r11 0x00000000000103C0
|
||||
#_ REGISTER_OUT r3 0x1
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -1,10 +0,0 @@
|
|||
|
||||
instr_x.o: file format elf64-powerpc
|
||||
|
||||
|
||||
Disassembly of section .text:
|
||||
|
||||
0000000000100000 <.text>:
|
||||
100000: 10 01 12 46 vcmpgtuh v0,v1,v2
|
||||
100004: 10 03 20 83 lvewx128 v0,r3,r4
|
||||
100008: 4e 80 00 20 blr
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
vcmpgtuh v0, v1, v2
|
||||
lvewx128 v0, r3, r4
|
||||
|
||||
blr
|
||||
Loading…
Reference in New Issue