Test runner now supports multiple tests per file.

src/alloy/frontend/ppc/test/Makefile

@@ -1,11 +1,13 @@
-BINUTILS=../../../third_party/binutils/bin/
+BINUTILS=../../../../../third_party/binutils/bin/
 PPC_AS=$(BINUTILS)/powerpc-none-elf-as
 PPC_LD=$(BINUTILS)/powerpc-none-elf-ld
 PPC_OBJDUMP=$(BINUTILS)/powerpc-none-elf-objdump
+PPC_NM=$(BINUTILS)/powerpc-none-elf-nm
 
 SRCS=$(wildcard *.s)
 BINS=$(SRCS:.s=.bin)
 DISASMS=$(SRCS:.s=.dis)
+MAPS=$(SRCS:.s=.map)
 
 %.o: %.s
 	$(PPC_AS) \
@@ -34,5 +36,10 @@ DISASMS=$(SRCS:.s=.dis)
 		-e 0x100000 \
 		-o $@ \
 		$<
+%.map: %.o
+	$(PPC_NM) \
+		--numeric-sort \
+		$< \
+		> $@
 
-all: $(BINS) $(DISASMS)
+all: $(BINS) $(DISASMS) $(MAPS)

src/alloy/frontend/ppc/test/alloy-ppc-test.cc

@@ -82,30 +82,133 @@ class ThreadState : public alloy::runtime::ThreadState {
   PPCContext* context_;
 };
 
-bool ReadAnnotations(std::wstring& src_file_path, AnnotationList& annotations) {
+struct TestCase {
-  // TODO(benvanik): use PAL instead of this
+  TestCase(uint64_t address, std::string& name)
-  FILE* f = fopen(poly::to_string(src_file_path).c_str(), "r");
+      : address(address), name(name) {}
-  char line_buffer[BUFSIZ];
+  uint64_t address;
-  while (fgets(line_buffer, sizeof(line_buffer), f)) {
+  std::string name;
-    if (strlen(line_buffer) > 3 && line_buffer[0] == '#' &&
+  AnnotationList annotations;
-        line_buffer[1] == ' ') {
+};
-      // Comment - check if formed like an annotation.
+
-      // We don't actually verify anything here.
+class TestSuite {
-      char* next_space = strchr(line_buffer + 3, ' ');
+ public:
-      if (next_space) {
+  TestSuite(const std::wstring& src_file_path) : src_file_path(src_file_path) {
-        // Looks legit.
+    name = src_file_path.substr(
-        std::string key(line_buffer + 2, next_space);
+        src_file_path.find_last_of(poly::path_separator) + 1);
-        std::string value(next_space + 1);
+    name.replace(name.end() - 2, name.end(), L"");
-        while (value.find_last_of(" \t\n") == value.size() - 1) {
+    map_file_path = src_file_path;
-          value.erase(value.end() - 1);
+    map_file_path.replace(map_file_path.end() - 2, map_file_path.end(),
-        }
+                          L".map");
-        annotations.emplace_back(key, value);
+    bin_file_path = src_file_path;
+    bin_file_path.replace(bin_file_path.end() - 2, bin_file_path.end(),
+                          L".bin");
+  }
+
+  bool Load() {
+    if (!ReadMap(map_file_path)) {
+      PLOGE("Unable to read map for test %ls", src_file_path.c_str());
+      return false;
+    }
+    if (!ReadAnnotations(src_file_path)) {
+      PLOGE("Unable to read annotations for test %ls", src_file_path.c_str());
+      return false;
+    }
+    return true;
+  }
+
+  std::wstring name;
+  std::wstring src_file_path;
+  std::wstring map_file_path;
+  std::wstring bin_file_path;
+  std::vector<TestCase> test_cases;
+
+ private:
+  TestCase* FindTestCase(const std::string& name) {
+    for (auto& test_case : test_cases) {
+      if (test_case.name == name) {
+        return &test_case;
       }
     }
+    return nullptr;
   }
-  fclose(f);
+
-  return true;
+  bool ReadMap(const std::wstring& map_file_path) {
-}
+    FILE* f = fopen(poly::to_string(map_file_path).c_str(), "r");
+    if (!f) {
+      return false;
+    }
+    char line_buffer[BUFSIZ];
+    while (fgets(line_buffer, sizeof(line_buffer), f)) {
+      if (!strlen(line_buffer)) {
+        continue;
+      }
+      // 0000000000000000 t test_add1\n
+      char* newline = strrchr(line_buffer, '\n');
+      if (newline) {
+        *newline = 0;
+      }
+      char* t_test_ = strstr(line_buffer, " t test_");
+      if (!t_test_) {
+        continue;
+      }
+      std::string address(line_buffer, t_test_ - line_buffer);
+      std::string name(t_test_ + strlen(" t test_"));
+      test_cases.emplace_back(START_ADDRESS + std::stoull(address, 0, 16),
+                              name);
+    }
+    fclose(f);
+    return true;
+  }
+
+  bool ReadAnnotations(const std::wstring& src_file_path) {
+    TestCase* current_test_case = nullptr;
+    FILE* f = fopen(poly::to_string(src_file_path).c_str(), "r");
+    if (!f) {
+      return false;
+    }
+    char line_buffer[BUFSIZ];
+    while (fgets(line_buffer, sizeof(line_buffer), f)) {
+      if (!strlen(line_buffer)) {
+        continue;
+      }
+      // Eat leading whitespace.
+      char* start = line_buffer;
+      while (*start == ' ') {
+        ++start;
+      }
+      if (strncmp(start, "test_", strlen("test_")) == 0) {
+        // Global test label.
+        std::string label(start + strlen("test_"), strchr(start, ':'));
+        current_test_case = FindTestCase(label);
+        if (!current_test_case) {
+          PLOGE("Test case %s not found in corresponding map for %ls",
+                label.c_str(), src_file_path.c_str());
+          return false;
+        }
+      } else if (strlen(start) > 3 && start[0] == '#' && start[1] == '_') {
+        // Annotation.
+        // We don't actually verify anything here.
+        char* next_space = strchr(start + 3, ' ');
+        if (next_space) {
+          // Looks legit.
+          std::string key(start + 3, next_space);
+          std::string value(next_space + 1);
+          while (value.find_last_of(" \t\n") == value.size() - 1) {
+            value.erase(value.end() - 1);
+          }
+          if (!current_test_case) {
+            PLOGE("Annotation outside of test case in %ls",
+                  src_file_path.c_str());
+            return false;
+          }
+          current_test_case->annotations.emplace_back(key, value);
+        }
+      }
+    }
+    fclose(f);
+    return true;
+  }
+};
 
 class TestRunner {
  public:
@@ -125,23 +228,11 @@ class TestRunner {
     memory.reset();
   }
 
-  bool Setup(std::wstring& src_file_path) {
+  bool Setup(TestSuite& suite) {
-    // test.s -> test.bin
-    std::wstring bin_file_path;
-    size_t dot = src_file_path.find_last_of(L".s");
-    bin_file_path = src_file_path;
-    bin_file_path.replace(dot - 1, 2, L".bin");
-
-    // Read annotations so we can setup state/etc.
-    if (!ReadAnnotations(src_file_path, annotations)) {
-      PLOGE("Unable to read annotations for test %ls", src_file_path.c_str());
-      return false;
-    }
-
     // Load the binary module.
     auto module = std::make_unique<alloy::runtime::RawModule>(runtime.get());
-    if (module->LoadFile(START_ADDRESS, bin_file_path)) {
+    if (module->LoadFile(START_ADDRESS, suite.bin_file_path)) {
-      PLOGE("Unable to load test binary %ls", bin_file_path.c_str());
+      PLOGE("Unable to load test binary %ls", suite.bin_file_path.c_str());
       return false;
     }
     runtime->AddModule(std::move(module));
@@ -156,16 +247,16 @@ class TestRunner {
     return true;
   }
 
-  bool Run() {
+  bool Run(TestCase& test_case) {
     // Setup test state from annotations.
-    if (!SetupTestState()) {
+    if (!SetupTestState(test_case)) {
       PLOGE("Test setup failed");
       return false;
     }
 
     // Execute test.
     alloy::runtime::Function* fn;
-    runtime->ResolveFunction(START_ADDRESS, &fn);
+    runtime->ResolveFunction(test_case.address, &fn);
     if (!fn) {
       PLOGE("Entry function not found");
       return false;
@@ -176,38 +267,35 @@ class TestRunner {
     fn->Call(thread_state.get(), ctx->lr);
 
     // Assert test state expectations.
-    bool result = CheckTestResults();
+    bool result = CheckTestResults(test_case);
 
     return result;
   }
 
-  bool SetupTestState() {
+  bool SetupTestState(TestCase& test_case) {
     auto ppc_state = thread_state->context();
-
+    for (auto& it : test_case.annotations) {
-    for (AnnotationList::iterator it = annotations.begin();
+      if (it.first == "REGISTER_IN") {
-         it != annotations.end(); ++it) {
+        size_t space_pos = it.second.find(" ");
-      if (it->first == "REGISTER_IN") {
+        auto reg_name = it.second.substr(0, space_pos);
-        size_t space_pos = it->second.find(" ");
+        auto reg_value = it.second.substr(space_pos + 1);
-        auto reg_name = it->second.substr(0, space_pos);
-        auto reg_value = it->second.substr(space_pos + 1);
         ppc_state->SetRegFromString(reg_name.c_str(), reg_value.c_str());
       }
     }
     return true;
   }
 
-  bool CheckTestResults() {
+  bool CheckTestResults(TestCase& test_case) {
     auto ppc_state = thread_state->context();
 
     char actual_value[2048];
 
     bool any_failed = false;
-    for (AnnotationList::iterator it = annotations.begin();
+    for (auto& it : test_case.annotations) {
-         it != annotations.end(); ++it) {
+      if (it.first == "REGISTER_OUT") {
-      if (it->first == "REGISTER_OUT") {
+        size_t space_pos = it.second.find(" ");
-        size_t space_pos = it->second.find(" ");
+        auto reg_name = it.second.substr(0, space_pos);
-        auto reg_name = it->second.substr(0, space_pos);
+        auto reg_value = it.second.substr(space_pos + 1);
-        auto reg_value = it->second.substr(space_pos + 1);
         if (!ppc_state->CompareRegWithString(reg_name.c_str(),
                                              reg_value.c_str(), actual_value,
                                              poly::countof(actual_value))) {
@@ -225,7 +313,6 @@ class TestRunner {
   std::unique_ptr<Memory> memory;
   std::unique_ptr<Runtime> runtime;
   std::unique_ptr<ThreadState> thread_state;
-  AnnotationList annotations;
 };
 
 bool DiscoverTests(std::wstring& test_path,
@@ -295,24 +382,44 @@ bool RunTests(const std::wstring& test_name) {
   PLOGI("%d tests discovered.", (int)test_files.size());
   PLOGI("");
 
+  std::vector<TestSuite> test_suites;
+  bool load_failed = false;
   for (auto& test_path : test_files) {
     if (!test_name.empty() && test_path != test_name) {
       continue;
     }
 
-    PLOGI("Running %ls...", test_path.c_str());
+    TestSuite test_suite(test_path);
-    TestRunner runner;
+    if (!test_suite.Load()) {
-    if (!runner.Setup(test_path)) {
+      PLOGE("TEST SUITE %ls FAILED TO LOAD", test_path.c_str());
-      PLOGE("TEST FAILED SETUP");
+      load_failed = true;
-      ++failed_count;
+      continue;
     }
-    if (runner.Run()) {
+    test_suites.push_back(std::move(test_suite));
-      PLOGI("Passed");
+  }
-      ++passed_count;
+  if (load_failed) {
-    } else {
+    return false;
-      PLOGE("TEST FAILED");
+  }
-      ++failed_count;
+
+  for (auto& test_suite : test_suites) {
+    PLOGI("%ls.s:", test_suite.name.c_str());
+
+    for (auto& test_case : test_suite.test_cases) {
+      PLOGI("  - %s", test_case.name.c_str());
+      TestRunner runner;
+      if (!runner.Setup(test_suite)) {
+        PLOGE("    TEST FAILED SETUP");
+        ++failed_count;
+      }
+      if (runner.Run(test_case)) {
+        ++passed_count;
+      } else {
+        PLOGE("    TEST FAILED");
+        ++failed_count;
+      }
     }
+
+    PLOGI("");
   }
 
   PLOGI("");

src/alloy/frontend/ppc/test/instr_add.dis

@@ -4,6 +4,10 @@ instr_add.o:     file format elf64-powerpc
 
 Disassembly of section .text:
 
-0000000000100000 <.text>:
+0000000000100000 <test_add1>:
   100000:	7d 65 ca 14 	add     r11,r5,r25
   100004:	4e 80 00 20 	blr
+
+0000000000100008 <test_add2>:
+  100008:	7d 60 ca 14 	add     r11,r0,r25
+  10000c:	4e 80 00 20 	blr

src/alloy/frontend/ppc/test/instr_add.map

@@ -0,0 +1,2 @@
+0000000000000000 t test_add1
+0000000000000008 t test_add2

src/alloy/frontend/ppc/test/instr_add.s

@@ -1,9 +1,21 @@
-# REGISTER_IN r5 0x00100000
+test_add1:
-# REGISTER_IN r25 0x0000FFFF
+  #_ REGISTER_IN r5 0x00100000
+  #_ REGISTER_IN r25 0x0000FFFF
 
-add r11, r5, r25
+  add r11, r5, r25
 
-blr
+  blr
-# REGISTER_OUT r5 0x00100000
+  #_ REGISTER_OUT r5 0x00100000
-# REGISTER_OUT r25 0x0000FFFF
+  #_ REGISTER_OUT r25 0x0000FFFF
-# REGISTER_OUT r11 0x0010FFFF
+  #_ REGISTER_OUT r11 0x0010FFFF
+
+test_add2:
+  #_ REGISTER_IN r0 0x00100000
+  #_ REGISTER_IN r25 0x0000FFFF
+
+  add r11, r0, r25
+
+  blr
+  #_ REGISTER_OUT r0 0x00100000
+  #_ REGISTER_OUT r25 0x0000FFFF
+  #_ REGISTER_OUT r11 0x0010FFFF

src/alloy/frontend/ppc/test/instr_extrwi.dis

@@ -4,6 +4,6 @@ instr_extrwi.o:     file format elf64-powerpc
 
 Disassembly of section .text:
 
-0000000000100000 <.text>:
+0000000000100000 <test_extrwi>:
   100000:	54 a7 ef 3e 	rlwinm  r7,r5,29,28,31
   100004:	4e 80 00 20 	blr

src/alloy/frontend/ppc/test/instr_extrwi.map

@@ -0,0 +1 @@
+0000000000000000 t test_extrwi

src/alloy/frontend/ppc/test/instr_extrwi.s

@@ -1,11 +1,12 @@
 # This is a variant of rlwinmx:
 # extrwi ra,rs,n,b (n > 0) == rlwinm ra,rs,b+n,32-n,31
 
-# REGISTER_IN r5 0x30
+test_extrwi:
+  #_ REGISTER_IN r5 0x30
 
-# rlwinm r7, r5, 29, 28, 31
+  # rlwinm r7, r5, 29, 28, 31
-extrwi r7, r5, 4, 25
+  extrwi r7, r5, 4, 25
 
-blr
+  blr
-# REGISTER_OUT r5 0x30
+  #_ REGISTER_OUT r5 0x30
-# REGISTER_OUT r7 0x06
+  #_ REGISTER_OUT r7 0x06

src/alloy/frontend/ppc/test/instr_ori.dis

@@ -4,6 +4,6 @@ instr_ori.o:     file format elf64-powerpc
 
 Disassembly of section .text:
 
-0000000000100000 <.text>:
+0000000000100000 <test_ori>:
   100000:	60 83 fe dc 	ori     r3,r4,65244
   100004:	4e 80 00 20 	blr

src/alloy/frontend/ppc/test/instr_ori.map

@@ -0,0 +1 @@
+0000000000000000 t test_ori

src/alloy/frontend/ppc/test/instr_ori.s

@@ -1,7 +1,8 @@
-# REGISTER_IN r4 0xDEADBEEF00000000
+test_ori:
+  #_ REGISTER_IN r4 0xDEADBEEF00000000
 
-ori r3, r4, 0xFEDC
+  ori r3, r4, 0xFEDC
 
-blr
+  blr
-# REGISTER_OUT r3 0xDEADBEEF0000FEDC
+  #_ REGISTER_OUT r3 0xDEADBEEF0000FEDC
-# REGISTER_OUT r4 0xDEADBEEF00000000
+  #_ REGISTER_OUT r4 0xDEADBEEF00000000

src/alloy/frontend/ppc/test/instr_rlwimi.dis

@@ -4,6 +4,6 @@ instr_rlwimi.o:     file format elf64-powerpc
 
 Disassembly of section .text:
 
-0000000000100000 <.text>:
+0000000000100000 <test_rlwimi>:
   100000:	50 86 10 3a 	rlwimi  r6,r4,2,0,29
   100004:	4e 80 00 20 	blr

src/alloy/frontend/ppc/test/instr_rlwimi.map

@@ -0,0 +1 @@
+0000000000000000 t test_rlwimi

src/alloy/frontend/ppc/test/instr_rlwimi.s

@@ -1,8 +1,9 @@
-# REGISTER_IN r4 0xCAFEBABE90003000
+test_rlwimi:
-# REGISTER_IN r6 0xDEADBEEF00000003
+  #_ REGISTER_IN r4 0xCAFEBABE90003000
+  #_ REGISTER_IN r6 0xDEADBEEF00000003
 
-rlwimi r6, r4, 2, 0, 0x1D
+  rlwimi r6, r4, 2, 0, 0x1D
 
-blr
+  blr
-# REGISTER_OUT r4 0xCAFEBABE90003000
+  #_ REGISTER_OUT r4 0xCAFEBABE90003000
-# REGISTER_OUT r6 0xDEADBEEF4000C003
+  #_ REGISTER_OUT r6 0xDEADBEEF4000C003

src/alloy/frontend/ppc/test/instr_subfe.dis

@@ -4,6 +4,6 @@ instr_subfe.o:     file format elf64-powerpc
 
 Disassembly of section .text:
 
-0000000000100000 <.text>:
+0000000000100000 <test_subfe>:
   100000:	7c 6a 59 10 	subfe   r3,r10,r11
   100004:	4e 80 00 20 	blr

src/alloy/frontend/ppc/test/instr_subfe.map

@@ -0,0 +1 @@
+0000000000000000 t test_subfe

src/alloy/frontend/ppc/test/instr_subfe.s

@@ -1,9 +1,10 @@
-# REGISTER_IN r10 0x00000000000103BF
+test_subfe:
-# REGISTER_IN r11 0x00000000000103C0
+  #_ REGISTER_IN r10 0x00000000000103BF
+  #_ REGISTER_IN r11 0x00000000000103C0
 
-subfe r3, r10, r11
+  subfe r3, r10, r11
 
-blr
+  blr
-# REGISTER_OUT r10 0x00000000000103BF
+  #_ REGISTER_OUT r10 0x00000000000103BF
-# REGISTER_OUT r11 0x00000000000103C0
+  #_ REGISTER_OUT r11 0x00000000000103C0
-# REGISTER_OUT r3 0x1
+  #_ REGISTER_OUT r3 0x1

src/alloy/frontend/ppc/test/instr_x.dis

@@ -1,10 +0,0 @@
-
-instr_x.o:     file format elf64-powerpc
-
-
-Disassembly of section .text:
-
-0000000000100000 <.text>:
-  100000:	10 01 12 46 	vcmpgtuh v0,v1,v2
-  100004:	10 03 20 83 	lvewx128 v0,r3,r4
-  100008:	4e 80 00 20 	blr

src/alloy/frontend/ppc/test/instr_x.s

@@ -1,4 +0,0 @@
-vcmpgtuh v0, v1, v2
-lvewx128 v0, r3, r4
-
-blr