Improving tests to include save/load full cycles and direct comparison against quicknes

meson.build

@@ -34,7 +34,7 @@ executable('player',
 
 # Building tester tool for QuickerNES
 
-quickerNESTester = executable('quickerNESTester',
+quickerNESTester = executable('qrTester',
   'source/tester.cpp',
   cpp_args            : [ commonCompileArgs, '-Werror' ],
   dependencies        : [ quickerNESDependency, toolDependency ],
@@ -43,7 +43,7 @@ quickerNESTester = executable('quickerNESTester',
 
 # Building tester tool for the original QuickNES
 
-quickNESTester = executable('quickNESTester',
+quickNESTester = executable('qTester',
   'source/tester.cpp',
   cpp_args            : [ commonCompileArgs ],
   dependencies        : [ quickNESDependency, toolDependency ],

source/tester.cpp

@@ -16,12 +16,21 @@
 int main(int argc, char *argv[])
 {
    // Parsing command line arguments
-  argparse::ArgumentParser program("player", "1.0");
+  argparse::ArgumentParser program("tester", "1.0");
 
   program.add_argument("scriptFile")
     .help("Path to the test script file to run.")
     .required();
 
+  program.add_argument("--fullCycle")
+    .help("Performs the full load / advance frame / save cycle, as opposed to just advance frame")
+    .default_value(false)
+    .implicit_value(true);
+
+  program.add_argument("--hashOutputFile")
+    .help("Path to write the hash output to.")
+    .default_value(std::string(""));
+
   // Try to parse arguments
   try { program.parse_args(argc, argv);  }
   catch (const std::runtime_error &err) { EXIT_WITH_ERROR("%s\n%s", err.what(), program.help().str().c_str()); }
@@ -29,6 +38,12 @@ int main(int argc, char *argv[])
   // Getting test script file path
   std::string scriptFilePath = program.get<std::string>("scriptFile");
 
+  // Getting path where to save the hash output (if any)
+  std::string hashOutputFile = program.get<std::string>("--hashOutputFile");
+
+  // Getting reproduce flag
+  bool isFullCycle = program.get<bool>("--fullCycle");
+
   // Loading script file
   std::string scriptJsonRaw;
   if (loadStringFromFile(scriptJsonRaw, scriptFilePath) == false)  EXIT_WITH_ERROR("Could not find/read script file: %s\n", scriptFilePath.c_str());
@@ -46,11 +61,6 @@ int main(int argc, char *argv[])
   if (scriptJson["Initial State File"].is_string() == false) EXIT_WITH_ERROR("Script file 'Initial State File' entry is not a string\n");
   std::string initialStateFilePath = scriptJson["Initial State File"].get<std::string>();
 
-  // Getting verification state file path
-  if (scriptJson.contains("Verification State File") == false) EXIT_WITH_ERROR("Script file missing 'Verification State File' entry\n");
-  if (scriptJson["Verification State File"].is_string() == false) EXIT_WITH_ERROR("Script file 'Verification State File' entry is not a string\n");
-  std::string verificationStateFilePath = scriptJson["Verification State File"].get<std::string>();
-
   // Getting sequence file path
   if (scriptJson.contains("Sequence File") == false) EXIT_WITH_ERROR("Script file missing 'Sequence File' entry\n");
   if (scriptJson["Sequence File"].is_string() == false) EXIT_WITH_ERROR("Script file 'Sequence File' entry is not a string\n");
@@ -91,18 +101,6 @@ int main(int argc, char *argv[])
   // Checking with the expected SHA1 hash
   if (romSHA1 != expectedROMSHA1) EXIT_WITH_ERROR("Wrong ROM SHA1. Found: '%s', Expected: '%s'\n", romSHA1.c_str(), expectedROMSHA1.c_str());
 
-  // Loading state file for verication, if provided
-  std::string verificationState;
-  if (verificationStateFilePath != "")
-  {
-    // Loading file
-    if (loadStringFromFile(verificationState, verificationStateFilePath) == false)
-     EXIT_WITH_ERROR("[ERROR] Could not find or read from verification state file: %s\n", verificationStateFilePath.c_str());
-
-    // Checking size
-    if (verificationState.length() != stateSize) EXIT_WITH_ERROR("[ERROR] The provided verification state has different size from expected: %lu vs %lu\n", verificationState.length(), stateSize);
-  }
-
   // Loading sequence file
   std::string sequenceRaw;
   if (loadStringFromFile(sequenceRaw, sequenceFilePath) == false) EXIT_WITH_ERROR("[ERROR] Could not find or read from input sequence file: %s\n", sequenceFilePath.c_str());
@@ -119,10 +117,10 @@ int main(int argc, char *argv[])
   // Printing test information
   printf("[] -----------------------------------------\n");
   printf("[] Running Script:          '%s'\n", scriptFilePath.c_str());
+  printf("[] Cycle Type:              '%s'\n", isFullCycle ? "Load / Advance / Save" : "Advance Only");
   printf("[] Emulation Core:          '%s'\n", emulationCoreName.c_str());
   printf("[] ROM File:                '%s'\n", romFilePath.c_str());
   printf("[] ROM SHA1:                '%s'\n", romSHA1.c_str());
-  printf("[] Verification State File: '%s'\n", verificationStateFilePath.c_str());
   printf("[] Sequence File:           '%s'\n", sequenceFilePath.c_str());
   printf("[] Sequence Length:         %lu\n", sequenceLength);
   printf("[] Initial State Hash:      0x%lX%lX\n", initialHash.first, initialHash.second);
@@ -131,9 +129,18 @@ int main(int argc, char *argv[])
   
   fflush(stdout);
   
+  // Serializing initial state
+  uint8_t currentState[stateSize];
+  e.serializeState(currentState);
+
   // Actually running the sequence
   auto t0 = std::chrono::high_resolution_clock::now();
-  for (const std::string& input : sequence) e.advanceState(input);
+  for (const std::string& input : sequence)
+  {
+    if (isFullCycle == true) e.deserializeState(currentState);
+    e.advanceState(input);
+    if (isFullCycle == true) e.serializeState(currentState);
+  } 
   auto tf = std::chrono::high_resolution_clock::now();
 
   // Calculating running time
@@ -143,29 +150,19 @@ int main(int argc, char *argv[])
   // Calculating final state hash
   const auto finalStateHash = e.getStateHash();
 
-  // If provided, verify result against the passed verification movie
-  bool verificationPassed = true;
-  hash_t verificationStateHash;
-  if (verificationStateFilePath != "") 
-  {
-    // Loading verification state into the emulator
-    e.deserializeState((uint8_t*)verificationState.data());
-
-    // Calculating hash for the verification state
-    verificationStateHash = e.getStateHash();
-
-    // Comparing hashes
-    if (verificationStateHash != finalStateHash) verificationPassed = false;
-  }
+  // Creating hash string
+  char hashStringBuffer[256];
+  sprintf(hashStringBuffer, "0x%lX%lX", finalStateHash.first, finalStateHash.second);
 
   // Printing time information
   printf("[] Elapsed time:            %3.3fs\n", (double)dt * 1.0e-9);
   printf("[] Performance:             %.3f steps / s\n", (double)sequenceLength / elapsedTimeSeconds);
-  printf("[] Final State Hash:        0x%lX%lX\n", finalStateHash.first, finalStateHash.second);
-  if (verificationStateFilePath != "")
-  printf("[] Verification Hash:       0x%lX%lX (%s)\n", verificationStateHash.first, verificationStateHash.second, verificationPassed ? "Passed" : "Failed");
+  printf("[] Final State Hash:        %s\n", hashStringBuffer);
 
-  // Fail if verification didn't pass
-  return verificationPassed ? 0 : -1;
+  // If saving hash, do it now
+  if (hashOutputFile != "") saveStringToFile(std::string(hashStringBuffer), hashOutputFile.c_str());
+
+  // If reached this point, everything ran ok
+  return 0;
 }
 

tests/.gitignore

@@ -3,4 +3,6 @@
 *.zip
 *.fds
 test.sol
+*.hash
+*.out
 

tests/games/arkanoid/meson.build

@@ -1,7 +1,11 @@
 game = 'arkanoid'
 
+bash = find_program('bash')
+
 goal = 'warpless'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ])
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )
 
 goal = 'warps'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ])
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )

tests/games/castlevania1/meson.build

@@ -1,7 +1,9 @@
 game = 'castlevania1'
 
 goal = 'anyPercent'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ])
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )
 
 goal = 'pacifist'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ])
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )

tests/games/galaga/meson.build

@@ -1,4 +1,5 @@
 game = 'galaga'
 
 goal = 'anyPercent'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ], timeout: 60)
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )

tests/games/ironSword/meson.build

@@ -1,4 +1,5 @@
 game = 'ironSword'
 
 goal = 'anyPercent'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ])
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )

tests/games/meson.build

@@ -1,5 +1,8 @@
 nomalloc = environment({'MALLOC_PERTURB_': '0'})
 
+bash = find_program('bash')
+testCommands = ['../run_test.sh', quickerNESTester.path(), quickNESTester.path() ]
+
 subdir('arkanoid')
 subdir('castlevania1')
 subdir('superOffroad')

tests/games/nigelMansell/meson.build

@@ -1,4 +1,5 @@
 game = 'nigelMansell'
 
 goal = 'anyPercent'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ], timeout: 60)
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), timeout: 60, args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), timeout: 60, args : [ testCommands, goal + '.test'], suite: [ game, goal ] )

tests/games/ninjaGaiden/meson.build

@@ -1,7 +1,9 @@
 game = 'ninjaGaiden'
 
 goal = 'anyPercent'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ])
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )
 
 goal = 'pacifist'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ])
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )

tests/games/ninjaGaiden2/meson.build

@@ -1,7 +1,9 @@
 game = 'ninjaGaiden2'
 
 goal = 'anyPercent'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ])
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )
 
 goal = 'pacifist'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ])
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )

tests/games/princeOfPersia/meson.build

@@ -1,4 +1,5 @@
 game = 'princeOfPersia'
 
 goal = 'lvl7'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ])
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )

tests/games/run_test.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Stop if anything fails
+set -e
+
+# Getting executable paths
+quickerNESExecutable=${1}
+quickNESExecutable=${2}
+
+# Getting script name
+script=${3}
+
+# Getting additional arguments
+testerArgs=${@:4}
+
+# If running full cycle, add that to the hash file output
+mode="normal"
+if [ "${4}" = "--fullCycle" ]; then mode="fullCycle"; fi
+
+# Running script on quickerNES
+${quickerNESExecutable} ${script} --hashOutputFile quickerNES.${script}.${mode}.hash ${testerArgs}
+
+# Running script on quickNES
+${quickNESExecutable} ${script} --hashOutputFile quickNES.${script}.${mode}.hash ${testerArgs}
+
+# Comparing hashes
+quickerNESHash=`cat quickerNES.hash`
+ 
+# Comparing hashes
+quickNESHash=`cat quickNES.hash`
+
+# Compare hashes
+if [ "${quickerNESHash}" = "${quickNESHash}" ]; then
+ echo "[] Test Passed"
+ exit 0
+else
+ echo "[] Test Failed"
+ exit -1
+fi

tests/games/saintSeiyaKanketsuHen/meson.build

@@ -1,4 +1,5 @@
 game = 'saintSeiyaKanketsuHen'
 
 goal = 'anyPercent'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ], timeout: 60)
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )

tests/games/saintSeiyaOugonDensetsu/meson.build

@@ -1,4 +1,5 @@
 game = 'saintSeiyaOugonDensetsu'
 
 goal = 'anyPercent'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ], timeout: 60)
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )

tests/games/solarJetman/meson.build

@@ -1,4 +1,5 @@
 game = 'solarJetman'
 
 goal = 'anyPercent'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ])
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )

tests/games/superMarioBros/meson.build

@@ -1,7 +1,9 @@
 game = 'superMarioBros'
 
 goal = 'warps'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ])
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )
 
 goal = 'warpless'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ])
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )

tests/games/superOffroad/meson.build

@@ -1,4 +1,5 @@
 game = 'superOffroad'
 
 goal = 'anyPercent'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ])
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )

tests/games/tennis/meson.build

@@ -1,4 +1,5 @@
 game = 'tennis'
 
 goal = 'anyPercent'
-test(goal, quickerNESTester, workdir : meson.current_source_dir(), args : [ goal + '.test'], suite: [ game, goal ])
+test(goal + '-FullCycle', bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test', '--fullCycle'], suite: [ game, goal ] )
+test(goal, bash, workdir : meson.current_source_dir(), args : [ testCommands, goal + '.test'], suite: [ game, goal ] )