Update to v106r58 release.

byuu says:

The main thing I worked on today was emulating the MBC7 EEPROM.

And... I have many things to say about that, but not here, and not now...

The missing EEPROM support is why the accelerometer was broken. Although
it's not evidently clear that I'm emulating the actual values
incorrectly. I'll think about it and get it fixed, though.

bsnes went from ~308fps to ~328fps, and I don't even know why. Probably
something somewhere in the 140KB of changes to other things made in this
WIP.

higan/emulator/emulator.hpp

@@ -1,8 +1,23 @@
 #pragma once
 
-#include <nall/nall.hpp>
+#include <nall/platform.hpp>
 #include <nall/adaptive-array.hpp>
+#include <nall/any.hpp>
+#include <nall/bit-field.hpp>
+#include <nall/chrono.hpp>
+#include <nall/dl.hpp>
+#include <nall/endian.hpp>
+#include <nall/image.hpp>
+#include <nall/random.hpp>
+#include <nall/serializer.hpp>
+#include <nall/shared-pointer.hpp>
+#include <nall/string.hpp>
+#include <nall/traits.hpp>
+#include <nall/unique-pointer.hpp>
+#include <nall/vector.hpp>
 #include <nall/vfs.hpp>
+#include <nall/hash/crc32.hpp>
+#include <nall/hash/sha256.hpp>
 using namespace nall;
 
 #include "types.hpp"

higan/gb/cartridge/cartridge.cpp

@@ -17,6 +17,19 @@ Cartridge cartridge;
 #include "tama/tama.cpp"
 #include "serialization.cpp"
 
+auto Cartridge::Enter() -> void {
+  while(true) scheduler.synchronize(), cartridge.main();
+}
+
+auto Cartridge::main() -> void {
+  mapper->main();
+}
+
+auto Cartridge::step(uint clocks) -> void {
+  Thread::step(clocks);
+  synchronize(cpu);
+}
+
 auto Cartridge::load() -> bool {
   information = {};
   rom = {};
@@ -97,6 +110,7 @@ auto Cartridge::load() -> bool {
   }
 
   information.sha256 = Hash::SHA256(rom.data, rom.size).digest();
+  mapper->load(document);
   return true;
 }
 
@@ -118,6 +132,8 @@ auto Cartridge::save() -> void {
       }
     }
   }
+
+  mapper->save(document);
 }
 
 auto Cartridge::unload() -> void {
@@ -154,6 +170,8 @@ auto Cartridge::writeIO(uint16 addr, uint8 data) -> void {
 }
 
 auto Cartridge::power() -> void {
+  create(Enter, 4 * 1024 * 1024);
+
   for(uint n = 0x0000; n <= 0x7fff; n++) bus.mmio[n] = this;
   for(uint n = 0xa000; n <= 0xbfff; n++) bus.mmio[n] = this;
   bus.mmio[0xff50] = this;
@@ -179,4 +197,10 @@ auto Cartridge::Memory::write(uint address, uint8 byte) -> void {
   data[address] = byte;
 }
 
+//
+
+auto Cartridge::Mapper::main() -> void {
+  cartridge.step(cartridge.frequency());
+}
+
 }

higan/gb/cartridge/cartridge.hpp

@@ -1,9 +1,10 @@
-struct Cartridge : MMIO {
+struct Cartridge : Thread, MMIO {
   auto pathID() const -> uint { return information.pathID; }
   auto hash() const -> string { return information.sha256; }
   auto manifest() const -> string { return information.manifest; }
   auto title() const -> string { return information.title; }
 
+  static auto Enter() -> void;
   auto load() -> bool;
   auto save() -> void;
   auto unload() -> void;
@@ -11,6 +12,8 @@ struct Cartridge : MMIO {
   auto readIO(uint16 address) -> uint8;
   auto writeIO(uint16 address, uint8 data) -> void;
 
+  auto main() -> void;
+  auto step(uint clocks) -> void;
   auto power() -> void;
   auto second() -> void;
 
@@ -35,6 +38,9 @@ struct Cartridge : MMIO {
 
 private:
   struct Mapper {
+    virtual auto load(Markup::Node document) -> void {}
+    virtual auto save(Markup::Node document) -> void {}
+    virtual auto main() -> void;
     virtual auto second() -> void {}
     virtual auto read(uint16 address) -> uint8 = 0;
     virtual auto write(uint16 address, uint8 data) -> void = 0;

higan/gb/cartridge/mbc7/eeprom.cpp

@@ -0,0 +1,226 @@
+//Microchip 93LCx6
+//  93LC46 => 1024 cells => 128 x 8-bit or  64 x 16-bit
+//  93LC56 => 2048 cells => 256 x 8-bit or 128 x 16-bit
+//  93LC66 => 4096 cells => 512 x 8-bit or 256 x 16-bit
+
+auto Cartridge::MBC7::EEPROM::load(Markup::Node document) -> void {
+  for(auto& byte : data) byte = 0xff;
+  size = 4096;  //EEPROM size is in bits
+  width = 16;   //16-bit configuration
+
+  if(auto memory = Emulator::Game::Memory{document["game/board/memory(type=EEPROM,content=Save)"]}) {
+    if(memory.size == 128) size = 1024;  //manifest size is in bytes
+    if(memory.size == 256) size = 2048;
+    if(memory.size == 512) size = 4096;
+
+    if(auto fp = platform->open(cartridge.pathID(), memory.name(), File::Read, File::Optional)) {
+      fp->read(data, min(fp->size(), sizeof(data)));
+    }
+  }
+
+  command.length = 3;
+  if(size == 1024) address.length = width == 8 ? 6 : 7;
+  if(size == 2048) address.length = width == 8 ? 7 : 8;
+  if(size == 4096) address.length = width == 8 ? 8 : 9;
+  input.length = width;
+  output.length = 1 + width;  //there is an extra zero dummy bit on reads
+}
+
+auto Cartridge::MBC7::EEPROM::save(Markup::Node document) -> void {
+  if(auto memory = Emulator::Game::Memory{document["game/board/memory(type=EEPROM,content=Save)"]}) {
+    if(auto fp = platform->open(cartridge.pathID(), memory.name(), File::Write)) {
+      fp->write(data, size >> 3);  //bytes -> bits
+    }
+  }
+}
+
+auto Cartridge::MBC7::EEPROM::main() -> void {
+  //step by approximately one millisecond
+  cartridge.step(cartridge.frequency() / 1000);
+
+  //set during programming commands
+  if(busy) busy--;
+}
+
+auto Cartridge::MBC7::EEPROM::power(bool reset) -> void {
+  if(!reset) {
+    select = 0;
+    writable = 0;
+  }
+
+  clock = 0;
+  busy = 0;
+
+  command.flush();
+  address.flush();
+  input.flush();
+  output.flush();
+}
+
+auto Cartridge::MBC7::EEPROM::readIO() -> uint8 {
+  uint8 data = 0b00'1111'00;
+  data.bit(7) = select;
+  data.bit(6) = clock;
+  data.bit(1) = 1;
+  if(!select) {
+    data.bit(0) = 1;  //high-z when the chip is idle (not selected)
+  } else if(busy) {
+    data.bit(0) = 0;  //low when a programming command is in progress
+  } else if(output.count) {
+    data.bit(0) = output.peek();  //shift register data during read commands
+  } else {
+    data.bit(0) = 1;  //high-z during all other commands
+  }
+  return data;
+}
+
+auto Cartridge::MBC7::EEPROM::writeIO(uint8 data) -> void {
+  //bring chip out of idle state on rising CS edge
+  if(select.raise(data.bit(7))) return power(true);
+
+  //do nothing if chip is idle
+  if(!select) return;
+
+  //shift register clocks on rising edge
+  if(!clock.raise(data.bit(6))) return;
+
+  //sequential read mode
+  if(output.count && !data.bit(1)) {
+    output.read();
+    if(output.count == 0) {
+      address.value++;
+      read();
+    }
+    return;
+  }
+  output.flush();
+
+  //wait for start bit to be set
+  if(command.count == 0 && !data.bit(1)) return;
+
+  //waiting on command?
+  if(command.count < command.length) {
+    command.write(data.bit(1));
+    if(command.count < command.length) return;
+
+    return address.flush();
+  }
+
+  //waiting on address bits?
+  if(address.count < address.length) {
+    address.write(data.bit(1));
+    if(address.count < address.length) return;
+
+    uint3 opcode = command.bits(0, command.length - 1);
+    if(opcode == 0b100) {
+      uint2 mode = address.bits(address.length - 2, address.length - 1);
+      if(mode == 0b00) return writeDisable();
+      if(mode == 0b01) return input.flush();  //writeAll
+      if(mode == 0b10) return eraseAll();
+      if(mode == 0b11) return writeEnable();
+    }
+    if(opcode == 0b101) return input.flush();  //write
+    if(opcode == 0b110) return read();
+    if(opcode == 0b111) return erase();
+    return;
+  }
+
+  //waiting on data bits from a write or writeAll command?
+  if(input.count < input.length) {  //block new commands and inputs until the next clock edge
+    input.write(data.bit(1));
+    if(input.count < input.length) return;
+
+    uint3 opcode = command.bits(0, command.length - 1);
+    if(opcode == 0b101) return write();
+    if(opcode == 0b100) return writeAll();
+    return;
+  }
+}
+
+//
+
+auto Cartridge::MBC7::EEPROM::read() -> void {
+  command.flush();
+  auto address = this->address.value << (width == 16) & (size >> 3) - 1;
+  output.value = 0;
+  if(width >=  8) output.value |= data[address++] << 8;
+  if(width >= 16) output.value |= data[address++] << 0;
+  output.count = output.length;
+}
+
+auto Cartridge::MBC7::EEPROM::write() -> void {
+  command.flush();
+  if(!writable) return;
+  auto address = this->address.value << (width == 16) & (size >> 3) - 1;
+  if(width >=  8) data[address++] = input.value >> 8;
+  if(width >= 16) data[address++] = input.value >> 0;
+  input.flush();
+  busy = 4;  //ms
+}
+
+auto Cartridge::MBC7::EEPROM::erase() -> void {
+  command.flush();
+  if(!writable) return;
+  auto address = this->address.value << (width == 16) & (size >> 3) - 1;
+  if(width >=  8) data[address++] = 0xff;
+  if(width >= 16) data[address++] = 0xff;
+  busy = 4;  //ms
+}
+
+auto Cartridge::MBC7::EEPROM::writeAll() -> void {
+  command.flush();
+  if(!writable) return;
+  uint8 lo = input.byte(0);
+  uint8 hi = input.byte(width == 16);
+  for(uint address = 0; address < 512;) {
+    data[address++] = hi;
+    data[address++] = lo;
+  }
+  input.flush();
+  busy = 16;  //ms
+}
+
+auto Cartridge::MBC7::EEPROM::eraseAll() -> void {
+  command.flush();
+  if(!writable) return;
+  for(uint address; address < 512;) {
+    data[address++] = 0xff;
+    data[address++] = 0xff;
+  }
+  busy = 8;  //ms
+}
+
+auto Cartridge::MBC7::EEPROM::writeEnable() -> void {
+  command.flush();
+  writable = true;
+}
+
+auto Cartridge::MBC7::EEPROM::writeDisable() -> void {
+  command.flush();
+  writable = false;
+}
+
+//
+
+auto Cartridge::MBC7::EEPROM::ShiftRegister::flush() -> void {
+  value = 0;
+  count = 0;
+}
+
+//read the current bit in the shift register without clocking it
+auto Cartridge::MBC7::EEPROM::ShiftRegister::peek() -> bool {
+  return value.bit(length - 1);
+}
+
+auto Cartridge::MBC7::EEPROM::ShiftRegister::read() -> bool {
+  bool bit = value.bit(length - 1);
+  value <<= 1;
+  if(count) count--;
+  return bit;
+}
+
+auto Cartridge::MBC7::EEPROM::ShiftRegister::write(bool bit) -> void {
+  value <<= 1;
+  value |= bit;
+  count++;
+}

higan/gb/cartridge/mbc7/mbc7.cpp

@@ -1,3 +1,18 @@
+#include "eeprom.cpp"
+#include "serialization.cpp"
+
+auto Cartridge::MBC7::load(Markup::Node document) -> void {
+  eeprom.load(document);
+}
+
+auto Cartridge::MBC7::save(Markup::Node document) -> void {
+  eeprom.save(document);
+}
+
+auto Cartridge::MBC7::main() -> void {
+  eeprom.main();
+}
+
 auto Cartridge::MBC7::read(uint16 address) -> uint8 {
   if((address & 0xc000) == 0x0000) {  //$0000-3fff
     return cartridge.rom.read(address.bits(0,13));
@@ -17,7 +32,7 @@ auto Cartridge::MBC7::read(uint16 address) -> uint8 {
     case 5: return io.accelerometer.y.bits(8,15);
     case 6: return 0x00;  //z?
     case 7: return 0xff;  //z?
-    case 8: return 0xff;
+    case 8: return eeprom.readIO();
     }
 
     return 0xff;
@@ -48,25 +63,24 @@ auto Cartridge::MBC7::write(uint16 address, uint8 data) -> void {
     if(!io.ram.enable[0] || !io.ram.enable[1]) return;
 
     switch(address.bits(4,7)) {
-
     case 0: {
       if(data != 0x55) break;
-      io.accelerometer.x = 0x8000;
+      io.accelerometer.x = Center;
-      io.accelerometer.y = 0x8000;
+      io.accelerometer.y = Center;
       break;
     }
 
     case 1: {
       if(data != 0xaa) break;
-      io.accelerometer.x = 0x8000 + platform->inputPoll(ID::Port::Hardware, ID::Device::Controls, 8);
+      io.accelerometer.x = Center + platform->inputPoll(ID::Port::Hardware, ID::Device::Controls, 8);
-      io.accelerometer.y = 0x8000 + platform->inputPoll(ID::Port::Hardware, ID::Device::Controls, 9);
+      io.accelerometer.y = Center + platform->inputPoll(ID::Port::Hardware, ID::Device::Controls, 9);
       break;
     }
 
     case 8: {
+      eeprom.writeIO(data);
       break;
     }
-
     }
 
     return;
@@ -74,13 +88,6 @@ auto Cartridge::MBC7::write(uint16 address, uint8 data) -> void {
 }
 
 auto Cartridge::MBC7::power() -> void {
+  eeprom.power();
   io = {};
 }
-
-auto Cartridge::MBC7::serialize(serializer& s) -> void {
-  s.integer(io.rom.bank);
-  s.integer(io.ram.enable[0]);
-  s.integer(io.ram.enable[1]);
-  s.integer(io.accelerometer.x);
-  s.integer(io.accelerometer.y);
-}

higan/gb/cartridge/mbc7/mbc7.hpp

@@ -1,8 +1,67 @@
 struct MBC7 : Mapper {
-  auto read(uint16 address) -> uint8;
+  enum : uint { Center = 0x81d0 };
-  auto write(uint16 address, uint8 data) -> void;
+
-  auto power() -> void;
+  //mbc7.cpp
-  auto serialize(serializer&) -> void;
+  auto load(Markup::Node document) -> void override;
+  auto save(Markup::Node document) -> void override;
+  auto main() -> void override;
+  auto read(uint16 address) -> uint8 override;
+  auto write(uint16 address, uint8 data) -> void override;
+  auto power() -> void override;
+
+  //serialization.cpp
+  auto serialize(serializer&) -> void override;
+
+  struct EEPROM {
+    //eeprom.cpp
+    auto load(Markup::Node document) -> void;
+    auto save(Markup::Node document) -> void;
+    auto main() -> void;
+    auto power(bool reset = false) -> void;
+
+    //Game Boy MBC7 interface
+    auto readIO() -> uint8;
+    auto writeIO(uint8 data) -> void;
+
+    //chip commands
+    auto read() -> void;
+    auto write() -> void;
+    auto erase() -> void;
+    auto writeAll() -> void;
+    auto eraseAll() -> void;
+    auto writeEnable() -> void;
+    auto writeDisable() -> void;
+
+    //serialization.cpp
+    auto serialize(serializer&) -> void;
+
+    //it is awkward no matter if data is uint1[4096], uint8[512], or uint16[256]
+    uint8 data[512];  //uint8 was chosen solely for easier serialization and saving
+    uint size;        //in bits; not bytes
+    uint width;       //8-bit (ORG=0) or 16-bit (ORG=1) configuration
+
+    boolean select;    //CS
+    boolean clock;     //CLK
+    boolean writable;  //EWEN, EWDS
+    uint busy;         //busy cycles in milliseconds remaining for programming (write) operations to complete
+
+    struct ShiftRegister {
+      auto bit(uint index) { return value.bit(index); }
+      auto bits(uint lo, uint hi) { return value.bits(lo, hi); }
+      auto byte(uint index) { return value.byte(index); }
+      auto flush() -> void;
+      auto peek() -> bool;
+      auto read() -> bool;
+      auto write(bool data) -> void;
+
+      //serialization.cpp
+      auto serialize(serializer&) -> void;
+
+      uint32 value;
+      uint32 count;
+      uint32 length;
+    } command, address, input, output;
+  } eeprom;
 
   struct IO {
     struct ROM {
@@ -12,8 +71,8 @@ struct MBC7 : Mapper {
       uint1 enable[2];
     } ram;
     struct Accelerometer {
-      uint16 x = 0x8000;
+      uint16 x = Center;
-      uint16 y = 0x8000;
+      uint16 y = Center;
     } accelerometer;
   } io;
 } mbc7;

higan/gb/cartridge/mbc7/serialization.cpp

@@ -0,0 +1,28 @@
+auto Cartridge::MBC7::serialize(serializer& s) -> void {
+  eeprom.serialize(s);
+  s.integer(io.rom.bank);
+  s.integer(io.ram.enable[0]);
+  s.integer(io.ram.enable[1]);
+  s.integer(io.accelerometer.x);
+  s.integer(io.accelerometer.y);
+}
+
+auto Cartridge::MBC7::EEPROM::serialize(serializer& s) -> void {
+  s.array(data);
+  s.integer(size);
+  s.integer(width);
+  s.boolean(select);
+  s.boolean(clock);
+  s.boolean(writable);
+  s.integer(busy);
+  command.serialize(s);
+  address.serialize(s);
+  input.serialize(s);
+  output.serialize(s);
+}
+
+auto Cartridge::MBC7::EEPROM::ShiftRegister::serialize(serializer& s) -> void {
+  s.integer(value);
+  s.integer(count);
+  s.integer(length);
+}

higan/gb/cartridge/serialization.cpp

@@ -1,4 +1,5 @@
 auto Cartridge::serialize(serializer& s) -> void {
+  Thread::serialize(s);
   if(ram.size) s.array(ram.data, ram.size);
   if(rtc.size) s.array(rtc.data, rtc.size);
 

higan/gb/cpu/timing.cpp

@@ -19,6 +19,7 @@ auto CPU::step(uint clocks) -> void {
     Thread::step(1);
     synchronize(ppu);
     synchronize(apu);
+    synchronize(cartridge);
   }
 
   if(Model::SuperGameBoy()) {

higan/gb/system/system.cpp

@@ -15,6 +15,7 @@ auto System::runToSave() -> void {
   scheduler.synchronize(cpu);
   scheduler.synchronize(ppu);
   scheduler.synchronize(apu);
+  scheduler.synchronize(cartridge);
 }
 
 auto System::init() -> void {

higan/target-bsnes/bsnes.cpp

@@ -24,18 +24,31 @@ auto hiro::initialize() -> void {
 
 #include <nall/main.hpp>
 auto nall::main(vector<string> arguments) -> void {
-  Application::setScreenSaver(settings.general.screenSaver);
+  settings.location = locate("settings.bml");
-  Application::setToolTips(settings.general.toolTips);
 
-  string locale;  // = "日本語";
+  arguments.takeLeft();  //ignore program location in argument parsing
   for(auto argument : arguments) {
-    if(argument.beginsWith("--locale=")) {
+    if(argument == "--fullscreen") {
-      locale = argument.trimLeft("--locale=", 1L);
+      presentation.startFullScreen = true;
+    } else if(argument.beginsWith("--locale=")) {
+      Application::locale().scan(locate("locales/"));
+      Application::locale().select(argument.trimLeft("--locale=", 1L));
+    } else if(argument.beginsWith("--settings=")) {
+      settings.location = argument.trimLeft("--settings=", 1L);
+    } else if(inode::exists(argument)) {
+      //game without option
+      program.gameQueue.append({";", argument});
+    } else if(argument.find(";")) {
+      //game with option
+      auto game = argument.split(";", 1L);
+      if(inode::exists(game.last())) program.gameQueue.append(argument);
     }
   }
-  Application::locale().scan(locate("locales/"));
+
-  Application::locale().select(locale);
+  settings.load();
+  Application::setScreenSaver(settings.general.screenSaver);
+  Application::setToolTips(settings.general.toolTips);
   emulator = new SuperFamicom::Interface;
-  program.create(arguments);
+  program.create();
   Application::run();
 }

higan/target-bsnes/bsnes.hpp

@@ -1,18 +1,19 @@
-#include <nall/nall.hpp>
 #include <ruby/ruby.hpp>
-#include <hiro/hiro.hpp>
-using namespace nall;
 using namespace ruby;
-using namespace hiro;
 extern Video video;
 extern Audio audio;
 extern Input input;
 
+#include <hiro/hiro.hpp>
+using namespace hiro;
+
 #include <emulator/emulator.hpp>
 extern unique_pointer<Emulator::Interface> emulator;
 
-#include <nall/encode/rle.hpp>
 #include <nall/decode/rle.hpp>
+#include <nall/decode/zip.hpp>
+#include <nall/encode/rle.hpp>
+#include <nall/encode/zip.hpp>
 
 #include "program/program.hpp"
 #include "input/input.hpp"

higan/target-bsnes/presentation/presentation.cpp

@@ -189,6 +189,7 @@ auto Presentation::create() -> void {
   setBackgroundColor({0, 0, 0});
   resizeWindow();
   setCentered();
+  setFullScreen(startFullScreen);
 
   #if defined(PLATFORM_MACOS)
   Application::Cocoa::onAbout([&] { about.doActivate(); });
@@ -211,8 +212,17 @@ auto Presentation::updateStatusIcon() -> void {
   statusIcon.setIcon(icon);
 }
 
+auto Presentation::configureViewport() -> void {
+  uint width = viewport.geometry().width();
+  uint height = viewport.geometry().height();
+  video.configure(width, height, 60, 60);
+}
+
 auto Presentation::clearViewport() -> void {
-  if(!emulator->loaded()) viewportLayout.setPadding();
+  if(!emulator->loaded()) {
+    viewportLayout.setPadding();
+    configureViewport();
+  }
   if(!visible() || !video) return;
 
   uint32_t* output;
@@ -277,7 +287,7 @@ auto Presentation::resizeViewport() -> void {
     paddingWidth - paddingWidth / 2, paddingHeight - paddingHeight / 2
   });
 
-//clearViewport();
+  configureViewport();
 }
 
 auto Presentation::resizeWindow() -> void {
@@ -449,7 +459,7 @@ auto Presentation::updateRecentGames() -> void {
     bool missing = false;
     if(games) {
       for(auto& game : games.split("|")) {
-        if(!inode::exists(game)) missing = true;
+        if(!inode::exists(game.split(";").last())) missing = true;
       }
     }
     if(missing) {
@@ -467,12 +477,12 @@ auto Presentation::updateRecentGames() -> void {
 
   //update list
   for(auto index : range(RecentGames)) {
-    MenuItem item;
+    MenuItem item{&loadRecentGame};
     if(auto game = settings[{"Game/Recent/", 1 + index}].text()) {
       string displayName;
       auto games = game.split("|");
-      for(auto& part : games) {
+      for(auto& game : games) {
-        displayName.append(Location::prefix(part), " + ");
+        displayName.append(Location::prefix(game.split(";", 1L).last()), " + ");
       }
       displayName.trimRight(" + ", 1L);
       item.setIcon(games(0).endsWith("/") ? (image)Icon::Action::Open : (image)Icon::Emblem::File);
@@ -485,7 +495,6 @@ auto Presentation::updateRecentGames() -> void {
       item.setText({"(", tr("empty"), ")"});
       item.setEnabled(false);
     }
-    loadRecentGame.append(item);
   }
 
   loadRecentGame.append(MenuSeparator());
@@ -514,6 +523,8 @@ auto Presentation::addRecentGame(string location) -> void {
 
 auto Presentation::updateShaders() -> void {
   shaderMenu.reset();
+  shaderMenu.setEnabled(video.hasShader());
+  if(!video.hasShader()) return;
 
   Group shaders;
 
@@ -533,7 +544,7 @@ auto Presentation::updateShaders() -> void {
 
   auto location = locate("shaders/");
 
-  if(settings.video.driver == "OpenGL") {
+  if(settings.video.driver == "OpenGL 3.2") {
     for(auto shader : directory::folders(location, "*.shader")) {
       if(shaders.objectCount() == 2) shaderMenu.append(MenuSeparator());
       MenuRadioItem item{&shaderMenu};

higan/target-bsnes/presentation/presentation.hpp

@@ -26,6 +26,7 @@ struct Presentation : Window {
   enum : uint { StatusHeight = 24 };
 
   auto updateStatusIcon() -> void;
+  auto configureViewport() -> void;
   auto clearViewport() -> void;
   auto resizeViewport() -> void;
   auto resizeWindow() -> void;
@@ -40,6 +41,8 @@ struct Presentation : Window {
   auto addRecentGame(string location) -> void;
   auto updateShaders() -> void;
 
+  bool startFullScreen = false;
+
   MenuBar menuBar{this};
     Menu systemMenu{&menuBar};
       MenuItem loadGame{&systemMenu};

higan/target-bsnes/program/game.cpp

@@ -50,12 +50,13 @@ auto Program::load() -> void {
   stateManager.loadStates();
   manifestViewer.loadManifest();
 
-  string locations = superFamicom.location;
+  string games;
-  if(auto& location = gameBoy.location) locations.append("|", location);
+  if(auto& game = superFamicom) games.append(game.option, ";", game.location, "|");
-  if(auto& location = bsMemory.location) locations.append("|", location);
+  if(auto& game = gameBoy     ) games.append(game.option, ";", game.location, "|");
-  if(auto& location = sufamiTurboA.location) locations.append("|", location);
+  if(auto& game = bsMemory    ) games.append(game.option, ";", game.location, "|");
-  if(auto& location = sufamiTurboB.location) locations.append("|", location);
+  if(auto& game = sufamiTurboA) games.append(game.option, ";", game.location, "|");
-  presentation.addRecentGame(locations);
+  if(auto& game = sufamiTurboB) games.append(game.option, ";", game.location, "|");
+  presentation.addRecentGame(games.trimRight("|", 1L));
 
   updateVideoPalette();
   updateAudioEffects();

higan/target-bsnes/program/patch.cpp

@@ -84,7 +84,7 @@ auto Program::applyPatchIPS(vector<uint8_t>& data, string location) -> bool {
   return true;
 }
 
-#include <nall/beat/patch.hpp>
+#include <nall/beat/single/apply.hpp>
 
 auto Program::applyPatchBPS(vector<uint8_t>& input, string location) -> bool {
   vector<uint8_t> patch;
@@ -107,22 +107,16 @@ auto Program::applyPatchBPS(vector<uint8_t>& input, string location) -> bool {
   }
   if(!patch) return false;
 
-  bpspatch beat;
+  string manifest;
-  beat.modify(patch.data(), patch.size());
+  string result;
-  beat.source(input.data(), input.size());
+  if(auto output = Beat::Single::apply(input.data(), input.size(), patch.data(), patch.size(), manifest, result)) {
-  vector<uint8_t> output;
+    input = move(*output);
-  output.resize(beat.size());
-  beat.target(output.data(), output.size());
-  auto result = beat.apply();
-
-  if(result == bpspatch::result::success) {
-    input = output;
     return true;
   }
 
-  MessageDialog(
+  MessageDialog({
-    "Error: patch checksum failure.\n\n"
+    result, "\n\n",
     "Please ensure you are using the correct (headerless) ROM for this patch."
-  ).setParent(*presentation).error();
+  }).setParent(*presentation).error();
   return false;
 }

higan/target-bsnes/program/platform.cpp

@@ -106,85 +106,100 @@ auto Program::load(uint id, string name, string type, vector<string> options) ->
 
   if(id == 1 && name == "Super Famicom" && type == "sfc") {
     if(gameQueue) {
-      superFamicom.location = gameQueue.takeLeft();
+      auto game = gameQueue.takeLeft().split(";", 1L);
+      superFamicom.option = game(0);
+      superFamicom.location = game(1);
     } else {
       dialog.setTitle("Load Super Famicom");
       dialog.setPath(path("Games", settings.path.recent.superFamicom));
       dialog.setFilters({string{"Super Famicom Games|*.sfc:*.smc:*.zip"}});
       superFamicom.location = dialog.openObject();
+      superFamicom.option = dialog.option();
     }
     if(inode::exists(superFamicom.location)) {
       settings.path.recent.superFamicom = Location::dir(superFamicom.location);
       if(loadSuperFamicom(superFamicom.location)) {
-        return {id, dialog.option()};
+        return {id, superFamicom.option};
       }
     }
   }
 
   if(id == 2 && name == "Game Boy" && type == "gb") {
     if(gameQueue) {
-      gameBoy.location = gameQueue.takeLeft();
+      auto game = gameQueue.takeLeft().split(";", 1L);
+      gameBoy.option = game(0);
+      gameBoy.location = game(1);
     } else {
       dialog.setTitle("Load Game Boy");
       dialog.setPath(path("Games", settings.path.recent.gameBoy));
       dialog.setFilters({string{"Game Boy Games|*.gb:*.gbc:*.zip"}});
       gameBoy.location = dialog.openObject();
+      gameBoy.option = dialog.option();
     }
     if(inode::exists(gameBoy.location)) {
       settings.path.recent.gameBoy = Location::dir(gameBoy.location);
       if(loadGameBoy(gameBoy.location)) {
-        return {id, dialog.option()};
+        return {id, gameBoy.option};
       }
     }
   }
 
   if(id == 3 && name == "BS Memory" && type == "bs") {
     if(gameQueue) {
-      bsMemory.location = gameQueue.takeLeft();
+      auto game = gameQueue.takeLeft().split(";", 1L);
+      bsMemory.option = game(0);
+      bsMemory.location = game(1);
     } else {
       dialog.setTitle("Load BS Memory");
       dialog.setPath(path("Games", settings.path.recent.bsMemory));
       dialog.setFilters({string{"BS Memory Games|*.bs:*.zip"}});
       bsMemory.location = dialog.openObject();
+      bsMemory.option = dialog.option();
     }
     if(inode::exists(bsMemory.location)) {
       settings.path.recent.bsMemory = Location::dir(bsMemory.location);
       if(loadBSMemory(bsMemory.location)) {
-        return {id, dialog.option()};
+        return {id, bsMemory.option};
       }
     }
   }
 
   if(id == 4 && name == "Sufami Turbo" && type == "st") {
     if(gameQueue) {
-      sufamiTurboA.location = gameQueue.takeLeft();
+      auto game = gameQueue.takeLeft().split(";", 1L);
+      sufamiTurboA.option = game(0);
+      sufamiTurboA.location = game(1);
     } else {
       dialog.setTitle("Load Sufami Turbo - Slot A");
       dialog.setPath(path("Games", settings.path.recent.sufamiTurboA));
       dialog.setFilters({string{"Sufami Turbo Games|*.st:*.zip"}});
       sufamiTurboA.location = dialog.openObject();
+      sufamiTurboA.option = dialog.option();
     }
     if(inode::exists(sufamiTurboA.location)) {
       settings.path.recent.sufamiTurboA = Location::dir(sufamiTurboA.location);
       if(loadSufamiTurboA(sufamiTurboA.location)) {
-        return {id, dialog.option()};
+        return {id, sufamiTurboA.option};
       }
     }
   }
 
   if(id == 5 && name == "Sufami Turbo" && type == "st") {
     if(gameQueue) {
-      sufamiTurboB.location = gameQueue.takeLeft();
+      auto game = gameQueue.takeLeft().split(";", 1L);
+      sufamiTurboB.option = game(0);
+      sufamiTurboB.location = game(1);
     } else {
       dialog.setTitle("Load Sufami Turbo - Slot B");
       dialog.setPath(path("Games", settings.path.recent.sufamiTurboB));
       dialog.setFilters({string{"Sufami Turbo Games|*.st:*.zip"}});
       sufamiTurboB.location = dialog.openObject();
+      sufamiTurboB.option = dialog.option();
     }
     if(inode::exists(sufamiTurboB.location)) {
       settings.path.recent.sufamiTurboB = Location::dir(sufamiTurboB.location);
       if(loadSufamiTurboB(sufamiTurboB.location)) {
-        return {id, dialog.option()};
+        return {id, sufamiTurboB.option};
       }
     }
   }

higan/target-bsnes/program/program.cpp

@@ -13,7 +13,7 @@
 #include "hacks.cpp"
 Program program;
 
-auto Program::create(vector<string> arguments) -> void {
+auto Program::create() -> void {
   Emulator::platform = this;
 
   presentation.create();
@@ -59,15 +59,6 @@ auto Program::create(vector<string> arguments) -> void {
   driverSettings.audioDriverChanged();
   driverSettings.inputDriverChanged();
 
-  arguments.takeLeft();  //ignore program location in argument parsing
-  for(auto& argument : arguments) {
-    if(argument == "--fullscreen") {
-      presentation.toggleFullscreenMode();
-    } else if(inode::exists(argument)) {
-      gameQueue.append(argument);
-    }
-  }
-
   if(gameQueue) load();
   Application::onMain({&Program::main, this});
 }

higan/target-bsnes/program/program.hpp

@@ -2,7 +2,7 @@ struct Program : Lock, Emulator::Platform {
   Application::Namespace tr{"Program"};
 
   //program.cpp
-  auto create(vector<string> arguments) -> void;
+  auto create() -> void;
   auto main() -> void;
   auto quit() -> void;
 
@@ -108,6 +108,7 @@ public:
   struct Game {
     explicit operator bool() const { return (bool)location; }
 
+    string option;
     string location;
     string manifest;
     Markup::Node document;

higan/target-bsnes/program/states.cpp

@@ -62,7 +62,7 @@ auto Program::loadState(string filename) -> bool {
   if(auto memory = loadStateData(filename)) {
     if(filename != "Quick/Undo") saveUndoState();
     if(filename == "Quick/Undo") saveRedoState();
-    auto serializerRLE = Decode::RLE<uint8_t>(memory.data() + 3 * sizeof(uint));
+    auto serializerRLE = Decode::RLE<1>(memory.data() + 3 * sizeof(uint));
     serializer s{serializerRLE.data(), serializerRLE.size()};
     if(!emulator->unserialize(s)) return showMessage({"[", prefix, "] is in incompatible format"}), false;
     return showMessage({"Loaded [", prefix, "]"}), true;
@@ -77,7 +77,7 @@ auto Program::saveState(string filename) -> bool {
 
   serializer s = emulator->serialize();
   if(!s.size()) return showMessage({"Failed to save [", prefix, "]"}), false;
-  auto serializerRLE = Encode::RLE<uint8_t>(s.data(), s.size());
+  auto serializerRLE = Encode::RLE<1>(s.data(), s.size());
 
   vector<uint8_t> previewRLE;
   //this can be null if a state is captured before the first frame of video output after power/reset
@@ -86,7 +86,7 @@ auto Program::saveState(string filename) -> bool {
     preview.copy(screenshot.data, screenshot.pitch, screenshot.width, screenshot.height);
     if(preview.width() != 256 || preview.height() != 240) preview.scale(256, 240, true);
     preview.transform(0, 15, 0x8000, 0x7c00, 0x03e0, 0x001f);
-    previewRLE = Encode::RLE<uint16_t>(preview.data(), preview.size() / sizeof(uint16_t));
+    previewRLE = Encode::RLE<2>(preview.data(), preview.size() / sizeof(uint16_t));
   }
 
   vector<uint8_t> saveState;

higan/target-bsnes/program/video.cpp

@@ -12,6 +12,7 @@ auto Program::updateVideoDriver(Window parent) -> void {
   updateVideoShader();
 
   if(video.ready()) {
+    presentation.configureViewport();
     presentation.clearViewport();
     updateVideoShader();
   }
@@ -52,16 +53,7 @@ auto Program::updateVideoFormat() -> void {
 }
 
 auto Program::updateVideoShader() -> void {
-  if(settings.video.driver == "OpenGL"
+  video.setShader(settings.video.shader);
-  && settings.video.shader != "None"
-  && settings.video.shader != "Blur"
-  ) {
-    video.setSmooth(false);
-    video.setShader(settings.video.shader);
-  } else {
-    video.setSmooth(settings.video.shader == "Blur");
-    video.setShader("");
-  }
 }
 
 auto Program::updateVideoPalette() -> void {

higan/target-bsnes/settings/settings.cpp

@@ -17,14 +17,14 @@ DriverSettings driverSettings;
 SettingsWindow settingsWindow;
 
 auto Settings::load() -> void {
-  Markup::Node::operator=(BML::unserialize(string::read(locate("settings.bml")), " "));
+  Markup::Node::operator=(BML::unserialize(string::read(location), " "));
   process(true);
   file::write(locate("settings.bml"), BML::serialize(*this, " "));
 }
 
 auto Settings::save() -> void {
   process(false);
-  file::write(locate("settings.bml"), BML::serialize(*this, " "));
+  file::write(location ? location : locate("settings.bml"), BML::serialize(*this, " "));
 }
 
 auto Settings::process(bool load) -> void {

higan/target-bsnes/settings/settings.hpp

@@ -1,11 +1,10 @@
 struct Settings : Markup::Node {
-  Settings() { load(); }
-  ~Settings() { save(); }
-
   auto load() -> void;
   auto save() -> void;
   auto process(bool load) -> void;
 
+  string location;
+
   struct Video {
     string driver;
     bool exclusive = false;

higan/target-bsnes/tools/state-manager.cpp

@@ -158,7 +158,7 @@ auto StateManager::updateSelection() -> void {
         uint preview    = memory::readl<sizeof(uint)>(saveState.data() + 2 * sizeof(uint));
         if(signature == Program::State::Signature && preview) {
           uint offset = 3 * sizeof(uint) + serializer;
-          auto preview = Decode::RLE<uint16_t>(saveState.data() + offset, max(offset, saveState.size()) - offset);
+          auto preview = Decode::RLE<2>(saveState.data() + offset, max(offset, saveState.size()) - offset);
           image icon{0, 15, 0x8000, 0x7c00, 0x03e0, 0x001f};
           icon.copy(preview.data(), 256 * sizeof(uint16_t), 256, 240);
           icon.transform();

higan/target-higan/higan.hpp

@@ -1,13 +1,12 @@
-#include <nall/nall.hpp>
 #include <ruby/ruby.hpp>
-#include <hiro/hiro.hpp>
-using namespace nall;
 using namespace ruby;
-using namespace hiro;
 extern unique_pointer<Video> video;
 extern unique_pointer<Audio> audio;
 extern unique_pointer<Input> input;
 
+#include <hiro/hiro.hpp>
+using namespace hiro;
+
 #include <emulator/emulator.hpp>
 extern Emulator::Interface* emulator;
 

higan/target-higan/presentation/presentation.cpp

@@ -317,8 +317,17 @@ auto Presentation::updateSizeMenu() -> void {
   }));
 }
 
+auto Presentation::configureViewport() -> void {
+  uint width = viewport.geometry().width();
+  uint height = viewport.geometry().height();
+  if(video) video->configure(width, height, 60, 60);
+}
+
 auto Presentation::clearViewport() -> void {
-  if(!emulator || !emulator->loaded()) viewportLayout.setPadding();
+  if(!emulator || !emulator->loaded()) {
+    viewportLayout.setPadding();
+    configureViewport();
+  }
   if(!visible() || !video) return;
 
   uint32_t* output;
@@ -399,7 +408,7 @@ auto Presentation::resizeViewport() -> void {
     paddingWidth - paddingWidth / 2, paddingHeight - paddingHeight / 2
   });
 
-  clearViewport();
+  configureViewport();
 }
 
 auto Presentation::resizeWindow() -> void {
@@ -499,7 +508,7 @@ auto Presentation::loadSystems() -> void {
 auto Presentation::loadShaders() -> void {
   auto pathname = locate("shaders/");
 
-  if(settings["Video/Driver"].text() == "OpenGL") {
+  if(settings["Video/Driver"].text() == "OpenGL 3.2") {
     for(auto shader : directory::folders(pathname, "*.shader")) {
       if(videoShaders.objectCount() == 2) videoShaderMenu.append(MenuSeparator());
       MenuRadioItem item{&videoShaderMenu};

higan/target-higan/presentation/presentation.hpp

@@ -15,6 +15,7 @@ struct Presentation : Window {
   auto updateEmulatorMenu() -> void;
   auto updateEmulatorDeviceSelections() -> void;
   auto updateSizeMenu() -> void;
+  auto configureViewport() -> void;
   auto clearViewport() -> void;
   auto resizeViewport() -> void;
   auto resizeWindow() -> void;

higan/target-higan/program/utility.cpp

@@ -14,6 +14,7 @@ auto Program::initializeVideoDriver() -> void {
     video->create("None");
   }
 
+  presentation->configureViewport();
   presentation->clearViewport();
 }
 
@@ -137,17 +138,7 @@ auto Program::updateVideoPalette() -> void {
 }
 
 auto Program::updateVideoShader() -> void {
-  if(settings["Video/Driver"].text() == "OpenGL"
+  video->setShader(settings["Video/Shader"].text());
-  && settings["Video/Shader"].text() != "None"
-  && settings["Video/Shader"].text() != "Blur"
-  && directory::exists(settings["Video/Shader"].text())
-  ) {
-    video->setSmooth(false);
-    video->setShader(settings["Video/Shader"].text());
-  } else {
-    video->setSmooth(settings["Video/Shader"].text() == "Blur");
-    video->setShader("");
-  }
 }
 
 auto Program::updateAudioDriver() -> void {

hiro/gtk/header.hpp

@@ -1,10 +1,4 @@
 #if defined(DISPLAY_WINDOWS)
-  #define UNICODE
-  #define WINVER 0x0601
-  #define _WIN32_WINNT WINVER
-  #define _WIN32_IE WINVER
-  #define __MSVCRT_VERSION__ WINVER
-  #define NOMINMAX
   #define TBS_TRANSPARENTBKGND 0x1000
 
   #include <nall/windows/guard.hpp>

hiro/windows/header.hpp

@@ -1,15 +1,9 @@
-#define UNICODE
-#define WINVER 0x0601
-#define _WIN32_WINNT WINVER
-#define _WIN32_IE WINVER
-#define __MSVCRT_VERSION__ WINVER
-#define NOMINMAX
-
 #include <nall/windows/guard.hpp>
 #include <winsock2.h>
 #include <windows.h>
 #include <windowsx.h>
 #include <commctrl.h>
+#include <commdlg.h>
 #include <uxtheme.h>
 #include <io.h>
 #include <shlobj.h>

icarus/GNUmakefile

@@ -59,3 +59,5 @@ else ifneq ($(filter $(platform),linux bsd),)
 	rm -f $(prefix)/share/applications/$(name).desktop
 	rm -f $(prefix)/share/icons/$(name).png
 endif
+
+-include obj/*.d

icarus/heuristics/game-boy.cpp

@@ -33,15 +33,17 @@ auto GameBoy::manifest() const -> string {
   bool black = (read(0x0143) & 0xc0) == 0x80;  //cartridge works in DMG+CGB mode
   bool clear = (read(0x0143) & 0xc0) == 0xc0;  //cartridge works in CGB mode only
 
-  bool flash = false;
-  bool battery = false;
   bool ram = false;
+  bool battery = false;
+  bool eeprom = false;
+  bool flash = false;
   bool rtc = false;
   bool accelerometer = false;
   bool rumble = false;
 
   uint romSize = 0;
   uint ramSize = 0;
+  uint eepromSize = 0;
   uint flashSize = 0;
   uint rtcSize = 0;
 
@@ -176,7 +178,7 @@ auto GameBoy::manifest() const -> string {
   case 0x22:
     mapper = "MBC7";
     battery = true;
-    ram = true;
+    eeprom = true;
     accelerometer = true;
     rumble = true;
     break;
@@ -204,6 +206,27 @@ auto GameBoy::manifest() const -> string {
 
   }
 
+  //Game Boy: title = $0134-0143
+  //Game Boy Color (early games): title = $0134-0142; model = $0143
+  //Game Boy Color (later games): title = $0134-013e; serial = $013f-0142; model = $0143
+  string title;
+  for(uint n : range(black || clear ? 15 : 16)) {
+    char byte = read(0x0134 + n);
+    if(byte < 0x20 || byte > 0x7e) byte = ' ';
+    title.append(byte);
+  }
+
+  string serial = title.slice(-4);
+  if(!black && !clear) serial = "";
+  for(auto& byte : serial) {
+    if(byte >= 'A' && byte <= 'Z') continue;
+    //invalid serial
+    serial = "";
+    break;
+  }
+  title.trimRight(serial, 1L);  //remove the serial from the title, if it exists
+  title.strip();  //remove any excess whitespace from the title
+
   switch(read(0x0148)) { default:
   case 0x00: romSize =   2 * 16 * 1024; break;
   case 0x01: romSize =   4 * 16 * 1024; break;
@@ -227,11 +250,21 @@ auto GameBoy::manifest() const -> string {
 
   if(mapper == "MBC2" && ram) ramSize = 256;
   if(mapper == "MBC6" && ram) ramSize =  32 * 1024;
-  if(mapper == "MBC7" && ram) ramSize = 256;
   if(mapper == "TAMA" && ram) ramSize =  32;
 
   if(mapper == "MBC6" && flash) flashSize = 1024 * 1024;
 
+  //Game Boy header does not specify EEPROM size: detect via game title instead
+  //Command Master:        EEPROM = 512 bytes
+  //Kirby Tilt 'n' Tumble: EEPROM = 256 bytes
+  //Korokoro Kirby:        EEPROM = 256 bytes
+  if(mapper == "MBC7" && eeprom) {
+    eepromSize = 256;  //fallback guess; supported values are 128, 256, 512
+    if(title == "CMASTER"     && serial == "KCEJ") eepromSize = 512;
+    if(title == "KIRBY TNT"   && serial == "KTNE") eepromSize = 256;
+    if(title == "KORO2 KIRBY" && serial == "KKKJ") eepromSize = 256;
+  }
+
   if(mapper == "MBC3" && rtc) rtcSize = 13;
   if(mapper == "TAMA" && rtc) rtcSize = 21;
 
@@ -240,12 +273,17 @@ auto GameBoy::manifest() const -> string {
   output.append("  sha256: ", Hash::SHA256(data).digest(), "\n");
   output.append("  label:  ", Location::prefix(location), "\n");
   output.append("  name:   ", Location::prefix(location), "\n");
+  output.append("  title:  ", title, "\n");
+if(serial)
+  output.append("  serial: ", serial, "\n");
   output.append("  board:  ", mapper, "\n");
   output.append(Memory{}.type("ROM").size(data.size()).content("Program").text());
 if(ram && ramSize && battery)
   output.append(Memory{}.type("RAM").size(ramSize).content("Save").text());
 if(ram && ramSize && !battery)
   output.append(Memory{}.type("RAM").size(ramSize).content("Save").isVolatile().text());
+if(eeprom && eepromSize)
+  output.append(Memory{}.type("EEPROM").size(eepromSize).content("Save").text());
 if(flash && flashSize)
   output.append(Memory{}.type("Flash").size(flashSize).content("Download").text());
 if(rtc && rtcSize)

nall/adaptive-array.hpp

@@ -3,6 +3,7 @@
 #pragma once
 
 #include <nall/range.hpp>
+#include <nall/traits.hpp>
 
 namespace nall {
 

nall/array.hpp

@@ -1,5 +1,4 @@
 #pragma once
-#define DEBUG
 
 #include <nall/range.hpp>
 
@@ -7,7 +6,7 @@ namespace nall {
 
 template<typename T> struct array;
 
-//usage: int x[256] => array<int, 256>
+//usage: int x[256] => array<int[256]> x
 template<typename T, uint Size> struct array<T[Size]> {
   array() = default;
 
@@ -39,8 +38,9 @@ template<typename T, uint Size> struct array<T[Size]> {
     return values[index];
   }
 
-  auto fill(const T& fill = {}) {
+  auto fill(const T& fill = {}) -> array& {
     for(auto& value : values) value = fill;
+    return *this;
   }
 
   auto data() -> T* { return values; }
@@ -57,74 +57,4 @@ private:
   T values[Size];
 };
 
-template<typename T> struct array_view;
-
-template<typename T, uint Size> struct array_view<T[Size]> {
-  array_view() = default;
-
-  template<uint Capacity>
-  array_view(array<T[Capacity]>& source, uint offset = 0) {
-    #ifdef DEBUG
-    struct out_of_bounds {};
-    if(offset + Size >= Capacity) throw out_of_bounds{};
-    #endif
-    values = &source.data()[offset];
-  }
-
-  template<uint Capacity>
-  array_view(T (&source)[Capacity], uint offset = 0) {
-    #ifdef DEBUG
-    struct out_of_bounds {};
-    if(offset + Size >= Capacity) throw out_of_bounds{};
-    #endif
-    values = &source[offset];
-  }
-
-  array_view(T* source, uint offset = 0) {
-    values = &source[offset];
-  }
-
-  explicit operator bool() const {
-    return values;
-  }
-
-  alwaysinline auto operator[](uint index) -> T& {
-    #ifdef DEBUG
-    struct out_of_bounds {};
-    if(index >= Size) throw out_of_bounds{};
-    #endif
-    return values[index];
-  }
-
-  alwaysinline auto operator[](uint index) const -> const T& {
-    #ifdef DEBUG
-    struct out_of_bounds {};
-    if(index >= Size) throw out_of_bounds{};
-    #endif
-    return values[index];
-  }
-
-  alwaysinline auto operator()(uint index, const T& fallback = {}) const -> const T& {
-    if(index >= Size) return fallback;
-    return values[index];
-  }
-
-  auto fill(const T& fill = {}) -> void {
-    for(uint index : range(Size)) values[index] = fill;
-  }
-
-  auto data() -> T* { return values; }
-  auto data() const -> const T* { return values; }
-  auto size() const -> uint { return Size; }
-
-  auto begin() -> T* { return &values[0]; }
-  auto end() -> T* { return &values[Size]; }
-
-  auto begin() const -> const T* { return &values[0]; }
-  auto end() const -> const T* { return &values[Size]; }
-
-private:
-  T* values = nullptr;
-};
-
 }

nall/beat/archive.hpp

@@ -1,131 +0,0 @@
-#pragma once
-
-#include <nall/beat/file.hpp>
-
-namespace nall { namespace Beat {
-
-struct Archive {
-  static auto create(const string& beatname, const string& pathname, const string& metadata = "") -> bool;
-  static auto unpack(const string& beatname, const string& pathname) -> bool;
-  static auto extract(const string& beatname, const string& pathname) -> vector<uint8_t>;
-
-private:
-  static auto scan(vector<string>& result, const string& basename, const string& pathname) -> void;
-};
-
-auto Archive::create(const string& beatname, const string& pathname, const string& metadata) -> bool {
-  if(!beatname.endsWith(".bpa") || !pathname.endsWith("/")) return false;  //protect against reversed arguments
-
-  File beat{beatname, file::mode::write};
-  if(!beat) return false;  //file not writable?
-
-  beat.writes("BPA1");
-  beat.writevu(metadata.size());
-  beat.writes(metadata);
-
-  vector<string> contents;
-  scan(contents, pathname, pathname);
-
-  for(auto& name : contents) {
-    string location{pathname, name};
-    bool directory = name.endsWith("/");
-    bool writable = inode::writable(location);
-    bool executable = inode::executable(location);
-    uint info = directory << 0 | writable << 1 | executable << 2 | (name.trimRight("/").size() - 1) << 3;
-
-    beat.writevu(info);
-    beat.writes(name);
-    if(directory) continue;
-
-    File input{location, file::mode::read};
-    if(input) {
-      auto size = input.size();
-      beat.writevu(size);
-      while(size--) beat.write(input.read());
-      beat.writel(input.checksum.digest().hex(), 4);
-    } else {
-      beat.writevu(0);
-      beat.writel(0x00000000, 4);
-    }
-  }
-
-  beat.writel(beat.checksum.digest().hex(), 4);
-  return true;
-}
-
-auto Archive::unpack(const string& beatname, const string& pathname) -> bool {
-  if(!beatname.endsWith(".bpa") || !pathname.endsWith("/")) return false;  //protect against reversed arguments
-
-  File beat{beatname, file::mode::read};
-  if(!beat) return false;  //file not readable?
-
-  if(beat.reads(4) != "BPA1") return false;
-  auto size = beat.readvu();
-  while(size--) beat.read();
-
-  directory::create(pathname);
-  while(beat.offset() < beat.size() - 4) {
-    auto info = beat.readvu();
-    auto name = beat.reads((info >> 3) + 1);
-    if(name.find("\\") || name.find("../")) return false;  //block path exploits
-
-    string location{pathname, name};
-    bool directory = info & 1;
-    bool writable = info & 2;
-    bool executable = info & 4;
-
-    if(directory) {
-      if(!nall::directory::create(location)) return false;
-    } else {
-      File output{location, file::mode::write};
-      if(!output) return false;
-
-      auto size = beat.readvu();
-      while(size--) output.write(beat.read());
-      if(beat.readl(4) != output.checksum.digest().hex()) return false;
-    }
-  }
-
-  uint32_t checksum = beat.checksum.digest().hex();
-  return beat.readl(4) == checksum;
-}
-
-auto Archive::extract(const string& beatname, const string& filename) -> vector<uint8_t> {
-  File beat{beatname, file::mode::read};
-  if(!beat) return {};  //file not readable?
-
-  if(beat.reads(4) != "BPA1") return {};
-  auto size = beat.readvu();
-  beat.seek(beat.offset() + size);
-
-  while(beat.offset() < beat.size() - 4) {
-    auto info = beat.readvu();
-    auto name = beat.reads((info >> 3) + 1);
-    if(info & 1) continue;  //ignore directories
-
-    auto size = beat.readvu();
-    if(name != filename) {
-      beat.seek(beat.offset() + size + 4);
-      continue;
-    }
-
-    vector<uint8_t> result;
-    result.resize(size);
-    beat.checksum.reset();
-    for(auto n : range(size)) result[n] = beat.read();
-    uint32_t checksum = beat.checksum.digest().hex();
-    if(beat.readl(4) != checksum) return {};
-    return result;
-  }
-
-  return {};
-}
-
-auto Archive::scan(vector<string>& result, const string& basename, const string& pathname) -> void {
-  for(auto& name : directory::contents(pathname)) {
-    result.append(string{pathname, name}.trimLeft(basename, 1L));
-    if(name.endsWith("/")) scan(result, basename, {pathname, name});
-  }
-}
-
-}}

nall/beat/delta.hpp

@@ -1,211 +0,0 @@
-#pragma once
-
-#include <nall/file.hpp>
-#include <nall/filemap.hpp>
-#include <nall/stdint.hpp>
-#include <nall/string.hpp>
-#include <nall/hash/crc32.hpp>
-
-namespace nall {
-
-struct bpsdelta {
-  inline auto source(const uint8_t* data, uint size) -> void;
-  inline auto target(const uint8_t* data, uint size) -> void;
-
-  inline auto source(const string& filename) -> bool;
-  inline auto target(const string& filename) -> bool;
-  inline auto create(const string& filename, const string& metadata = "") -> bool;
-
-protected:
-  enum : uint { SourceRead, TargetRead, SourceCopy, TargetCopy };
-  enum : uint { Granularity = 1 };
-
-  struct Node {
-    Node() = default;
-    ~Node() { if(next) delete next; }
-    uint offset = 0;
-    Node* next = nullptr;
-  };
-
-  filemap sourceFile;
-  const uint8_t* sourceData;
-  uint sourceSize;
-
-  filemap targetFile;
-  const uint8_t* targetData;
-  uint targetSize;
-};
-
-auto bpsdelta::source(const uint8_t* data, uint size) -> void {
-  sourceData = data;
-  sourceSize = size;
-}
-
-auto bpsdelta::target(const uint8_t* data, uint size) -> void {
-  targetData = data;
-  targetSize = size;
-}
-
-auto bpsdelta::source(const string& filename) -> bool {
-  if(sourceFile.open(filename, filemap::mode::read) == false) return false;
-  source(sourceFile.data(), sourceFile.size());
-  return true;
-}
-
-auto bpsdelta::target(const string& filename) -> bool {
-  if(targetFile.open(filename, filemap::mode::read) == false) return false;
-  target(targetFile.data(), targetFile.size());
-  return true;
-}
-
-auto bpsdelta::create(const string& filename, const string& metadata) -> bool {
-  file modifyFile;
-  if(modifyFile.open(filename, file::mode::write) == false) return false;
-
-  Hash::CRC32 sourceChecksum, modifyChecksum;
-  uint sourceRelativeOffset = 0, targetRelativeOffset = 0, outputOffset = 0;
-
-  auto write = [&](uint8_t data) {
-    modifyFile.write(data);
-    modifyChecksum.input(data);
-  };
-
-  auto encode = [&](uint64_t data) {
-    while(true) {
-      uint64_t x = data & 0x7f;
-      data >>= 7;
-      if(data == 0) {
-        write(0x80 | x);
-        break;
-      }
-      write(x);
-      data--;
-    }
-  };
-
-  write('B');
-  write('P');
-  write('S');
-  write('1');
-
-  encode(sourceSize);
-  encode(targetSize);
-
-  uint markupSize = metadata.length();
-  encode(markupSize);
-  for(uint n = 0; n < markupSize; n++) write(metadata[n]);
-
-  Node* sourceTree[65536];
-  Node* targetTree[65536];
-  for(uint n = 0; n < 65536; n++) sourceTree[n] = nullptr, targetTree[n] = nullptr;
-
-  //source tree creation
-  for(uint offset = 0; offset < sourceSize; offset++) {
-    uint16_t symbol = sourceData[offset + 0];
-    sourceChecksum.input(symbol);
-    if(offset < sourceSize - 1) symbol |= sourceData[offset + 1] << 8;
-    Node *node = new Node;
-    node->offset = offset;
-    node->next = sourceTree[symbol];
-    sourceTree[symbol] = node;
-  }
-
-  uint targetReadLength = 0;
-
-  auto targetReadFlush = [&]() {
-    if(targetReadLength) {
-      encode(TargetRead | ((targetReadLength - 1) << 2));
-      uint offset = outputOffset - targetReadLength;
-      while(targetReadLength) write(targetData[offset++]), targetReadLength--;
-    }
-  };
-
-  while(outputOffset < targetSize) {
-    uint maxLength = 0, maxOffset = 0, mode = TargetRead;
-
-    uint16_t symbol = targetData[outputOffset + 0];
-    if(outputOffset < targetSize - 1) symbol |= targetData[outputOffset + 1] << 8;
-
-    { //source read
-      uint length = 0, offset = outputOffset;
-      while(offset < sourceSize && offset < targetSize && sourceData[offset] == targetData[offset]) {
-        length++;
-        offset++;
-      }
-      if(length > maxLength) maxLength = length, mode = SourceRead;
-    }
-
-    { //source copy
-      Node* node = sourceTree[symbol];
-      while(node) {
-        uint length = 0, x = node->offset, y = outputOffset;
-        while(x < sourceSize && y < targetSize && sourceData[x++] == targetData[y++]) length++;
-        if(length > maxLength) maxLength = length, maxOffset = node->offset, mode = SourceCopy;
-        node = node->next;
-      }
-    }
-
-    { //target copy
-      Node* node = targetTree[symbol];
-      while(node) {
-        uint length = 0, x = node->offset, y = outputOffset;
-        while(y < targetSize && targetData[x++] == targetData[y++]) length++;
-        if(length > maxLength) maxLength = length, maxOffset = node->offset, mode = TargetCopy;
-        node = node->next;
-      }
-
-      //target tree append
-      node = new Node;
-      node->offset = outputOffset;
-      node->next = targetTree[symbol];
-      targetTree[symbol] = node;
-    }
-
-    { //target read
-      if(maxLength < 4) {
-        maxLength = min((uint)Granularity, targetSize - outputOffset);
-        mode = TargetRead;
-      }
-    }
-
-    if(mode != TargetRead) targetReadFlush();
-
-    switch(mode) {
-    case SourceRead:
-      encode(SourceRead | ((maxLength - 1) << 2));
-      break;
-    case TargetRead:
-      //delay write to group sequential TargetRead commands into one
-      targetReadLength += maxLength;
-      break;
-    case SourceCopy:
-    case TargetCopy:
-      encode(mode | ((maxLength - 1) << 2));
-      int relativeOffset;
-      if(mode == SourceCopy) {
-        relativeOffset = maxOffset - sourceRelativeOffset;
-        sourceRelativeOffset = maxOffset + maxLength;
-      } else {
-        relativeOffset = maxOffset - targetRelativeOffset;
-        targetRelativeOffset = maxOffset + maxLength;
-      }
-      encode((relativeOffset < 0) | (abs(relativeOffset) << 1));
-      break;
-    }
-
-    outputOffset += maxLength;
-  }
-
-  targetReadFlush();
-
-  for(uint n = 0; n < 32; n += 8) write(sourceChecksum.digest().hex() >> n);
-  uint32_t targetChecksum = Hash::CRC32(targetData, targetSize).digest().hex();
-  for(uint n = 0; n < 32; n += 8) write(targetChecksum >> n);
-  uint32_t outputChecksum = modifyChecksum.digest().hex();
-  for(uint n = 0; n < 32; n += 8) write(outputChecksum >> n);
-
-  modifyFile.close();
-  return true;
-}
-
-}

nall/beat/file.hpp

@@ -1,23 +0,0 @@
-#pragma once
-
-namespace nall { namespace Beat {
-
-struct File : file {
-  using file::file;
-  auto read() -> uint8_t override;
-  auto write(uint8_t) -> void override;
-  Hash::CRC32 checksum;
-};
-
-auto File::read() -> uint8_t {
-  uint8_t data = file::read();
-  checksum.input(data);
-  return data;
-}
-
-auto File::write(uint8_t data) -> void {
-  checksum.input(data);
-  return file::write(data);
-}
-
-}}

nall/beat/linear.hpp

@@ -1,148 +0,0 @@
-#pragma once
-
-#include <nall/file.hpp>
-#include <nall/filemap.hpp>
-#include <nall/stdint.hpp>
-#include <nall/string.hpp>
-
-namespace nall {
-
-struct bpslinear {
-  inline auto source(const uint8_t* data, uint size) -> void;
-  inline auto target(const uint8_t* data, uint size) -> void;
-
-  inline auto source(const string& filename) -> bool;
-  inline auto target(const string& filename) -> bool;
-  inline auto create(const string& filename, const string& metadata = "") -> bool;
-
-protected:
-  enum : uint { SourceRead, TargetRead, SourceCopy, TargetCopy };
-  enum : uint { Granularity = 1 };
-
-  filemap sourceFile;
-  const uint8_t* sourceData;
-  uint sourceSize;
-
-  filemap targetFile;
-  const uint8_t* targetData;
-  uint targetSize;
-};
-
-auto bpslinear::source(const uint8_t* data, uint size) -> void {
-  sourceData = data;
-  sourceSize = size;
-}
-
-auto bpslinear::target(const uint8_t* data, uint size) -> void {
-  targetData = data;
-  targetSize = size;
-}
-
-auto bpslinear::source(const string& filename) -> bool {
-  if(sourceFile.open(filename, filemap::mode::read) == false) return false;
-  source(sourceFile.data(), sourceFile.size());
-  return true;
-}
-
-auto bpslinear::target(const string& filename) -> bool {
-  if(targetFile.open(filename, filemap::mode::read) == false) return false;
-  target(targetFile.data(), targetFile.size());
-  return true;
-}
-
-auto bpslinear::create(const string& filename, const string& metadata) -> bool {
-  file modifyFile;
-  if(modifyFile.open(filename, file::mode::write) == false) return false;
-
-  Hash::CRC32 modifyChecksum;
-  uint targetRelativeOffset = 0, outputOffset = 0;
-
-  auto write = [&](uint8_t data) {
-    modifyFile.write(data);
-    modifyChecksum.input(data);
-  };
-
-  auto encode = [&](uint64_t data) {
-    while(true) {
-      uint64_t x = data & 0x7f;
-      data >>= 7;
-      if(data == 0) {
-        write(0x80 | x);
-        break;
-      }
-      write(x);
-      data--;
-    }
-  };
-
-  uint targetReadLength = 0;
-
-  auto targetReadFlush = [&]() {
-    if(targetReadLength) {
-      encode(TargetRead | ((targetReadLength - 1) << 2));
-      uint offset = outputOffset - targetReadLength;
-      while(targetReadLength) write(targetData[offset++]), targetReadLength--;
-    }
-  };
-
-  write('B');
-  write('P');
-  write('S');
-  write('1');
-
-  encode(sourceSize);
-  encode(targetSize);
-
-  uint markupSize = metadata.length();
-  encode(markupSize);
-  for(uint n = 0; n < markupSize; n++) write(metadata[n]);
-
-  while(outputOffset < targetSize) {
-    uint sourceLength = 0;
-    for(uint n = 0; outputOffset + n < min(sourceSize, targetSize); n++) {
-      if(sourceData[outputOffset + n] != targetData[outputOffset + n]) break;
-      sourceLength++;
-    }
-
-    uint rleLength = 0;
-    for(uint n = 1; outputOffset + n < targetSize; n++) {
-      if(targetData[outputOffset] != targetData[outputOffset + n]) break;
-      rleLength++;
-    }
-
-    if(rleLength >= 4) {
-      //write byte to repeat
-      targetReadLength++;
-      outputOffset++;
-      targetReadFlush();
-
-      //copy starting from repetition byte
-      encode(TargetCopy | ((rleLength - 1) << 2));
-      uint relativeOffset = (outputOffset - 1) - targetRelativeOffset;
-      encode(relativeOffset << 1);
-      outputOffset += rleLength;
-      targetRelativeOffset = outputOffset - 1;
-    } else if(sourceLength >= 4) {
-      targetReadFlush();
-      encode(SourceRead | ((sourceLength - 1) << 2));
-      outputOffset += sourceLength;
-    } else {
-      targetReadLength += Granularity;
-      outputOffset += Granularity;
-    }
-  }
-
-  targetReadFlush();
-
-  uint32_t sourceChecksum = Hash::CRC32(sourceData, sourceSize).digest().hex();
-  for(uint n = 0; n < 32; n += 8) write(sourceChecksum >> n);
-  uint32_t targetChecksum = Hash::CRC32(targetData, targetSize).digest().hex();
-  for(uint n = 0; n < 32; n += 8) write(targetChecksum >> n);
-  uint32_t outputChecksum = modifyChecksum.digest().hex();
-  for(uint n = 0; n < 32; n += 8) write(outputChecksum >> n);
-
-  modifyFile.close();
-  return true;
-}
-
-}

nall/beat/metadata.hpp

@@ -1,117 +0,0 @@
-#pragma once
-
-#include <nall/file.hpp>
-#include <nall/filemap.hpp>
-#include <nall/stdint.hpp>
-#include <nall/string.hpp>
-
-namespace nall {
-
-struct bpsmetadata {
-  inline auto load(const string& filename) -> bool;
-  inline auto save(const string& filename, const string& metadata) -> bool;
-  inline auto metadata() const -> string;
-
-protected:
-  file sourceFile;
-  string metadataString;
-};
-
-auto bpsmetadata::load(const string& filename) -> bool {
-  if(sourceFile.open(filename, file::mode::read) == false) return false;
-
-  auto read = [&]() -> uint8_t {
-    return sourceFile.read();
-  };
-
-  auto decode = [&]() -> uint64_t {
-    uint64_t data = 0, shift = 1;
-    while(true) {
-      uint8_t x = read();
-      data += (x & 0x7f) * shift;
-      if(x & 0x80) break;
-      shift <<= 7;
-      data += shift;
-    }
-    return data;
-  };
-
-  if(read() != 'B') return false;
-  if(read() != 'P') return false;
-  if(read() != 'S') return false;
-  if(read() != '1') return false;
-  decode();
-  decode();
-  uint metadataSize = decode();
-  char data[metadataSize + 1];
-  for(uint n = 0; n < metadataSize; n++) data[n] = read();
-  data[metadataSize] = 0;
-  metadataString = (const char*)data;
-
-  return true;
-}
-
-auto bpsmetadata::save(const string& filename, const string& metadata) -> bool {
-  file targetFile;
-  if(targetFile.open(filename, file::mode::write) == false) return false;
-  if(sourceFile.open() == false) return false;
-  sourceFile.seek(0);
-
-  auto read = [&]() -> uint8_t {
-    return sourceFile.read();
-  };
-
-  auto decode = [&]() -> uint64_t {
-    uint64_t data = 0, shift = 1;
-    while(true) {
-      uint8_t x = read();
-      data += (x & 0x7f) * shift;
-      if(x & 0x80) break;
-      shift <<= 7;
-      data += shift;
-    }
-    return data;
-  };
-
-  Hash::CRC32 checksum;
-
-  auto write = [&](uint8_t data) {
-    targetFile.write(data);
-    checksum.input(data);
-  };
-
-  auto encode = [&](uint64_t data) {
-    while(true) {
-      uint64_t x = data & 0x7f;
-      data >>= 7;
-      if(data == 0) {
-        write(0x80 | x);
-        break;
-      }
-      write(x);
-      data--;
-    }
-  };
-
-  for(uint n = 0; n < 4; n++) write(read());
-  encode(decode());
-  encode(decode());
-  uint sourceLength = decode();
-  uint targetLength = metadata.length();
-  encode(targetLength);
-  sourceFile.seek(sourceLength, file::index::relative);
-  for(uint n = 0; n < targetLength; n++) write(metadata[n]);
-  uint length = sourceFile.size() - sourceFile.offset() - 4;
-  for(uint n = 0; n < length; n++) write(read());
-  uint32_t outputChecksum = checksum.digest().hex();
-  for(uint n = 0; n < 32; n += 8) write(outputChecksum >> n);
-
-  targetFile.close();
-  return true;
-}
-
-auto bpsmetadata::metadata() const -> string {
-  return metadataString;
-}
-
-}

nall/beat/multi.hpp

@@ -1,239 +0,0 @@
-#pragma once
-
-#include <nall/beat/patch.hpp>
-#include <nall/beat/linear.hpp>
-#include <nall/beat/delta.hpp>
-
-namespace nall {
-
-struct bpsmulti {
-  enum : uint {
-    CreatePath = 0,
-    CreateFile = 1,
-    ModifyFile = 2,
-    MirrorFile = 3,
-  };
-
-  enum : uint {
-    OriginSource = 0,
-    OriginTarget = 1,
-  };
-
-  auto create(const string& patchName, const string& sourcePath, const string& targetPath, bool delta = false, const string& metadata = "") -> bool {
-    if(fp.open()) fp.close();
-    fp.open(patchName, file::mode::write);
-    checksum.reset();
-
-    writeString("BPM1");  //signature
-    writeNumber(metadata.length());
-    writeString(metadata);
-
-    vector<string> sourceList, targetList;
-    ls(sourceList, sourcePath, sourcePath);
-    ls(targetList, targetPath, targetPath);
-
-    for(auto& targetName : targetList) {
-      if(targetName.endsWith("/")) {
-        targetName.trimRight("/");
-        writeNumber(CreatePath | ((targetName.length() - 1) << 2));
-        writeString(targetName);
-      } else if(auto position = sourceList.find(targetName)) {  //if sourceName == targetName
-        file sp, dp;
-        sp.open({sourcePath, targetName}, file::mode::read);
-        dp.open({targetPath, targetName}, file::mode::read);
-
-        bool identical = sp.size() == dp.size();
-        Hash::CRC32 cksum;
-
-        for(uint n = 0; n < sp.size(); n++) {
-          uint8_t byte = sp.read();
-          if(identical && byte != dp.read()) identical = false;
-          cksum.input(byte);
-        }
-
-        if(identical) {
-          writeNumber(MirrorFile | ((targetName.length() - 1) << 2));
-          writeString(targetName);
-          writeNumber(OriginSource);
-          writeChecksum(cksum.digest().hex());
-        } else {
-          writeNumber(ModifyFile | ((targetName.length() - 1) << 2));
-          writeString(targetName);
-          writeNumber(OriginSource);
-
-          if(delta == false) {
-            bpslinear patch;
-            patch.source({sourcePath, targetName});
-            patch.target({targetPath, targetName});
-            patch.create({Path::temporary(), "temp.bps"});
-          } else {
-            bpsdelta patch;
-            patch.source({sourcePath, targetName});
-            patch.target({targetPath, targetName});
-            patch.create({Path::temporary(), "temp.bps"});
-          }
-
-          auto buffer = file::read({Path::temporary(), "temp.bps"});
-          writeNumber(buffer.size());
-          for(auto &byte : buffer) write(byte);
-        }
-      } else {
-        writeNumber(CreateFile | ((targetName.length() - 1) << 2));
-        writeString(targetName);
-        auto buffer = file::read({targetPath, targetName});
-        writeNumber(buffer.size());
-        for(auto& byte : buffer) write(byte);
-        writeChecksum(Hash::CRC32(buffer.data(), buffer.size()).digest().hex());
-      }
-    }
-
-    //checksum
-    writeChecksum(checksum.digest().hex());
-    fp.close();
-    return true;
-  }
-
-  auto apply(const string& patchName, const string& sourcePath, const string& targetPath) -> bool {
-    directory::remove(targetPath);  //start with a clean directory
-    directory::create(targetPath);
-
-    if(fp.open()) fp.close();
-    fp.open(patchName, file::mode::read);
-    checksum.reset();
-
-    if(readString(4) != "BPM1") return false;
-    auto metadataLength = readNumber();
-    while(metadataLength--) read();
-
-    while(fp.offset() < fp.size() - 4) {
-      auto encoding = readNumber();
-      uint action = encoding & 3;
-      uint targetLength = (encoding >> 2) + 1;
-      string targetName = readString(targetLength);
-
-      if(action == CreatePath) {
-        directory::create({targetPath, targetName, "/"});
-      } else if(action == CreateFile) {
-        file fp;
-        fp.open({targetPath, targetName}, file::mode::write);
-        auto fileSize = readNumber();
-        while(fileSize--) fp.write(read());
-        uint32_t cksum = readChecksum();
-      } else if(action == ModifyFile) {
-        auto encoding = readNumber();
-        string originPath = encoding & 1 ? targetPath : sourcePath;
-        string sourceName = (encoding >> 1) == 0 ? targetName : readString(encoding >> 1);
-        auto patchSize = readNumber();
-        vector<uint8_t> buffer;
-        buffer.resize(patchSize);
-        for(uint n = 0; n < patchSize; n++) buffer[n] = read();
-        bpspatch patch;
-        patch.modify(buffer.data(), buffer.size());
-        patch.source({originPath, sourceName});
-        patch.target({targetPath, targetName});
-        if(patch.apply() != bpspatch::result::success) return false;
-      } else if(action == MirrorFile) {
-        auto encoding = readNumber();
-        string originPath = encoding & 1 ? targetPath : sourcePath;
-        string sourceName = (encoding >> 1) == 0 ? targetName : readString(encoding >> 1);
-        file::copy({originPath, sourceName}, {targetPath, targetName});
-        uint32_t cksum = readChecksum();
-      }
-    }
-
-    uint32_t cksum = checksum.digest().hex();
-    if(read() != (uint8_t)(cksum >>  0)) return false;
-    if(read() != (uint8_t)(cksum >>  8)) return false;
-    if(read() != (uint8_t)(cksum >> 16)) return false;
-    if(read() != (uint8_t)(cksum >> 24)) return false;
-
-    fp.close();
-    return true;
-  }
-
-protected:
-  file fp;
-  Hash::CRC32 checksum;
-
-  //create() functions
-  auto ls(vector<string>& list, const string& path, const string& basepath) -> void {
-    auto paths = directory::folders(path);
-    for(auto& pathname : paths) {
-      list.append(string{path, pathname}.trimLeft(basepath, 1L));
-      ls(list, {path, pathname}, basepath);
-    }
-
-    auto files = directory::files(path);
-    for(auto& filename : files) {
-      list.append(string{path, filename}.trimLeft(basepath, 1L));
-    }
-  }
-
-  auto write(uint8_t data) -> void {
-    fp.write(data);
-    checksum.input(data);
-  }
-
-  auto writeNumber(uint64_t data) -> void {
-    while(true) {
-      uint64_t x = data & 0x7f;
-      data >>= 7;
-      if(data == 0) {
-        write(0x80 | x);
-        break;
-      }
-      write(x);
-      data--;
-    }
-  }
-
-  auto writeString(const string& text) -> void {
-    uint length = text.length();
-    for(uint n = 0; n < length; n++) write(text[n]);
-  }
-
-  auto writeChecksum(uint32_t cksum) -> void {
-    write(cksum >>  0);
-    write(cksum >>  8);
-    write(cksum >> 16);
-    write(cksum >> 24);
-  }
-
-  //apply() functions
-  auto read() -> uint8_t {
-    uint8_t data = fp.read();
-    checksum.input(data);
-    return data;
-  }
-
-  auto readNumber() -> uint64_t {
-    uint64_t data = 0, shift = 1;
-    while(true) {
-      uint8_t x = read();
-      data += (x & 0x7f) * shift;
-      if(x & 0x80) break;
-      shift <<= 7;
-      data += shift;
-    }
-    return data;
-  }
-
-  auto readString(uint length) -> string {
-    string text;
-    text.resize(length + 1);
-    char* p = text.get();
-    while(length--) *p++ = read();
-    return text;
-  }
-
-  auto readChecksum() -> uint32_t {
-    uint32_t checksum = 0;
-    checksum |= read() <<  0;
-    checksum |= read() <<  8;
-    checksum |= read() << 16;
-    checksum |= read() << 24;
-    return checksum;
-  }
-};
-
-}

nall/beat/patch.hpp

@@ -1,214 +0,0 @@
-#pragma once
-
-#include <nall/file.hpp>
-#include <nall/filemap.hpp>
-#include <nall/stdint.hpp>
-#include <nall/string.hpp>
-
-namespace nall {
-
-struct bpspatch {
-  inline auto modify(const uint8_t* data, uint size) -> bool;
-  inline auto source(const uint8_t* data, uint size) -> void;
-  inline auto target(uint8_t* data, uint size) -> void;
-
-  inline auto modify(const string& filename) -> bool;
-  inline auto source(const string& filename) -> bool;
-  inline auto target(const string& filename) -> bool;
-
-  inline auto metadata() const -> string;
-  inline auto size() const -> uint;
-
-  enum result : uint {
-    unknown,
-    success,
-    patch_too_small,
-    patch_invalid_header,
-    source_too_small,
-    target_too_small,
-    source_checksum_invalid,
-    target_checksum_invalid,
-    patch_checksum_invalid,
-  };
-
-  inline auto apply() -> result;
-
-protected:
-  enum : uint { SourceRead, TargetRead, SourceCopy, TargetCopy };
-
-  filemap modifyFile;
-  const uint8_t* modifyData;
-  uint modifySize;
-
-  filemap sourceFile;
-  const uint8_t* sourceData;
-  uint sourceSize;
-
-  filemap targetFile;
-  uint8_t* targetData;
-  uint targetSize;
-
-  uint modifySourceSize;
-  uint modifyTargetSize;
-  uint modifyMarkupSize;
-  string metadataString;
-};
-
-auto bpspatch::modify(const uint8_t* data, uint size) -> bool {
-  if(size < 19) return false;
-  modifyData = data;
-  modifySize = size;
-
-  uint offset = 4;
-  auto decode = [&]() -> uint64_t {
-    uint64_t data = 0, shift = 1;
-    while(true) {
-      uint8_t x = modifyData[offset++];
-      data += (x & 0x7f) * shift;
-      if(x & 0x80) break;
-      shift <<= 7;
-      data += shift;
-    }
-    return data;
-  };
-
-  modifySourceSize = decode();
-  modifyTargetSize = decode();
-  modifyMarkupSize = decode();
-
-  char buffer[modifyMarkupSize + 1];
-  for(uint n = 0; n < modifyMarkupSize; n++) buffer[n] = modifyData[offset++];
-  buffer[modifyMarkupSize] = 0;
-  metadataString = (const char*)buffer;
-
-  return true;
-}
-
-auto bpspatch::source(const uint8_t* data, uint size) -> void {
-  sourceData = data;
-  sourceSize = size;
-}
-
-auto bpspatch::target(uint8_t* data, uint size) -> void {
-  targetData = data;
-  targetSize = size;
-}
-
-auto bpspatch::modify(const string& filename) -> bool {
-  if(modifyFile.open(filename, filemap::mode::read) == false) return false;
-  return modify(modifyFile.data(), modifyFile.size());
-}
-
-auto bpspatch::source(const string& filename) -> bool {
-  if(sourceFile.open(filename, filemap::mode::read) == false) return false;
-  source(sourceFile.data(), sourceFile.size());
-  return true;
-}
-
-auto bpspatch::target(const string& filename) -> bool {
-  file fp;
-  if(fp.open(filename, file::mode::write) == false) return false;
-  fp.truncate(modifyTargetSize);
-  fp.close();
-
-  if(targetFile.open(filename, filemap::mode::readwrite) == false) return false;
-  target(targetFile.data(), targetFile.size());
-  return true;
-}
-
-auto bpspatch::metadata() const -> string {
-  return metadataString;
-}
-
-auto bpspatch::size() const -> uint {
-  return modifyTargetSize;
-}
-
-auto bpspatch::apply() -> result {
-  if(modifySize < 19) return result::patch_too_small;
-
-  Hash::CRC32 modifyChecksum, targetChecksum;
-  uint modifyOffset = 0, sourceRelativeOffset = 0, targetRelativeOffset = 0, outputOffset = 0;
-
-  auto read = [&]() -> uint8_t {
-    uint8_t data = modifyData[modifyOffset++];
-    modifyChecksum.input(data);
-    return data;
-  };
-
-  auto decode = [&]() -> uint64_t {
-    uint64_t data = 0, shift = 1;
-    while(true) {
-      uint8_t x = read();
-      data += (x & 0x7f) * shift;
-      if(x & 0x80) break;
-      shift <<= 7;
-      data += shift;
-    }
-    return data;
-  };
-
-  auto write = [&](uint8_t data) {
-    targetData[outputOffset++] = data;
-    targetChecksum.input(data);
-  };
-
-  if(read() != 'B') return result::patch_invalid_header;
-  if(read() != 'P') return result::patch_invalid_header;
-  if(read() != 'S') return result::patch_invalid_header;
-  if(read() != '1') return result::patch_invalid_header;
-
-  modifySourceSize = decode();
-  modifyTargetSize = decode();
-  modifyMarkupSize = decode();
-  for(uint n = 0; n < modifyMarkupSize; n++) read();
-
-  if(modifySourceSize > sourceSize) return result::source_too_small;
-  if(modifyTargetSize > targetSize) return result::target_too_small;
-
-  while(modifyOffset < modifySize - 12) {
-    uint length = decode();
-    uint mode = length & 3;
-    length = (length >> 2) + 1;
-
-    switch(mode) {
-    case SourceRead:
-      while(length--) write(sourceData[outputOffset]);
-      break;
-    case TargetRead:
-      while(length--) write(read());
-      break;
-    case SourceCopy:
-    case TargetCopy:
-      int offset = decode();
-      bool negative = offset & 1;
-      offset >>= 1;
-      if(negative) offset = -offset;
-
-      if(mode == SourceCopy) {
-        sourceRelativeOffset += offset;
-        while(length--) write(sourceData[sourceRelativeOffset++]);
-      } else {
-        targetRelativeOffset += offset;
-        while(length--) write(targetData[targetRelativeOffset++]);
-      }
-      break;
-    }
-  }
-
-  uint32_t modifySourceChecksum = 0, modifyTargetChecksum = 0, modifyModifyChecksum = 0;
-  for(uint n = 0; n < 32; n += 8) modifySourceChecksum |= read() << n;
-  for(uint n = 0; n < 32; n += 8) modifyTargetChecksum |= read() << n;
-  uint32_t checksum = modifyChecksum.digest().hex();
-  for(uint n = 0; n < 32; n += 8) modifyModifyChecksum |= read() << n;
-
-  uint32_t sourceChecksum = Hash::CRC32(sourceData, modifySourceSize).digest().hex();
-
-  if(sourceChecksum != modifySourceChecksum) return result::source_checksum_invalid;
-  if(targetChecksum.digest().hex() != modifyTargetChecksum) return result::target_checksum_invalid;
-  if(checksum != modifyModifyChecksum) return result::patch_checksum_invalid;
-
-  return result::success;
-}
-
-}

nall/beat/single/apply.hpp

@@ -0,0 +1,92 @@
+#pragma once
+
+namespace nall { namespace Beat { namespace Single {
+
+inline auto apply(
+  const uint8_t* sourceData, uint64_t sourceSize,
+  const uint8_t* beatData, uint64_t beatSize,
+  maybe<string&> manifest = {}, maybe<string&> result = {}
+) -> maybe<vector<uint8_t>> {
+  #define error(text) { if(result) *result = {"error: ", text}; return {}; }
+  #define warning(text) { if(result) *result = {"warning: ", text}; return targetData; }
+  #define success() { if(result) *result = ""; return targetData; }
+  if(beatSize < 19) error("beat size mismatch");
+
+  vector<uint8_t> targetData;
+
+  uint beatOffset = 0;
+  auto read = [&]() -> uint8_t {
+    return beatData[beatOffset++];
+  };
+  auto decode = [&]() -> uint64_t {
+    uint64_t data = 0, shift = 1;
+    while(true) {
+      uint8_t x = read();
+      data += (x & 0x7f) * shift;
+      if(x & 0x80) break;
+      shift <<= 7;
+      data += shift;
+    }
+    return data;
+  };
+
+  auto targetOffset = 0;
+  auto write = [&](uint8_t data) {
+    targetData.append(data);
+  };
+
+  if(read() != 'B') error("beat header invalid");
+  if(read() != 'P') error("beat header invalid");
+  if(read() != 'S') error("beat header invalid");
+  if(read() != '1') error("beat version mismatch");
+  if(decode() != sourceSize) error("source size mismatch");
+  uint targetSize = decode();
+  targetData.reserve(targetSize);
+  auto metadataSize = decode();
+  for(uint n : range(metadataSize)) {
+    auto data = read();
+    if(manifest) manifest->append((char)data);
+  }
+
+  enum : uint { SourceRead, TargetRead, SourceCopy, TargetCopy };
+
+  uint sourceRelativeOffset = 0, targetRelativeOffset = 0;
+  while(beatOffset < beatSize - 12) {
+    uint length = decode();
+    uint mode = length & 3;
+    length = (length >> 2) + 1;
+
+    if(mode == SourceRead) {
+      while(length--) write(sourceData[targetOffset]);
+    } else if(mode == TargetRead) {
+      while(length--) write(read());
+    } else {
+      int offset = decode();
+      offset = offset & 1 ? -(offset >> 1) : (offset >> 1);
+      if(mode == SourceCopy) {
+        sourceRelativeOffset += offset;
+        while(length--) write(sourceData[sourceRelativeOffset++]);
+      } else {
+        targetRelativeOffset += offset;
+        while(length--) write(targetData[targetRelativeOffset++]);
+      }
+    }
+  }
+
+  uint32_t sourceHash = 0, targetHash = 0, beatHash = 0;
+  for(uint shift : range(0, 32, 8)) sourceHash |= read() << shift;
+  for(uint shift : range(0, 32, 8)) targetHash |= read() << shift;
+  for(uint shift : range(0, 32, 8)) beatHash |= read() << shift;
+
+  if(targetOffset != targetSize) warning("target size mismatch");
+  if(sourceHash != Hash::CRC32(sourceData, sourceSize).value()) warning("source hash mismatch");
+  if(targetHash != Hash::CRC32(targetData).value()) warning("target hash mismatch");
+  if(beatHash != Hash::CRC32(beatData, beatSize - 4).value()) warning("beat hash mismatch");
+
+  success();
+  #undef error
+  #undef warning
+  #undef success
+}
+
+}}}

nall/beat/single/create.hpp

@@ -0,0 +1,123 @@
+#pragma once
+
+#include <nall/encode/dictionary.hpp>
+
+namespace nall { namespace Beat { namespace Single {
+
+inline auto create(
+  const uint8_t* sourceData, uint64_t sourceSize,
+  const uint8_t* targetData, uint64_t targetSize,
+  const string& manifest = {}
+) -> maybe<vector<uint8_t>> {
+  vector<uint8_t> beatData;
+
+  auto write = [&](uint8_t data) {
+    beatData.append(data);
+  };
+
+  auto encode = [&](uint64_t data) {
+    while(true) {
+      uint64_t x = data & 0x7f;
+      data >>= 7;
+      if(data == 0) { write(0x80 | x); break; }
+      write(x);
+      data--;
+    }
+  };
+
+  write('B'), write('P'), write('S'), write('1');
+  encode(sourceSize), encode(targetSize), encode(manifest.size());
+  for(auto& byte : manifest) write(byte);
+
+  auto read = [&](const uint8_t* data, uint size, uint offset) -> uint {
+    if(offset + 3 >= size) return 0;
+    return data[offset + 0] << 24 | data[offset + 1] << 16 | data[offset + 2] << 8 | data[offset + 3] << 0;
+  };
+
+  Encode::Dictionary sourceDictionary(sourceData, sourceSize);
+  Encode::Dictionary targetDictionary(targetData, targetSize);
+  sourceDictionary.scan();
+  targetDictionary.scan();
+
+  enum : uint { SourceRead, TargetRead, SourceCopy, TargetCopy };
+  uint outputOffset = 0, sourceRelativeOffset = 0, targetRelativeOffset = 0;
+
+  uint targetReadLength = 0;
+  auto flush = [&] {
+    if(!targetReadLength) return;
+    encode(TargetRead | ((targetReadLength - 1) << 2));
+    uint offset = outputOffset - targetReadLength;
+    while(targetReadLength) write(targetData[offset++]), targetReadLength--;
+  };
+
+  uint longestSize = max(sourceSize, targetSize);
+  while(outputOffset < targetSize) {
+    uint mode = TargetRead, longestLength = 3, longestOffset = 0;
+    uint prefix = read(targetData, targetSize, outputOffset), lower, upper;
+
+    uint length = 0, offset = outputOffset;
+    while(offset < longestOffset) {
+      if(sourceData[offset] != targetData[offset]) break;
+      length++, offset++;
+    }
+    if(length > longestLength) {
+      mode = SourceRead, longestLength = length;
+    }
+
+    sourceDictionary.find(prefix, lower, upper);
+    for(uint index = lower; index <= upper; index++) {
+      uint length = 0, sourceOffset = sourceDictionary[index], targetOffset = outputOffset;
+      while(sourceOffset < sourceSize && targetOffset < targetSize) {
+        if(sourceData[sourceOffset++] != targetData[targetOffset++]) break;
+        length++;
+      }
+      if(length > longestLength) {
+        mode = SourceCopy, longestLength = length, longestOffset = sourceDictionary[index];
+      }
+    }
+
+    targetDictionary.find(prefix, lower, upper);
+    for(uint index = lower; index <= upper; index++) {
+      uint length = 0, sourceOffset = targetDictionary[index], targetOffset = outputOffset;
+      if(sourceOffset >= outputOffset) continue;
+      while(targetOffset < targetSize) {
+        if(targetData[sourceOffset++] != targetData[targetOffset++]) break;
+        length++;
+      }
+      if(length > longestLength) {
+        mode = TargetCopy, longestLength = length, longestOffset = targetDictionary[index];
+      }
+    }
+
+    if(mode == TargetRead) {
+      targetReadLength++;  //queue writes to group sequential commands
+      outputOffset++;
+    } else {
+      flush();
+      encode(mode | ((longestLength - 1) << 2));
+      if(mode == SourceCopy) {
+        int relativeOffset = longestOffset - sourceRelativeOffset;
+        sourceRelativeOffset = longestOffset + longestLength;
+        encode(relativeOffset < 0 | abs(relativeOffset) << 1);
+      }
+      if(mode == TargetCopy) {
+        int relativeOffset = longestOffset - targetRelativeOffset;
+        targetRelativeOffset = longestOffset + longestLength;
+        encode(relativeOffset < 0 | abs(relativeOffset) << 1);
+      }
+      outputOffset += longestLength;
+    }
+  }
+  flush();
+
+  auto sourceHash = Hash::CRC32(sourceData, sourceSize);
+  for(uint shift : range(0, 32, 8)) write(sourceHash.value() >> shift);
+  auto targetHash = Hash::CRC32(targetData, targetSize);
+  for(uint shift : range(0, 32, 8)) write(targetHash.value() >> shift);
+  auto beatHash = Hash::CRC32(beatData);
+  for(uint shift : range(0, 32, 8)) write(beatHash.value() >> shift);
+
+  return beatData;
+}
+
+}}}

nall/counting-sort.hpp

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <nall/range.hpp>
+
+namespace nall {
+
+//counting sort by powers of two: used to implement radix sort
+template<uint Bits, uint Shift, typename T>
+auto counting_sort(T* output, const T* input, uint size) -> void {
+  static_assert(Bits >= 1 && Bits <= 20, "must be between 1 and 20 bits");
+  enum : uint { Base = 1 << Bits, Mask = Base - 1 };
+
+  uint64_t count[Base] = {}, last = 0;
+  for(uint n : range(size)) ++count[(input[n] >> Shift) & Mask];
+  for(uint n : range(Base)) last += count[n], count[n] = last - count[n];
+  for(uint n : range(size)) output[count[(input[n] >> Shift) & Mask]++] = input[n];
+}
+
+}

nall/decode/bwt.hpp

@@ -0,0 +1,56 @@
+#pragma once
+
+//burrows-wheeler transform
+
+#include <nall/suffix-array.hpp>
+
+namespace nall { namespace Decode {
+
+inline auto BWT(const void* data) -> vector<uint8_t> {
+  auto input = (const uint8_t*)data;
+  vector<uint8_t> output;
+
+  uint size = 0;
+  for(uint byte : range(8)) size |= *input++ << byte * 8;
+  output.resize(size);
+
+  uint I = 0;
+  for(uint byte : range(8)) I |= *input++ << byte * 8;
+
+  auto suffixes = new int[size];
+  suffix_array(suffixes, input, size);
+
+  auto L = input;
+  auto F = new uint8_t[size];
+  for(uint byte : range(size)) F[byte] = L[suffixes[byte]];
+
+  delete[] suffixes;
+
+  uint64_t K[256] = {};
+  auto C = new int[size];
+  for(uint i : range(size)) {
+    C[i] = K[L[i]];
+    K[L[i]]++;
+  }
+
+  int M[256];
+  memory::fill<int>(M, 256, -1);
+  for(uint i : range(size)) {
+    if(M[F[i]] == -1) M[F[i]] = i;
+  }
+
+  uint i = I;
+  for(uint j : reverse(range(size))) {
+    output[j] = L[i];
+    i = C[i] + M[L[i]];
+  }
+
+  return output;
+}
+
+template<typename T>
+inline auto BWT(const vector<T>& buffer) -> vector<uint8_t> {
+  return move(BWT(buffer.data()));
+}
+
+}}

nall/decode/huffman.hpp

@@ -0,0 +1,42 @@
+#pragma once
+
+namespace nall { namespace Decode {
+
+inline auto Huffman(const void* data) -> vector<uint8_t> {
+  auto input = (const uint8_t*)data;
+  vector<uint8_t> output;
+
+  uint size = 0;
+  for(uint byte : range(8)) size |= *input++ << byte * 8;
+  output.reserve(size);
+
+  uint byte = 0, bits = 0;
+  auto read = [&]() -> bool {
+    if(bits == 0) bits = 8, byte = *input++;
+    return byte >> --bits & 1;
+  };
+
+  uint nodes[256][2] = {};
+  for(uint offset : range(256)) {
+    for(uint index : range(9)) nodes[offset][0] = nodes[offset][0] << 1 | read();
+    for(uint index : range(9)) nodes[offset][1] = nodes[offset][1] << 1 | read();
+  }
+
+  uint node = 511;
+  while(output.size() < size) {
+    node = nodes[node - 256][read()];
+    if(node < 256) {
+      output.append(node);
+      node = 511;
+    }
+  }
+
+  return output;
+}
+
+template<typename T>
+inline auto Huffman(const vector<T>& buffer) -> vector<uint8_t> {
+  return move(Huffman(buffer.data()));
+}
+
+}}

nall/decode/lzsa.hpp

@@ -0,0 +1,75 @@
+#pragma once
+
+#include <nall/decode/huffman.hpp>
+
+namespace nall { namespace Decode {
+
+inline auto LZSA(const void* data) -> vector<uint8_t> {
+  vector<uint8_t> output;
+
+  auto input = (const uint8_t*)data;
+  uint index = 0;
+
+  uint size = 0;
+  for(uint byte : range(8)) size |= *input++ << byte * 8;
+  output.resize(size);
+
+  auto load = [&]() -> vector<uint8_t> {
+    uint size = 0;
+    for(uint byte : range(8)) size |= *input++ << byte * 8;
+    vector<uint8_t> buffer;
+    buffer.reserve(size);
+    while(size--) buffer.append(*input++);
+    return buffer;
+  };
+
+  auto flags = Decode::Huffman(load());
+  auto literals = Decode::Huffman(load());
+//auto literals = Decode::BWT(Decode::Huffman(load()));
+  auto lengths = Decode::Huffman(load());
+  auto offsets = Decode::Huffman(load());
+
+  auto flagData = flags.data();
+  uint byte = 0, bits = 0;
+  auto flagRead = [&]() -> bool {
+    if(bits == 0) bits = 8, byte = *flagData++;
+    return byte >> --bits & 1;
+  };
+
+  auto literalData = literals.data();
+  auto literalRead = [&]() -> uint8_t {
+    return *literalData++;
+  };
+
+  auto lengthData = lengths.data();
+  auto lengthRead = [&]() -> uint64_t {
+    uint byte = *lengthData++, bytes = 1;
+    while(!(byte & 1)) byte >>= 1, bytes++;
+    uint length = byte >> 1, shift = 8 - bytes;
+    while(--bytes) length |= *lengthData++ << shift, shift += 8;
+    return length;
+  };
+
+  auto offsetData = offsets.data();
+  auto offsetRead = [&]() -> uint {
+    uint offset = 0;
+    offset |= *offsetData++ <<  0; if(index < 1 <<  8) return offset;
+    offset |= *offsetData++ <<  8; if(index < 1 << 16) return offset;
+    offset |= *offsetData++ << 16; if(index < 1 << 24) return offset;
+    offset |= *offsetData++ << 24; return offset;
+  };
+
+  while(index < size) {
+    if(!flagRead()) {
+      output[index++] = literalRead();
+    } else {
+      uint length = lengthRead() + 6;
+      uint offset = index - offsetRead();
+      while(length--) output[index++] = output[offset++];
+    }
+  }
+
+  return output;
+}
+
+}}

nall/decode/lzss.hpp

@@ -0,0 +1,44 @@
+#pragma once
+
+namespace nall { namespace Decode {
+
+inline auto LZSS(const void* data) -> vector<uint8_t> {
+  vector<uint8_t> output;
+
+  auto input = (const uint8_t*)data;
+  uint64_t size = 0;
+  for(uint byte : range(8)) size |= *input++ << byte * 8;
+  output.resize(size);
+  const uint windowBits = *input++;
+  const uint lengthBits = *input++;
+
+  const uint lengthExtends = 4 + (1 << lengthBits) - 1;
+  const uint windowMask = (1 << windowBits) - 1;
+
+  for(uint offset = 0, flags = 0, bit = 7; offset < size;) {
+    if(++bit == 8) bit = 0, flags = *input++;
+
+    if(flags & 1 << bit) {
+      uint encoding = 0;
+      encoding |= *input++ <<  0;
+      encoding |= *input++ <<  8;
+      encoding |= *input++ << 16;
+
+      uint length = 4 + (encoding >> windowBits);
+      uint window = 1 + (encoding & windowMask);
+      if(length == lengthExtends) length += *input++;
+
+      for(uint index : range(length)) {
+        if(offset + index >= size) break;
+        output[offset + index] = output[offset + index - window];
+      }
+      offset += length;
+    } else {
+      output[offset++] = *input++;
+    }
+  }
+
+  return output;
+}
+
+}}

nall/decode/mtf.hpp

@@ -0,0 +1,31 @@
+#pragma once
+
+//move to front
+
+namespace nall { namespace Decode {
+
+inline auto MTF(const void* data, uint size) -> vector<uint8_t> {
+  auto input = (const uint8_t*)data;
+  vector<uint8_t> output;
+  output.resize(size);
+
+  uint8_t order[256];
+  for(uint n : range(256)) order[n] = n;
+
+  for(uint offset = 0; offset < size; offset++) {
+    auto data = input[offset];
+    for(uint index = 0; index < 256; index++) {
+      uint value = order[data];
+      if(value == index) {
+        output[offset] = value;
+        memory::move(&order[1], &order[0], index);
+        order[0] = index;
+        break;
+      }
+    }
+  }
+
+  return output;
+}
+
+}}

nall/decode/rle.hpp

@@ -2,45 +2,46 @@
 
 namespace nall { namespace Decode {
 
-template<typename T> inline auto RLE(const uint8_t* data, uint remaining = ~0, uint minimum = 0) -> vector<T> {
+template<uint S = 1, uint M = 4 / S>  //S = word size; M = match length
-  if(!minimum) minimum = max(1, 4 / sizeof(T));
+inline auto RLE(const void* data, uint remaining = ~0) -> vector<uint8_t> {
-  vector<T> result;
+  vector<uint8_t> output;
+
+  auto input = (const uint8_t*)data;
 
   auto load = [&]() -> uint8_t {
     if(!remaining) return 0x00;
-    return --remaining, *data++;
+    return --remaining, *input++;
   };
 
   uint base = 0;
-  uint size = 0;
+  uint64_t size = 0;
-  for(uint byte : range(sizeof(uint))) size |= load() << byte * 8;
+  for(uint byte : range(8)) size |= load() << byte * 8;
-  size /= sizeof(T);
+  output.resize(size);
-  result.resize(size);
 
-  auto read = [&]() -> T {
+  auto read = [&]() -> uint64_t {
-    T value = 0;
+    uint64_t value = 0;
-    for(uint byte : range(sizeof(T))) value |= load() << byte * 8;
+    for(uint byte : range(S)) value |= load() << byte * 8;
     return value;
   };
 
-  auto write = [&](T value) -> void {
+  auto write = [&](uint64_t value) -> void {
     if(base >= size) return;
-    result[base++] = value;
+    for(uint byte : range(S)) output[base++] = value >> byte * 8;
   };
 
   while(base < size) {
-    auto byte = *data++;
+    auto byte = load();
     if(byte < 128) {
       byte++;
       while(byte--) write(read());
     } else {
       auto value = read();
-      byte = (byte & 127) + minimum;
+      byte = (byte & 127) + M;
       while(byte--) write(value);
     }
   }
 
-  return result;
+  return output;
 }
 
 }}

nall/directory.hpp

@@ -3,7 +3,7 @@
 #include <nall/file.hpp>
 #include <nall/inode.hpp>
 #include <nall/intrinsics.hpp>
-#include <nall/sort.hpp>
+#include <nall/merge-sort.hpp>
 #include <nall/string.hpp>
 #include <nall/vector.hpp>
 
@@ -169,7 +169,7 @@ private:
     return list;
   }
 #else
-  inline auto directory_is_folder(DIR* dp, struct dirent* ep) -> bool {
+  inline auto directoryIsFolder(DIR* dp, struct dirent* ep) -> bool {
     if(ep->d_type == DT_DIR) return true;
     if(ep->d_type == DT_LNK || ep->d_type == DT_UNKNOWN) {
       //symbolic links must be resolved to determine type
@@ -218,7 +218,7 @@ private:
       while(ep = readdir(dp)) {
         if(!strcmp(ep->d_name, ".")) continue;
         if(!strcmp(ep->d_name, "..")) continue;
-        if(!directory_is_folder(dp, ep)) continue;
+        if(!directoryIsFolder(dp, ep)) continue;
         string name{ep->d_name};
         if(name.match(pattern)) list.append(std::move(name));
       }
@@ -239,7 +239,7 @@ private:
       while(ep = readdir(dp)) {
         if(!strcmp(ep->d_name, ".")) continue;
         if(!strcmp(ep->d_name, "..")) continue;
-        if(directory_is_folder(dp, ep)) continue;
+        if(directoryIsFolder(dp, ep)) continue;
         string name{ep->d_name};
         if(name.match(pattern)) list.append(std::move(name));
       }

nall/dl.hpp

@@ -9,7 +9,6 @@
 #include <nall/utility.hpp>
 
 #if defined(PLATFORM_WINDOWS)
-  #include <windows.h>
   #include <nall/windows/utf8.hpp>
 #else
   #include <dlfcn.h>

nall/encode/bwt.hpp

@@ -0,0 +1,49 @@
+#pragma once
+
+//burrows-wheeler transform
+
+#include <nall/suffix-array.hpp>
+
+namespace nall { namespace Encode {
+
+inline auto BWT(const void* data, uint size) -> vector<uint8_t> {
+  auto input = (const uint8_t*)data;
+  vector<uint8_t> output;
+  output.reserve(8 + 8 + size);
+  for(uint byte : range(8)) output.append(size >> byte * 8);
+  for(uint byte : range(8)) output.append(0x00);
+
+  auto suffixes = new int[size];
+//suffix_array(suffixes, input, size);
+  for(uint n : range(size)) suffixes[n] = n;
+  sort(suffixes, size, [&](int lhs, int rhs) -> bool {
+    uint l = size;
+    while(l--) {
+      auto x = input[lhs++];
+      auto y = input[rhs++];
+      if(x != y) return x - y < 0;
+      if(lhs >= size) lhs = 0;
+      if(rhs >= size) rhs = 0;
+    }
+    return 0;
+  });
+
+  uint64_t root = 0;
+  for(uint offset : range(size)) {
+    if(suffixes[offset] == 0) root = offset;
+    uint suffix = suffixes[offset];
+    if(suffix == 0) suffix = size;
+    output.append(input[--suffix]);
+  }
+
+  for(uint byte : range(8)) output[8 + byte] = root >> byte * 8;
+  delete[] suffixes;
+  return output;
+}
+
+template<typename T>
+inline auto BWT(const vector<T>& buffer) -> vector<uint8_t> {
+  return move(BWT(buffer.data(), buffer.size() * sizeof(T)));
+}
+
+}}

nall/encode/dictionary.hpp

@@ -0,0 +1,73 @@
+#pragma once
+
+#include <nall/suffix-array.hpp>
+
+namespace nall { namespace Encode {
+
+struct Dictionary {
+  inline Dictionary(const void* data, uint size, uint capacity = 0);
+  inline ~Dictionary();
+
+  inline auto operator[](uint index) const -> uint;
+  inline auto scan(uint offset = 0, uint size = 0) -> uint;
+  inline auto find(uint prefix, uint& lower, uint& upper) -> void;
+
+private:
+  const uint8_t* data = nullptr;
+  uint size = 0;
+
+  uint capacity = 0;
+  uint unique = 0;
+  uint* suffixes = nullptr;
+  uint* prefixes = nullptr;
+};
+
+Dictionary::Dictionary(const void* data, uint size, uint capacity) {
+  this->data = (const uint8_t*)data;
+  this->size = size;
+  this->capacity = capacity ? capacity : size;
+  suffixes = new uint[2 * this->capacity];
+  prefixes = &suffixes[this->capacity];
+}
+
+Dictionary::~Dictionary() {
+  delete[] suffixes;
+}
+
+auto Dictionary::operator[](uint index) const -> uint {
+  return suffixes[index];
+}
+
+auto Dictionary::scan(uint offset, uint size) -> uint {
+  size = min(size ? size : capacity, this->size - offset);
+  partial_suffix_array<32, 32>(suffixes, prefixes, data + offset, size, offset);
+  uint target = 0, source = 0;
+  while(source < size) {
+    prefixes[target] = prefixes[source];
+    suffixes[target] = suffixes[source];
+    uint length = 1;
+    while(source + length < size) {
+      if(suffixes[source + length] != suffixes[source] + length) break;
+      length++;
+    }
+    source += length;
+    target += 1;
+  }
+  return unique = target;
+}
+
+auto Dictionary::find(uint prefix, uint& lower, uint& upper) -> void {
+  uint l = 0, r = unique - 1;
+  while(l < r - 1) {
+    uint m = l + r >> 1;
+    prefixes[m] >= prefix ? r = m : l = m;
+  }
+  lower = l, r = unique - 1;
+  while(l < r - 1) {
+    uint m = l + r >> 1;
+    prefixes[m] <= prefix ? l = m : r = m;
+  }
+  upper = r;
+}
+
+}}

nall/encode/huffman.hpp

@@ -0,0 +1,90 @@
+#pragma once
+
+namespace nall { namespace Encode {
+
+inline auto Huffman(const void* data, uint size) -> vector<uint8_t> {
+  auto input = (const uint8_t*)data;
+  vector<uint8_t> output;
+  for(uint byte : range(8)) output.append(size >> byte * 8);
+
+  struct Node {
+    uint frequency = 0;
+    uint parent = 0;
+    uint lhs = 0;
+    uint rhs = 0;
+  };
+  array<Node[512]> nodes;
+  for(uint offset : range(size)) nodes[input[offset]].frequency++;
+
+  uint count = 0;
+  for(uint offset : range(511)) {
+    if(nodes[offset].frequency) count++;
+    else nodes[offset].parent = 511;
+  }
+
+  auto minimum = [&] {
+    uint frequency = ~0, minimum = 511;
+    for(uint index : range(511)) {
+      if(!nodes[index].parent && nodes[index].frequency && nodes[index].frequency < frequency) {
+        frequency = nodes[index].frequency;
+        minimum = index;
+      }
+    }
+    return minimum;
+  };
+
+  //group the least two frequently used nodes until only one node remains
+  uint index = 256;
+  for(uint remaining = max(2, count); remaining >= 2; remaining--) {
+    uint lhs = minimum();
+    nodes[lhs].parent = index;
+    uint rhs = minimum();
+    nodes[rhs].parent = index;
+    if(remaining == 2) index = nodes[lhs].parent = nodes[rhs].parent = 511;
+    nodes[index].lhs = lhs;
+    nodes[index].rhs = rhs;
+    nodes[index].parent = 0;
+    nodes[index].frequency = nodes[lhs].frequency + nodes[rhs].frequency;
+    index++;
+  }
+
+  uint byte = 0, bits = 0;
+  auto write = [&](bool bit) {
+    byte = byte << 1 | bit;
+    if(++bits == 8) output.append(byte), bits = 0;
+  };
+
+  //only the upper half of the table is needed for decompression
+  //the first 256 nodes are always treated as leaf nodes
+  for(uint offset : range(256)) {
+    for(uint index : reverse(range(9))) write(nodes[256 + offset].lhs >> index & 1);
+    for(uint index : reverse(range(9))) write(nodes[256 + offset].rhs >> index & 1);
+  }
+
+  for(uint offset : range(size)) {
+    uint node = input[offset], length = 0;
+    uint256_t sequence = 0;
+    //traversing the array produces the bitstream in reverse order
+    do {
+      uint parent = nodes[node].parent;
+      bool bit = nodes[nodes[node].parent].rhs == node;
+      sequence = sequence << 1 | bit;
+      length++;
+      node = parent;
+    } while(node != 511);
+    //output the generated bits in the correct order
+    for(uint index : range(length)) {
+      write(sequence >> index & 1);
+    }
+  }
+  while(bits) write(0);
+
+  return output;
+}
+
+template<typename T>
+inline auto Huffman(const vector<T>& buffer) -> vector<uint8_t> {
+  return move(Huffman(buffer.data(), buffer.size() * sizeof(T)));
+}
+
+}}

nall/encode/lzsa.hpp

@@ -0,0 +1,98 @@
+#pragma once
+
+#include <nall/suffix-array.hpp>
+#include <nall/encode/bwt.hpp>
+#include <nall/encode/huffman.hpp>
+#include <nall/encode/mtf.hpp>
+#include <nall/encode/rle.hpp>
+
+namespace nall { namespace Encode {
+
+inline auto LZSA(const void* data, uint64_t size) -> vector<uint8_t> {
+  vector<uint8_t> output;
+  for(uint byte : range(8)) output.append(size >> byte * 8);
+
+  auto input = (const uint8_t*)data;
+  uint index = 0;
+
+  auto buffers = new int[size * 4];
+  auto suffixes = &buffers[0 * size];
+  auto phi = &buffers[1 * size];
+  auto lengths = &buffers[2 * size];
+  auto offsets = &buffers[3 * size];
+  suffix_array(suffixes, input, size);
+  suffix_array_phi(phi, suffixes, size);
+  suffix_array_lps(lengths, offsets, phi, input, size);
+
+  vector<uint8_t> flags;
+  vector<uint8_t> literals;
+  vector<uint8_t> stringLengths;
+  vector<uint8_t> stringOffsets;
+
+  uint byte = 0, bits = 0;
+  auto flagWrite = [&](bool bit) {
+    byte = byte << 1 | bit;
+    if(++bits == 8) flags.append(byte), bits = 0;
+  };
+
+  auto literalWrite = [&](uint8_t literal) {
+    literals.append(literal);
+  };
+
+  auto lengthWrite = [&](uint64_t length) {
+         if(length < 1 <<  7) length = length << 1 |     0b1;
+    else if(length < 1 << 14) length = length << 2 |    0b10;
+    else if(length < 1 << 21) length = length << 3 |   0b100;
+    else if(length < 1 << 28) length = length << 4 |  0b1000;
+    else  /*length < 1 << 35*/length = length << 5 | 0b10000;
+    while(length) stringLengths.append(length), length >>= 8;
+  };
+
+  auto offsetWrite = [&](uint offset) {
+    stringOffsets.append(offset >>  0); if(index < 1 <<  8) return;
+    stringOffsets.append(offset >>  8); if(index < 1 << 16) return;
+    stringOffsets.append(offset >> 16); if(index < 1 << 24) return;
+    stringOffsets.append(offset >> 24);
+  };
+
+  while(index < size) {
+    int length = lengths[index];
+    int offset = offsets[index];
+
+    for(uint ahead = 1; ahead <= 2; ahead++) {
+      int aheadLength = lengths[index + ahead];
+      int aheadOffset = offsets[index + ahead];
+      if(aheadLength > length && aheadOffset >= 0) {
+        length = 0;
+        break;
+      }
+    }
+
+    if(length < 6 || offset < 0) {
+      flagWrite(0);
+      literalWrite(input[index++]);
+    } else {
+      flagWrite(1);
+      lengthWrite(length - 6);
+      offsetWrite(index - offset);
+      index += length;
+    }
+  }
+  while(bits) flagWrite(0);
+
+  auto save = [&](const vector<uint8_t>& buffer) {
+    for(uint byte : range(8)) output.append(buffer.size() >> byte * 8);
+    output.append(buffer);
+  };
+
+  save(Encode::Huffman(flags));
+  save(Encode::Huffman(literals));
+//save(Encode::Huffman(Encode::BWT(literals)));
+  save(Encode::Huffman(stringLengths));
+  save(Encode::Huffman(stringOffsets));
+
+  delete[] buffers;
+  return output;
+}
+
+}}

nall/encode/lzss.hpp

@@ -0,0 +1,76 @@
+#pragma once
+
+#include <nall/encode/dictionary.hpp>
+
+namespace nall { namespace Encode {
+
+inline auto LZSS(const void* data, uint64_t size, uint windowBits = 16, uint lengthBits = 8) -> vector<uint8_t> {
+  vector<uint8_t> output;
+  for(uint byte : range(8)) output.append(size >> byte * 8);
+  output.append(windowBits);
+  output.append(lengthBits);
+
+  const uint lengthExtends = 4 + (1 << lengthBits) - 1;
+  const uint lengthMaximum = lengthExtends + 255;
+  const uint windowMaximum = 1 << windowBits;
+  const uint windowRange = windowMaximum + lengthMaximum;
+
+  auto input = (const uint8_t*)data;
+  auto read = [&](uint address) -> uint {
+    if(address + 3 > size) return 0;
+    return input[address + 0] << 24 | input[address + 1] << 16 | input[address + 2] << 8 | input[address + 3] << 0;
+  };
+
+  Dictionary dictionary(data, size, 2 * windowRange);
+  dictionary.scan();
+
+  for(uint offset = 0, base = 0, flags = 0, bit = 7; offset < size;) {
+    if(offset - base >= 2 * windowRange) {
+      dictionary.scan(base = offset - windowRange);
+    }
+
+    uint prefix = read(offset), lower, upper;
+    dictionary.find(prefix, lower, upper);
+
+    uint lengthLongest = 0, windowLongest = 0;
+    for(uint index = lower; index <= upper; index++) {
+      int window = (int)offset - (int)dictionary[index];
+      if(window <= 0) continue;
+      window = min(window, windowMaximum);
+
+      uint length = 0;
+      do {
+        if(offset + length >= size) break;
+        if(input[offset + length] != input[offset + length - window]) break;
+      } while(++length < lengthMaximum);
+
+      if(length > lengthLongest) {
+        lengthLongest = length;
+        windowLongest = window;
+        if(length == lengthMaximum) break;
+      }
+    }
+
+    if(++bit == 8) {
+      flags = output.size();
+      output.append(bit = 0);
+    }
+
+    if(lengthLongest < 4) {
+      output.append(input[offset++]);
+    } else {
+      output[flags] |= 1 << bit;
+      offset += lengthLongest;
+
+      uint encoding = min(lengthLongest, lengthExtends) - 4 << windowBits | windowLongest - 1;
+      output.append(encoding >>  0);
+      output.append(encoding >>  8);
+      output.append(encoding >> 16);
+      if(lengthLongest >= lengthExtends) output.append(lengthLongest - lengthExtends);
+    }
+  }
+
+  return output;
+}
+
+}}

nall/encode/mtf.hpp

@@ -0,0 +1,36 @@
+#pragma once
+
+//move to front
+
+namespace nall { namespace Encode {
+
+inline auto MTF(const void* data, uint size) -> vector<uint8_t> {
+  auto input = (const uint8_t*)data;
+  vector<uint8_t> output;
+  output.resize(size);
+
+  uint8_t order[256];
+  for(uint n : range(256)) order[n] = n;
+
+  for(uint offset = 0; offset < size; offset++) {
+    auto data = input[offset];
+    for(uint index = 0; index < 256; index++) {
+      uint value = order[index];
+      if(value == data) {
+        output[offset] = index;
+        memory::move(&order[1], &order[0], index);
+        order[0] = value;
+        break;
+      }
+    }
+  }
+
+  return output;
+}
+
+template<typename T>
+inline auto MTF(const vector<T>& buffer) -> vector<uint8_t> {
+  return move(MTF(buffer.data(), buffer.size() * sizeof(T)));
+}
+
+}}

nall/encode/rle.hpp

@@ -2,49 +2,62 @@
 
 namespace nall { namespace Encode {
 
-template<typename T> inline auto RLE(const void* data_, uint size, uint minimum = 0) -> vector<uint8_t> {
+template<uint S = 1, uint M = 4 / S>  //S = word size; M = match length
-  if(!minimum) minimum = max(1, 4 / sizeof(T));
+inline auto RLE(const void* data, uint64_t size) -> vector<uint8_t> {
-  vector<uint8_t> result;
+  vector<uint8_t> output;
+  for(uint byte : range(8)) output.append(size >> byte * 8);
 
-  auto data = (const T*)data_;
+  auto input = (const uint8_t*)data;
   uint base = 0;
   uint skip = 0;
 
-  for(uint byte : range(sizeof(uint))) result.append(size * sizeof(T) >> byte * 8);
+  auto load = [&](uint offset) -> uint8_t {
-
+    if(offset >= size) return 0x00;
-  auto read = [&](uint offset) -> T {
+    return input[offset];
-    if(offset >= size) return {};
-    return data[offset];
   };
 
-  auto write = [&](T value) -> void {
+  auto read = [&](uint offset) -> uint64_t {
-    for(uint byte : range(sizeof(T))) result.append(value >> byte * 8);
+    uint64_t value = 0;
+    for(uint byte : range(S)) value |= load(offset + byte) << byte * 8;
+    return value;
   };
 
-  auto flush = [&]() -> void {
+  auto write = [&](uint64_t value) -> void {
-    result.append(skip - 1);
+    for(uint byte : range(S)) output.append(value >> byte * 8);
-    do { write(read(base++)); } while(--skip);
   };
 
-  while(base + skip < size) {
+  auto flush = [&] {
+    output.append(skip - 1);
+    do {
+      write(read(base));
+      base += S;
+    } while(--skip);
+  };
+
+  while(base + S * skip < size) {
     uint same = 1;
-    for(uint offset = base + skip + 1; offset < size; offset++) {
+    for(uint offset = base + S * (skip + 1); offset < size; offset += S) {
-      if(read(offset) != read(base + skip)) break;
+      if(read(offset) != read(base + S * skip)) break;
-      if(++same == 127 + minimum) break;
+      if(++same == 127 + M) break;
     }
 
-    if(same < minimum) {
+    if(same < M) {
       if(++skip == 128) flush();
     } else {
       if(skip) flush();
-      result.append(128 | same - minimum);
+      output.append(128 | same - M);
       write(read(base));
-      base += same;
+      base += S * same;
     }
   }
   if(skip) flush();
 
-  return result;
+  return output;
+}
+
+template<uint S = 1, uint M = 4 / S, typename T>
+inline auto RLE(const vector<T>& buffer) -> vector<uint8_t> {
+  return move(RLE<S, M>(buffer.data(), buffer.size() * sizeof(T)));
 }
 
 }}

nall/file.hpp

@@ -72,17 +72,18 @@ struct file : inode, varint {
     return S_ISREG(data.st_mode) ? data.st_size : 0u;
   }
 
-  static auto read(const string& filename) -> vector<uint8_t> {
+  static auto read(const string& filename, uint reserve = 0) -> vector<uint8_t> {
     vector<uint8_t> memory;
     file fp;
     if(fp.open(filename, mode::read)) {
+      memory.reserve(fp.size() + reserve);
       memory.resize(fp.size());
       fp.read(memory.data(), memory.size());
     }
     return memory;
   }
 
-  static auto read(const string& filename, uint8_t* data, uint size) -> bool {
+  static auto read(const string& filename, void* data, uint size) -> bool {
     file fp;
     if(fp.open(filename, mode::read) == false) return false;
     fp.read(data, size);
@@ -91,14 +92,14 @@ struct file : inode, varint {
   }
 
   static auto write(const string& filename, const string& text) -> bool {
-    return write(filename, (const uint8_t*)text.data(), text.size());
+    return write(filename, text.data(), text.size());
   }
 
   static auto write(const string& filename, const vector<uint8_t>& buffer) -> bool {
     return write(filename, buffer.data(), buffer.size());
   }
 
-  static auto write(const string& filename, const uint8_t* data, uint size) -> bool {
+  static auto write(const string& filename, const void* data, uint size) -> bool {
     file fp;
     if(fp.open(filename, mode::write) == false) return false;
     fp.write(data, size);
@@ -151,8 +152,9 @@ struct file : inode, varint {
     return result;
   }
 
-  auto read(uint8_t* buffer, uint length) -> void {
+  auto read(void* data, uint size) -> void {
-    while(length--) *buffer++ = read();
+    auto output = (uint8_t*)data;
+    while(size--) *output++ = read();
   }
 
   auto write(uint8_t data) -> void {
@@ -181,8 +183,9 @@ struct file : inode, varint {
     for(auto byte : s) write(byte);
   }
 
-  auto write(const uint8_t* buffer, uint length) -> void {
+  auto write(const void* data, uint size) -> void {
-    while(length--) write(*buffer++);
+    auto input = (const uint8_t*)data;
+    while(size--) write(*input++);
   }
 
   template<typename... Args> auto print(Args... args) -> void {

nall/filemap.hpp

@@ -2,12 +2,11 @@
 
 #include <nall/file.hpp>
 #include <nall/stdint.hpp>
-#include <nall/windows/utf8.hpp>
 
 #include <stdio.h>
 #include <stdlib.h>
-#if defined(_WIN32)
+#if defined(PLATFORM_WINDOWS)
-  #include <windows.h>
+  #include <nall/windows/utf8.hpp>
 #else
   #include <fcntl.h>
   #include <unistd.h>

nall/literals.hpp

@@ -0,0 +1,25 @@
+#pragma once
+
+//note: gcc 4.9 does not support user-defined literals with arguments other than const char*
+//once nall increases the minimum required GCC version, the use of nall/atoi.hpp can beremoved
+
+#include <nall/atoi.hpp>
+
+namespace nall { namespace Literal {
+
+struct Capacity { uint value; };
+struct Size { uint value; };
+
+}}
+
+namespace nall {
+
+constexpr inline auto operator"" _capacity(const char* s) -> Literal::Capacity {
+  return {(uint)toNatural(s)};
+}
+
+constexpr inline auto operator"" _size(const char* s) -> Literal::Size {
+  return {(uint)toNatural(s)};
+}
+
+}

nall/locale.hpp

@@ -29,10 +29,10 @@ struct Locale {
     return result;
   }
 
-  auto select(string language) -> bool {
+  auto select(string option) -> bool {
     selected.reset();
     for(auto& dictionary : dictionaries) {
-      if(dictionary.language == language) {
+      if(option == Location::prefix(dictionary.location) || option == dictionary.language) {
         selected = dictionary;
         return true;
       }

nall/main.hpp

@@ -14,7 +14,7 @@ namespace nall {
     _setmode(_fileno(stdin), O_BINARY);
     _setmode(_fileno(stdout), O_BINARY);
     _setmode(_fileno(stderr), O_BINARY);
-    utf8_args(argc, argv);
+    utf8_arguments(argc, argv);
     #endif
 
     vector<string> arguments;

nall/matrix.hpp

@@ -1,10 +1,12 @@
 #pragma once
 
-namespace nall {
+namespace nall { namespace Matrix {
 
-namespace Matrix {
+template<typename T> inline auto Multiply(
-
+T* output,
-template<typename T> inline auto Multiply(T* output, const T* xdata, uint xrows, uint xcols, const T* ydata, uint yrows, uint ycols) -> void {
+const T* xdata, uint xrows, uint xcols,
+const T* ydata, uint yrows, uint ycols
+) -> void {
   if(xcols != yrows) return;
 
   for(uint y : range(xrows)) {
@@ -18,13 +20,14 @@ template<typename T> inline auto Multiply(T* output, const T* xdata, uint xrows,
   }
 }
 
-template<typename T> inline auto Multiply(const T* xdata, uint xrows, uint xcols, const T* ydata, uint yrows, uint ycols) -> vector<T> {
+template<typename T> inline auto Multiply(
+const T* xdata, uint xrows, uint xcols,
+const T* ydata, uint yrows, uint ycols
+) -> vector<T> {
   vector<T> output;
   output.resize(xrows * ycols);
   Multiply(output.data(), xdata, xrows, xcols, ydata, yrows, ycols);
   return output;
 }
 
-}
+}}
-
-}

nall/maybe.hpp

@@ -4,6 +4,7 @@ namespace nall {
 
 struct nothing_t {};
 static nothing_t nothing;
+struct else_t {};
 
 template<typename T>
 struct maybe {

nall/memory.hpp

@@ -61,15 +61,16 @@ auto free(void* target) -> void {
 }
 
 template<typename T> auto compare(const void* target, uint capacity, const void* source, uint size) -> int {
-  auto t = (int8_t*)target;
+  auto t = (uint8_t*)target;
-  auto s = (int8_t*)source;
+  auto s = (uint8_t*)source;
   auto l = min(capacity, size) * sizeof(T);
   while(l--) {
     auto x = *t++;
     auto y = *s++;
     if(x != y) return x - y;
   }
-  return 0;
+  if(capacity == size) return 0;
+  return -(capacity < size);
 }
 
 template<typename T> auto compare(const void* target, const void* source, uint size) -> int {
@@ -77,8 +78,8 @@ template<typename T> auto compare(const void* target, const void* source, uint s
 }
 
 template<typename T> auto icompare(const void* target, uint capacity, const void* source, uint size) -> int {
-  auto t = (int8_t*)target;
+  auto t = (uint8_t*)target;
-  auto s = (int8_t*)source;
+  auto s = (uint8_t*)source;
   auto l = min(capacity, size) * sizeof(T);
   while(l--) {
     auto x = *t++;
@@ -87,7 +88,7 @@ template<typename T> auto icompare(const void* target, uint capacity, const void
     if(y - 'A' < 26) y += 32;
     if(x != y) return x - y;
   }
-  return 0;
+  return -(capacity < size);
 }
 
 template<typename T> auto icompare(const void* target, const void* source, uint size) -> int {

nall/merge-sort.hpp

@@ -13,33 +13,37 @@
 //note: merge sort was chosen over quick sort, because:
 //* it is a stable sort
 //* it lacks O(n^2) worst-case overhead
+//* it usually runs faster than quick sort anyway
 
-#define NALL_SORT_INSERTION
+//note: insertion sort is generally more performant than selection sort
-//#define NALL_SORT_SELECTION
+#define NALL_MERGE_SORT_INSERTION
+//#define NALL_MERGE_SORT_SELECTION
 
 namespace nall {
 
 template<typename T, typename Comparator> auto sort(T list[], uint size, const Comparator& lessthan) -> void {
   if(size <= 1) return;  //nothing to sort
 
-  //use insertion sort to quickly sort smaller blocks
+  //sort smaller blocks using an O(n^2) algorithm (which for small sizes, increases performance)
   if(size < 64) {
-    #if defined(NALL_SORT_INSERTION)
+    //insertion sort requires a copy (via move construction)
+    #if defined(NALL_MERGE_SORT_INSERTION)
     for(int i = 1, j; i < size; i++) {
-      T copy = std::move(list[i]);
+      T copy(move(list[i]));
       for(j = i - 1; j >= 0; j--) {
         if(!lessthan(copy, list[j])) break;
-        list[j + 1] = std::move(list[j]);
+        list[j + 1] = move(list[j]);
       }
-      list[j + 1] = std::move(copy);
+      list[j + 1] = move(copy);
     }
-    #elif defined(NALL_SORT_SELECTION)
+    //selection sort requires a swap
+    #elif defined(NALL_MERGE_SORT_SELECTION)
     for(uint i = 0; i < size; i++) {
       uint min = i;
       for(uint j = i + 1; j < size; j++) {
         if(lessthan(list[j], list[min])) min = j;
       }
-      if(min != i) std::swap(list[i], list[min]);
+      if(min != i) swap(list[i], list[min]);
     }
     #endif
     return;
@@ -51,20 +55,24 @@ template<typename T, typename Comparator> auto sort(T list[], uint size, const C
   sort(list + middle, size - middle, lessthan);
 
   //left and right are sorted here; perform merge sort
-  T* buffer = new T[size];
+  //use placement new to avoid needing T to be default-constructable
+  auto buffer = memory::allocate<T>(size);
   uint offset = 0, left = 0, right = middle;
   while(left < middle && right < size) {
     if(!lessthan(list[right], list[left])) {
-      buffer[offset++] = std::move(list[left++]);
+      new(buffer + offset++) T(move(list[left++]));
     } else {
-      buffer[offset++] = std::move(list[right++]);
+      new(buffer + offset++) T(move(list[right++]));
     }
   }
-  while(left < middle) buffer[offset++] = std::move(list[left++]);
+  while(left < middle) new(buffer + offset++) T(move(list[left++]));
-  while(right < size)  buffer[offset++] = std::move(list[right++]);
+  while(right < size ) new(buffer + offset++) T(move(list[right++]));
 
-  for(uint i = 0; i < size; i++) list[i] = std::move(buffer[i]);
+  for(uint i = 0; i < size; i++) {
-  delete[] buffer;
+    list[i] = move(buffer[i]);
+    buffer[i].~T();
+  }
+  memory::free(buffer);
 }
 
 template<typename T> auto sort(T list[], uint size) -> void {

nall/nall.hpp

@@ -41,6 +41,7 @@
 #include <nall/matrix.hpp>
 #include <nall/maybe.hpp>
 #include <nall/memory.hpp>
+#include <nall/merge-sort.hpp>
 #include <nall/path.hpp>
 #include <nall/pointer.hpp>
 #include <nall/primitives.hpp>
@@ -53,7 +54,6 @@
 #include <nall/set.hpp>
 #include <nall/shared-pointer.hpp>
 #include <nall/simd.hpp>
-#include <nall/sort.hpp>
 #include <nall/stdint.hpp>
 #include <nall/string.hpp>
 #include <nall/thread.hpp>

nall/platform.hpp

@@ -8,11 +8,16 @@ namespace Math {
 }
 
 #if defined(PLATFORM_WINDOWS)
-  //minimum version needed for _wstat64, AI_ADDRCONFIG, etc
+  #include <nall/windows/guard.hpp>
-  #undef  _WIN32_WINNT
+  #include <winsock2.h>
-  #define _WIN32_WINNT 0x0601
+  #include <ws2tcpip.h>
-  #undef  __MSVCRT_VERSION__
+  #include <windows.h>
-  #define __MSVCRT_VERSION__ _WIN32_WINNT
+  #include <direct.h>
+  #include <io.h>
+  #include <wchar.h>
+  #include <shlobj.h>
+  #include <shellapi.h>
+  #include <nall/windows/guard.hpp>
   #include <nall/windows/utf8.hpp>
 #endif
 
@@ -35,14 +40,7 @@ namespace Math {
 #include <sys/types.h>
 #include <sys/stat.h>
 
-#if defined(PLATFORM_WINDOWS)
+#if !defined(PLATFORM_WINDOWS)
-  #include <io.h>
-  #include <direct.h>
-  #include <shlobj.h>
-  #include <wchar.h>
-  #include <winsock2.h>
-  #include <ws2tcpip.h>
-#else
   #include <dlfcn.h>
   #include <unistd.h>
   #include <pwd.h>

nall/smtp.hpp

@@ -4,7 +4,7 @@
 #include <nall/stdint.hpp>
 #include <nall/string.hpp>
 
-#if !defined(_WIN32)
+#if !defined(PLATFORM_WINDOWS)
   #include <sys/types.h>
   #include <sys/socket.h>
   #include <netinet/in.h>
@@ -12,7 +12,6 @@
 #else
   #include <winsock2.h>
   #include <ws2tcpip.h>
-  #include <windows.h>
 #endif
 
 namespace nall {

nall/string.hpp

@@ -56,6 +56,9 @@ protected:
   mutable int _size;
 };
 
+//adaptive (SSO + COW) is by far the best choice, the others exist solely to:
+//1) demonstrate the performance benefit of combining SSO + COW
+//2) rule out allocator bugs by trying different allocators when needed
 #define NALL_STRING_ALLOCATOR_ADAPTIVE
 //#define NALL_STRING_ALLOCATOR_COPY_ON_WRITE
 //#define NALL_STRING_ALLOCATOR_SMALL_STRING_OPTIMIZATION
@@ -128,6 +131,8 @@ public:
   inline string();
   template<typename T = char> inline auto get() -> T*;
   template<typename T = char> inline auto data() const -> const T*;
+  template<typename T = char> auto size() const -> uint { return _size / sizeof(T); }
+  template<typename T = char> auto capacity() const -> uint { return _capacity / sizeof(T); }
   inline auto reset() -> type&;
   inline auto reserve(uint) -> type&;
   inline auto resize(uint) -> type&;
@@ -142,9 +147,6 @@ public:
   explicit operator bool() const { return _size; }
   operator const char*() const { return (const char*)data(); }
 
-  auto size() const -> uint { return _size; }
-  auto capacity() const -> uint { return _capacity; }
-
   auto operator==(const string& source) const -> bool {
     return size() == source.size() && memory::compare(data(), source.data(), size()) == 0;
   }
@@ -271,6 +273,7 @@ public:
   inline auto remove(uint offset, uint length) -> type&;
   inline auto reverse() -> type&;
   inline auto size(int length, char fill = ' ') -> type&;
+  inline auto slice(int offset = 0, int length = -1) -> string;
 };
 
 template<> struct vector<string> : vector_base<string> {
@@ -286,7 +289,7 @@ template<> struct vector<string> : vector_base<string> {
   inline auto operator=(vector& source) -> type& { return vector_base::operator=(source), *this; }
   inline auto operator=(vector&& source) -> type& { return vector_base::operator=(move(source)), *this; }
 
-  //list.hpp
+  //vector.hpp
   template<typename... P> inline auto append(const string&, P&&...) -> type&;
   inline auto append() -> type&;
 

nall/string/allocator/copy-on-write.hpp

@@ -5,17 +5,19 @@ namespace nall {
 string::string() : _data(nullptr), _refs(nullptr), _capacity(0), _size(0) {
 }
 
-auto string::get() -> char* {
+template<typename T>
+auto string::get() -> T* {
   static char _null[] = "";
-  if(!_data) return _null;
+  if(!_data) return (T*)_null;
   if(*_refs > 1) _data = _copy();  //make unique for write operations
-  return _data;
+  return (T*)_data;
 }
 
-auto string::data() const -> const char* {
+template<typename T>
+auto string::data() const -> const T* {
   static const char _null[] = "";
-  if(!_data) return _null;
+  if(!_data) return (const T*)_null;
-  return _data;
+  return (const T*)_data;
 }
 
 auto string::reset() -> type& {

nall/string/allocator/small-string-optimization.hpp

@@ -26,14 +26,16 @@ string::string() {
   _size = 0;
 }
 
-auto string::get() -> char* {
+template<typename T>
-  if(_capacity < SSO) return _text;
+auto string::get() -> T* {
-  return _data;
+  if(_capacity < SSO) return (T*)_text;
+  return (T*)_data;
 }
 
-auto string::data() const -> const char* {
+template<typename T>
-  if(_capacity < SSO) return _text;
+auto string::data() const -> const T* {
-  return _data;
+  if(_capacity < SSO) return (const T*)_text;
+  return (const T*)_data;
 }
 
 auto string::reset() -> type& {

nall/string/allocator/vector.hpp

@@ -19,14 +19,16 @@ cons:
 
 namespace nall {
 
-auto string::get() -> char* {
+template<typename T>
+auto string::get() -> T* {
   if(_capacity == 0) reserve(1);
-  return _data;
+  return (T*)_data;
 }
 
-auto string::data() const -> const char* {
+template<typename T>
-  if(_capacity == 0) return "";
+auto string::data() const -> const T* {
-  return _data;
+  if(_capacity == 0) return (const T*)"";
+  return (const T*)_data;
 }
 
 auto string::reset() -> type& {

nall/string/utility.hpp

@@ -95,6 +95,10 @@ auto slice(view<string> self, int offset, int length) -> string {
   return result;
 }
 
+auto string::slice(int offset, int length) -> string {
+  return nall::slice(*this, offset, length);
+}
+
 template<typename T> auto fromInteger(char* result, T value) -> char* {
   bool negative = value < 0;
   if(!negative) value = -value;  //negate positive integers to support eg INT_MIN

nall/suffix-array.hpp

@@ -0,0 +1,202 @@
+#pragma once
+
+#include <nall/counting-sort.hpp>
+#include <nall/div-suf-sort.hpp>
+#include <nall/range.hpp>
+
+namespace nall {
+
+/*
+
+input:
+  data = "acaaacatat"
+
+suffix_array:
+  suffixes = [2, 3, 0, 4, 8, 6, 1, 5, 9, 7]
+  2 "aaaacatat"
+  3 "aacatat"
+  0 "acaaacatat"
+  4 "acatat"
+  8 "at"
+  6 "atat"
+  1 "caaacatat"
+  5 "catat"
+  9 "t"
+  7 "tat"
+
+suffix_array_inv:
+  inverted = [2, 6, 0, 1, 3, 7, 5, 9, 4, 8]
+
+suffix_array_lcp:
+  prefixes = [-, 2, 1, 3, 1, 2, 0, 2, 0, 1]
+  "aaaacatat"  -
+  "aacatat"    2 "aa"
+  "acaaacatat" 1 "a"
+  "acatat"     3 "aca"
+  "at"         1 "a"
+  "atat"       2 "at"
+  "caaacatat"  0
+  "catat"      2 "ca"
+  "t"          0
+  "tat"        1 "t"
+
+suffix_array_phi:
+  phi = [3, 6, -, 2, 0, 1, 8, 9, 4, 5]
+
+suffix_array_lps:
+  lengths = [-, 0, 1, 2, 3, 2, 1, 0, 2, 1]
+  offsets = [-, -, 0, 2, 0, 1, 4, -, 6, 7]
+  "acaaacatat" (-,-)
+   "caaacatat" (0,-)
+    "aaacatat" (1,0) at 0, match "a"
+     "aacatat" (2,2) at 2, match "aa"
+      "acatat" (3,0) at 0, match "aca"
+       "catat" (2,1) at 1, match "ca"
+        "atat" (1,4) at 4, match "a" (not 0)
+         "tat" (0,-)
+          "at" (2,6) at 6, match "at"
+           "t" (1,7) at 7, match "a" (not 0)
+
+*/
+
+// O(n log n)
+inline auto suffix_array(int* suffixes, const uint8_t* data, int size) -> void {
+  for(int n : range(size)) suffixes[n] = n;
+  #if 1
+  div_suf_sort(suffixes, data, size);
+  #else
+  sort(suffixes, size, [&](int lhs, int rhs) -> bool {
+    return memory::compare(data + lhs, size - lhs, data + rhs, size - rhs) < 0;
+  });
+  #endif
+}
+
+// inverse
+// O(n)
+inline auto suffix_array_inv(int* inverted, const int* suffixes, int size) -> void {
+  for(int i : range(size)) inverted[suffixes[i]] = i;
+}
+
+// longest common prefix
+// O(n)
+inline auto suffix_array_lcp(int* prefixes, const int* inverted, const int* suffixes, const uint8_t* data, int size) -> void {
+  prefixes[0] = -1;
+  for(int i = 0, l = 0; i < size; i++) {
+    if(inverted[i] == size - 1) { l = 0; continue; }
+    int j = suffixes[inverted[i] + 1];
+    while(i + l < size && j + l < size && data[i + l] == data[j + l]) l++;
+    prefixes[1 + inverted[i]] = l;
+    if(l) l--;
+  }
+}
+
+// O(n)
+inline auto suffix_array_phi(int* phi, const int* suffixes, int size) -> void {
+  phi[suffixes[0]] = -1;
+  for(int i : range(1, size)) phi[suffixes[i]] = suffixes[i - 1];
+}
+
+// longest previous string (longest previous factor)
+// O(n)
+inline auto suffix_array_lps(int* lengths, int* offsets, const int* phi, const uint8_t* data, int size) -> void {
+  function<void (int, int, int)> sop = [&](int i, int l, int j) -> void {
+    if(lengths[i] < 0) {
+      lengths[i] = l;
+      offsets[i] = j;
+    } else {
+      if(lengths[i] < l) {
+        if(offsets[i] > j) {
+          sop(offsets[i], lengths[i], j);
+        } else {
+          sop(j, lengths[i], offsets[i]);
+        }
+        lengths[i] = l;
+        offsets[i] = j;
+      } else {
+        if(offsets[i] > j) {
+          sop(offsets[i], l, j);
+        } else {
+          sop(j, l, offsets[i]);
+        }
+      }
+    }
+  };
+
+  int l = 0;
+  for(int i : range(size)) lengths[i] = -1;
+  for(int i : range(size)) {
+    int j = phi[i];
+    while(i + l < size && j + l < size && data[i + l] == data[j + l]) l++;
+    if(i > j) {
+      sop(i, l, j);
+    } else {
+      sop(j, l, i);
+    }
+    if(l) l--;
+  }
+  lengths[0] = -1;
+}
+
+//partial_suffix_array computes a suffix array in O(n) time by only sorting by SuffixBits into each prefix
+//this is much faster than a proper suffix_array, but at the penalty of not being 100% sorted
+//thus, least common prefixes cannot readily be used with this; deduplication is suggested for binary searching
+//suffixes[] = (offsets) list of indexes into data[] in sorted order
+//prefixes[] = (values) sorted list of data[]
+
+template<uint SuffixBits, uint PrefixBits>
+inline auto partial_suffix_array(uint* suffixes, uint* prefixes, const void* data, uint64_t size, uint offset = 0) -> void;
+
+template<>
+inline auto partial_suffix_array<32, 24>(uint* suffixes, uint* prefixes, const void* data, uint64_t size, uint offset) -> void {
+  auto input = (const uint8_t*)data;
+  if(size == 0 || !data || !suffixes || !prefixes) return;
+  if(size == 1) return suffixes[0] = offset << 16, prefixes[0] = input[0], void();
+
+  auto elements = new uint64_t[2 * size], lhs = &elements[0], rhs = &elements[size];
+  for(uint index : range(size - 2)) {
+    elements[index] = index | uint64_t(input[0] << 16 | input[1] << 8 | input[2] << 0) << 32, input++;
+  }
+  elements[size - 2] = size - 2 | uint64_t(input[0] << 16 | input[1] << 8) << 32, input++;
+  elements[size - 1] = size - 1 | uint64_t(input[0] << 16) << 32;
+
+  counting_sort<12, 32>(rhs, lhs, size);
+  counting_sort<12, 44>(lhs, rhs, size);
+  for(uint n : range(size)) {
+    suffixes[n] = (uint32_t)lhs[n] + offset;
+    prefixes[n] = lhs[n] >> 32;
+  }
+
+  delete[] elements;
+}
+
+template<>
+inline auto partial_suffix_array<32, 32>(uint* suffixes, uint* prefixes, const void* data, uint64_t size, uint offset) -> void {
+  auto input = (const uint8_t*)data;
+  if(size == 0 || !data || !suffixes || !prefixes) return;
+  if(size == 1) return suffixes[0] = offset, prefixes[0] = input[0], void();
+  if(size == 2) {
+    suffixes[0] = offset, suffixes[1] = 1 + offset;
+    prefixes[0] = input[0] << 24 | input[1] << 16, prefixes[1] = input[1] << 24;
+    if(input[0] >= input[1]) swap(suffixes[0], suffixes[1]), swap(prefixes[0], prefixes[1]);
+    return;
+  }
+
+  auto elements = new uint64_t[2 * size], lhs = &elements[0], rhs = &elements[size];
+  for(uint index : range(size - 3)) {
+    elements[index] = index | uint64_t(input[0] << 24 | input[1] << 16 | input[2] << 8 |  input[3] << 0) << 32, input++;
+  }
+  elements[size - 3] = size - 3 | uint64_t(input[0] << 24 | input[1] << 16 | input[2] << 8) << 32, input++;
+  elements[size - 2] = size - 2 | uint64_t(input[0] << 24 | input[1] << 16) << 32, input++;
+  elements[size - 1] = size - 1 | uint64_t(input[0] << 24) << 32;
+
+  counting_sort<16, 32>(rhs, lhs, size);
+  counting_sort<16, 48>(lhs, rhs, size);
+  for(uint n : range(size)) {
+    suffixes[n] = (uint32_t)lhs[n] + offset;
+    prefixes[n] = lhs[n] >> 32;
+  }
+
+  delete[] elements;
+}
+
+}

nall/vector.hpp

@@ -5,10 +5,11 @@
 #include <nall/bit.hpp>
 #include <nall/function.hpp>
 #include <nall/iterator.hpp>
+#include <nall/literals.hpp>
 #include <nall/maybe.hpp>
 #include <nall/memory.hpp>
+#include <nall/merge-sort.hpp>
 #include <nall/range.hpp>
-#include <nall/sort.hpp>
 #include <nall/traits.hpp>
 
 namespace nall {
@@ -24,16 +25,18 @@ struct vector_base {
 
   //core.hpp
   vector_base() = default;
+  vector_base(Literal::Capacity capacity);
+  vector_base(Literal::Size size);
   vector_base(const initializer_list<T>& values);
   vector_base(const type& source);
   vector_base(type&& source);
   ~vector_base();
 
   explicit operator bool() const;
-  auto capacity() const -> uint;
+  template<typename Cast = T> auto capacity() const -> uint;
-  auto size() const -> uint;
+  template<typename Cast = T> auto size() const -> uint;
-  auto data() -> T*;
+  template<typename Cast = T> auto data(uint offset = 0) -> Cast*;
-  auto data() const -> const T*;
+  template<typename Cast = T> auto data(uint offset = 0) const -> const Cast*;
 
   //assign.hpp
   auto operator=(const type& source) -> type&;
@@ -112,6 +115,7 @@ struct vector_base {
   auto sort(const function<bool (const T& lhs, const T& rhs)>& comparator = [](auto& lhs, auto& rhs) { return lhs < rhs; }) -> void;
   auto find(const function<bool (const T& lhs)>& comparator) -> maybe<uint>;
   auto find(const T& value) const -> maybe<uint>;
+  auto findSorted(const T& value) const -> maybe<uint>;
   auto foreach(const function<void (const T&)>& callback) -> void;
   auto foreach(const function<void (uint, const T&)>& callback) -> void;
 

nall/vector/core.hpp

@@ -2,6 +2,14 @@
 
 namespace nall {
 
+template<typename T> vector<T>::vector(Literal::Capacity capacity) {
+  reserve(capacity.value);
+}
+
+template<typename T> vector<T>::vector(Literal::Size size) {
+  resize(size.value);
+}
+
 template<typename T> vector<T>::vector(const initializer_list<T>& values) {
   reserveRight(values.size());
   for(auto& value : values) append(value);
@@ -23,20 +31,20 @@ template<typename T> vector<T>::operator bool() const {
   return _size;
 }
 
-template<typename T> auto vector<T>::capacity() const -> uint {
+template<typename T> template<typename Cast> auto vector<T>::capacity() const -> uint {
-  return _left + _size + _right;
+  return (_left + _size + _right) * sizeof(T) / sizeof(Cast);
 }
 
-template<typename T> auto vector<T>::size() const -> uint {
+template<typename T> template<typename Cast> auto vector<T>::size() const -> uint {
-  return _size;
+  return _size * sizeof(T) / sizeof(Cast);
 }
 
-template<typename T> auto vector<T>::data() -> T* {
+template<typename T> template<typename Cast> auto vector<T>::data(uint offset) -> Cast* {
-  return _pool;
+  return (Cast*)_pool + offset;
 }
 
-template<typename T> auto vector<T>::data() const -> const T* {
+template<typename T> template<typename Cast> auto vector<T>::data(uint offset) const -> const Cast* {
-  return _pool;
+  return (const Cast*)_pool + offset;
 }
 
 }

nall/vector/utility.hpp

@@ -16,6 +16,16 @@ template<typename T> auto vector<T>::find(const T& value) const -> maybe<uint> {
   return nothing;
 }
 
+template<typename T> auto vector<T>::findSorted(const T& value) const -> maybe<uint> {
+  int l = 0, r = size() - 1;
+  while(l <= r) {
+    int m = l + (r - l >> 1);
+    if(value == _pool[m]) return m;
+    value < _pool[m] ? r = m - 1 : l = m + 1;
+  }
+  return nothing;
+}
+
 template<typename T> auto vector<T>::foreach(const function<void (const T&)>& callback) -> void {
   for(uint n : range(size())) callback(_pool[n]);
 }

nall/windows/guard.hpp

@@ -4,6 +4,24 @@
 #define boolean WindowsBoolean
 #define interface WindowsInterface
 
+#undef UNICODE
+#undef WINVER
+#undef WIN32_LEAN_AND_LEAN
+#undef _WIN32_WINNT
+#undef _WIN32_IE
+#undef __MSVCRT_VERSION__
+#undef NOMINMAX
+#undef PATH_MAX
+
+#define UNICODE
+#define WINVER 0x0601
+#define WIN32_LEAN_AND_MEAN
+#define _WIN32_WINNT WINVER
+#define _WIN32_IE WINVER
+#define __MSVCRT_VERSION__ WINVER
+#define NOMINMAX
+#define PATH_MAX 260
+
 #else
 #undef NALL_WINDOWS_GUARD_HPP
 

nall/windows/utf8.hpp

@@ -1,24 +1,5 @@
 #pragma once
 
-//UTF-8 <> UTF-16 conversion
-//used only for Win32; every other OS uses UTF-8 internally
-
-#if defined(_WIN32)
-
-#undef  UNICODE
-#define UNICODE
-#undef  NOMINMAX
-#define NOMINMAX
-
-#include <nall/windows/guard.hpp>
-#include <winsock2.h>
-#include <windows.h>
-#include <nall/windows/guard.hpp>
-
-#if !defined(PATH_MAX)
-  #define PATH_MAX 260
-#endif
-
 using uint = unsigned;
 
 namespace nall {
@@ -94,7 +75,7 @@ namespace nall {
     uint length = 0;
   };
 
-  inline auto utf8_args(int& argc, char**& argv) -> void {
+  inline auto utf8_arguments(int& argc, char**& argv) -> void {
     wchar_t** wargv = CommandLineToArgvW(GetCommandLineW(), &argc);
     argv = new char*[argc + 1]();
     for(uint i = 0; i < argc; i++) {
@@ -103,5 +84,3 @@ namespace nall {
     }
   }
 }
-
-#endif  //if defined(_WIN32)

ruby/audio/xaudio2.cpp

@@ -1,5 +1,4 @@
 #include "xaudio2.hpp"
-#include <windows.h>
 #undef interface
 
 struct AudioXAudio2 : AudioDriver, public IXAudio2VoiceCallback {

ruby/ruby.cpp

@@ -21,7 +21,7 @@ using namespace ruby;
   #include <Carbon/Carbon.h>
   #include <nall/macos/guard.hpp>
 #elif defined(DISPLAY_WINDOWS)
-  #include <windows.h>
+  #include <mmsystem.h>
 #endif
 
 #include <ruby/video/video.cpp>

ruby/video/cgl.cpp

@@ -26,7 +26,6 @@ struct VideoCGL : VideoDriver, OpenGL {
   auto hasContext() -> bool override { return true; }
   auto hasBlocking() -> bool override { return true; }
   auto hasFlush() -> bool override { return true; }
-  auto hasSmooth() -> bool override { return true; }
   auto hasShader() -> bool override { return true; }
 
   auto setContext(uintptr context) -> bool override {
@@ -47,14 +46,8 @@ struct VideoCGL : VideoDriver, OpenGL {
     return true;
   }
 
-  auto setSmooth(bool) -> bool override {
+  auto setShader(string shader) -> bool override {
-    if(!self.shader) OpenGL::filter = self.smooth ? GL_LINEAR : GL_NEAREST;
+    OpenGL::setShader(shader);
-    return true;
-  }
-
-  auto setShader(string) -> bool override {
-    OpenGL::setShader(self.shader);
-    if(!self.shader) OpenGL::filter = self.smooth ? GL_LINEAR : GL_NEAREST;
     return true;
   }
 

ruby/video/direct3d.cpp

@@ -22,12 +22,12 @@ struct VideoDirect3D : VideoDriver {
   auto hasExclusive() -> bool override { return true; }
   auto hasContext() -> bool override { return true; }
   auto hasBlocking() -> bool override { return true; }
-  auto hasSmooth() -> bool override { return true; }
+  auto hasShader() -> bool override { return true; }
 
   auto setExclusive(bool exclusive) -> bool override { return initialize(); }
   auto setContext(uintptr context) -> bool override { return initialize(); }
   auto setBlocking(bool blocking) -> bool override { return true; }
-  auto setSmooth(bool smooth) -> bool override { return updateFilter(); }
+  auto setShader(string shader) -> bool override { return updateFilter(); }
 
   auto clear() -> void override {
     if(!ready()) return;
@@ -169,7 +169,7 @@ private:
     if(!_device) return false;
     if(_lost && !recover()) return false;
 
-    auto filter = !self.smooth ? D3DTEXF_POINT : D3DTEXF_LINEAR;
+    auto filter = self.shader == "Blur" ? D3DTEXF_LINEAR : D3DTEXF_POINT;
     _device->SetSamplerState(0, D3DSAMP_MINFILTER, filter);
     _device->SetSamplerState(0, D3DSAMP_MAGFILTER, filter);
     return true;

ruby/video/directdraw.cpp

@@ -7,6 +7,7 @@ struct VideoDirectDraw : VideoDriver {
   ~VideoDirectDraw() { terminate(); }
 
   auto create() -> bool override {
+    super.setShader("Blur");
     return initialize();
   }
 

ruby/video/gdi.cpp

@@ -4,6 +4,7 @@ struct VideoGDI : VideoDriver {
   ~VideoGDI() { terminate(); }
 
   auto create() -> bool override {
+    super.setShader("None");
     return initialize();
   }
 

ruby/video/glx.cpp

@@ -23,7 +23,6 @@ struct VideoGLX : VideoDriver, OpenGL {
   auto hasContext() -> bool override { return true; }
   auto hasBlocking() -> bool override { return true; }
   auto hasFlush() -> bool override { return true; }
-  auto hasSmooth() -> bool override { return true; }
   auto hasShader() -> bool override { return true; }
 
   auto hasFormats() -> vector<string> override {
@@ -57,14 +56,8 @@ struct VideoGLX : VideoDriver, OpenGL {
     return false;
   }
 
-  auto setSmooth(bool) -> bool override {
+  auto setShader(string shader) -> bool override {
-    if(!self.shader) OpenGL::filter = self.smooth ? GL_LINEAR : GL_NEAREST;
+    OpenGL::setShader(shader);
-    return true;
-  }
-
-  auto setShader(string) -> bool override {
-    OpenGL::setShader(self.shader);
-    if(!self.shader) OpenGL::filter = self.smooth ? GL_LINEAR : GL_NEAREST;
     return true;
   }
 
@@ -214,7 +207,7 @@ private:
     _doubleBuffer = value;
     _isDirect = glXIsDirect(_display, _glXContext);
 
-    return _ready = OpenGL::initialize();
+    return _ready = OpenGL::initialize(self.shader);
   }
 
   auto terminate() -> void {

ruby/video/glx2.cpp

@@ -38,7 +38,7 @@ struct VideoGLX2 : VideoDriver {
   auto hasBlocking() -> bool override { return true; }
   auto hasFlush() -> bool override { return true; }
   auto hasFormats() -> vector<string> override { return {"RGB24"}; }
-  auto hasSmooth() -> bool override { return true; }
+  auto hasShader() -> bool override { return true; }
 
   auto setContext(uintptr context) -> bool override {
     return initialize();
@@ -63,6 +63,15 @@ struct VideoGLX2 : VideoDriver {
     return false;
   }
 
+  auto setShader(string shader) -> bool override {
+    return true;
+  }
+
+  auto configure(uint width, uint height, double inputFrequency, double outputFrequency) -> bool override {
+    XResizeWindow(_display, _window, width, height);
+    return true;
+  }
+
   auto clear() -> void override {
     memory::fill<uint32_t>(_glBuffer, _glWidth * _glHeight);
     glClearColor(0.0, 0.0, 0.0, 1.0);
@@ -81,22 +90,15 @@ struct VideoGLX2 : VideoDriver {
   }
 
   auto output() -> void override {
-    XWindowAttributes parent, child;
-    XGetWindowAttributes(_display, (Window)self.context, &parent);
-    XGetWindowAttributes(_display, _window, &child);
-    if(child.width != parent.width || child.height != parent.height) {
-      XResizeWindow(_display, _window, parent.width, parent.height);
-    }
-
     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, self.smooth ? GL_LINEAR : GL_NEAREST);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, self.shader == "Blur" ? GL_LINEAR : GL_NEAREST);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, self.smooth ? GL_LINEAR : GL_NEAREST);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, self.shader == "Blur" ? GL_LINEAR : GL_NEAREST);
 
     glMatrixMode(GL_PROJECTION);
     glLoadIdentity();
-    glOrtho(0, parent.width, 0, parent.height, -1.0, 1.0);
+    glOrtho(0, self.width, 0, self.height, -1.0, 1.0);
-    glViewport(0, 0, parent.width, parent.height);
+    glViewport(0, 0, self.width, self.height);
 
     glMatrixMode(GL_MODELVIEW);
     glLoadIdentity();
@@ -105,8 +107,8 @@ struct VideoGLX2 : VideoDriver {
 
     double w = (double)_width / (double)_glWidth;
     double h = (double)_height / (double)_glHeight;
-    int u = parent.width;
+    int u = self.width;
-    int v = parent.height;
+    int v = self.height;
 
     glBegin(GL_TRIANGLE_STRIP);
     glTexCoord2f(0, 0); glVertex3i(0, v, 0);
@@ -261,6 +263,7 @@ private:
   }
 
   bool _ready = false;
+  bool blur = false;
 
   Display* _display = nullptr;
   int _screen = 0;

ruby/video/opengl/main.hpp

@@ -11,7 +11,11 @@ auto OpenGL::setShader(const string& pathname) -> void {
   relativeWidth = 0, relativeHeight = 0;
 
   uint historySize = 0;
-  if(pathname) {
+  if(pathname == "None") {
+    filter = GL_NEAREST;
+  } else if(pathname == "Blur") {
+    filter = GL_LINEAR;
+  } else if(directory::exists(pathname)) {
     auto document = BML::unserialize(file::read({pathname, "manifest.bml"}));
 
     for(auto node : document["settings"]) {
@@ -180,7 +184,7 @@ auto OpenGL::output() -> void {
   }
 }
 
-auto OpenGL::initialize() -> bool {
+auto OpenGL::initialize(const string& shader) -> bool {
   if(!OpenGLBind()) return false;
 
   glDisable(GL_BLEND);
@@ -196,7 +200,7 @@ auto OpenGL::initialize() -> bool {
   OpenGLSurface::allocate();
   glrLinkProgram(program);
 
-  setShader("");
+  setShader(shader);
   return initialized = true;
 }
 

ruby/video/opengl/opengl.hpp

@@ -70,7 +70,7 @@ struct OpenGL : OpenGLProgram {
   auto clear() -> void;
   auto lock(uint32_t*& data, uint& pitch) -> bool;
   auto output() -> void;
-  auto initialize() -> bool;
+  auto initialize(const string& shader) -> bool;
   auto terminate() -> void;
 
   vector<OpenGLProgram> programs;

ruby/video/video.cpp

@@ -71,13 +71,6 @@ auto Video::setFormat(string format) -> bool {
   return true;
 }
 
-auto Video::setSmooth(bool smooth) -> bool {
-  if(instance->smooth == smooth) return true;
-  if(!instance->hasSmooth()) return false;
-  if(!instance->setSmooth(instance->smooth = smooth)) return false;
-  return true;
-}
-
 auto Video::setShader(string shader) -> bool {
   if(instance->shader == shader) return true;
   if(!instance->hasShader()) return false;
@@ -87,6 +80,14 @@ auto Video::setShader(string shader) -> bool {
 
 //
 
+auto Video::configure(uint width, uint height, double inputFrequency, double outputFrequency) -> bool {
+  instance->width = width;
+  instance->height = height;
+  instance->inputFrequency = inputFrequency;
+  instance->outputFrequency = outputFrequency;
+  return instance->configure(width, height, inputFrequency, outputFrequency);
+}
+
 auto Video::clear() -> void {
   return instance->clear();
 }

ruby/video/video.hpp

@@ -13,7 +13,6 @@ struct VideoDriver {
   virtual auto hasBlocking() -> bool { return false; }
   virtual auto hasFlush() -> bool { return false; }
   virtual auto hasFormats() -> vector<string> { return {"RGB24"}; }
-  virtual auto hasSmooth() -> bool { return false; }
   virtual auto hasShader() -> bool { return false; }
 
   auto hasFormat(string format) -> bool { return (bool)hasFormats().find(format); }
@@ -23,9 +22,9 @@ struct VideoDriver {
   virtual auto setBlocking(bool blocking) -> bool { return true; }
   virtual auto setFlush(bool flush) -> bool { return true; }
   virtual auto setFormat(string format) -> bool { return true; }
-  virtual auto setSmooth(bool smooth) -> bool { return true; }
   virtual auto setShader(string shader) -> bool { return true; }
 
+  virtual auto configure(uint width, uint height, double inputFrequency, double outputFrequency) -> bool { return true; }
   virtual auto clear() -> void {}
   virtual auto acquire(uint32_t*& data, uint& pitch, uint width, uint height) -> bool { return false; }
   virtual auto release() -> void {}
@@ -41,8 +40,12 @@ protected:
   bool blocking = false;
   bool flush = false;
   string format = "RGB24";
-  bool smooth = false;
+  string shader = "Blur";
-  string shader = "";
+
+  uint width = 0;
+  uint height = 0;
+  double inputFrequency = 0.0;
+  double outputFrequency = 0.0;
 };
 
 struct Video {
@@ -63,7 +66,6 @@ struct Video {
   auto hasBlocking() -> bool { return instance->hasBlocking(); }
   auto hasFlush() -> bool { return instance->hasFlush(); }
   auto hasFormats() -> vector<string> { return instance->hasFormats(); }
-  auto hasSmooth() -> bool { return instance->hasSmooth(); }
   auto hasShader() -> bool { return instance->hasShader(); }
 
   auto hasFormat(string format) -> bool { return instance->hasFormat(format); }
@@ -73,7 +75,6 @@ struct Video {
   auto blocking() -> bool { return instance->blocking; }
   auto flush() -> bool { return instance->flush; }
   auto format() -> string { return instance->format; }
-  auto smooth() -> bool { return instance->smooth; }
   auto shader() -> string { return instance->shader; }
 
   auto setExclusive(bool exclusive) -> bool;
@@ -81,9 +82,9 @@ struct Video {
   auto setBlocking(bool blocking) -> bool;
   auto setFlush(bool flush) -> bool;
   auto setFormat(string format) -> bool;
-  auto setSmooth(bool smooth) -> bool;
   auto setShader(string shader) -> bool;
 
+  auto configure(uint width, uint height, double inputFrequency, double outputFrequency) -> bool;
   auto clear() -> void;
   auto acquire(uint32_t*& data, uint& pitch, uint width, uint height) -> bool;
   auto release() -> void;

ruby/video/wgl.cpp

@@ -18,7 +18,6 @@ struct VideoWGL : VideoDriver, OpenGL {
   auto hasContext() -> bool override { return true; }
   auto hasBlocking() -> bool override { return true; }
   auto hasFlush() -> bool override { return true; }
-  auto hasSmooth() -> bool override { return true; }
   auto hasShader() -> bool override { return true; }
 
   auto setContext(uintptr context) -> bool override {
@@ -34,14 +33,8 @@ struct VideoWGL : VideoDriver, OpenGL {
     return true;
   }
 
-  auto setSmooth(bool) -> bool override {
+  auto setShader(string shader) -> bool override {
-    if(!self.shader) OpenGL::filter = self.smooth ? GL_LINEAR : GL_NEAREST;
-    return true;
-  }
-
-  auto setShader(string) -> bool override {
     OpenGL::setShader(self.shader);
-    if(!self.shader) OpenGL::filter = self.smooth ? GL_LINEAR : GL_NEAREST;
     return true;
   }
 
@@ -104,7 +97,7 @@ private:
     }
 
     if(wglSwapInterval) wglSwapInterval(self.blocking);
-    return _ready = OpenGL::initialize();
+    return _ready = OpenGL::initialize(self.shader);
   }
 
   auto terminate() -> void {

ruby/video/xshm.cpp

@@ -21,12 +21,29 @@ struct VideoXShm : VideoDriver {
   auto ready() -> bool override { return _ready; }
 
   auto hasContext() -> bool override { return true; }
-  auto hasSmooth() -> bool override { return true; }
+  auto hasShader() -> bool override { return true; }
 
   auto hasFormats() -> vector<string> override { return {"RGB24"}; }
 
   auto setContext(uintptr context) -> bool override { return initialize(); }
-  auto setSmooth(bool smooth) -> bool override { return true; }
+  auto setShader(string shader) -> bool override { return true; }
+
+  auto configure(uint width, uint height, double inputFrequency, double outputFrequency) -> bool override {
+    _outputWidth = width;
+    _outputHeight = height;
+    XResizeWindow(_display, _window, _outputWidth, _outputHeight);
+    free();
+
+    _shmInfo.shmid = shmget(IPC_PRIVATE, _outputWidth * _outputHeight * sizeof(uint32_t), IPC_CREAT | 0777);
+    if(_shmInfo.shmid < 0) return false;
+
+    _shmInfo.shmaddr = (char*)shmat(_shmInfo.shmid, 0, 0);
+    _shmInfo.readOnly = False;
+    XShmAttach(_display, &_shmInfo);
+    _outputBuffer = (uint32_t*)_shmInfo.shmaddr;
+    _image = XShmCreateImage(_display, _visual, _depth, ZPixmap, _shmInfo.shmaddr, &_shmInfo, _outputWidth, _outputHeight);
+    return true;
+  }
 
   auto clear() -> void override {
     auto dp = _inputBuffer;
@@ -65,7 +82,7 @@ struct VideoXShm : VideoDriver {
       uint32_t* sp = _inputBuffer + (uint)ystep * _inputWidth;
       uint32_t* dp = _outputBuffer + y * _outputWidth;
 
-      if(!self.smooth) {
+      if(self.shader != "Blur") {
         for(uint x = 0; x < _outputWidth; x++) {
           *dp++ = 255u << 24 | sp[(uint)xstep];
           xstep += xratio;
@@ -145,23 +162,6 @@ private:
   }
 
   auto size() -> bool {
-    XWindowAttributes windowAttributes;
-    XGetWindowAttributes(_display, (Window)self.context, &windowAttributes);
-
-    if(_outputBuffer && _outputWidth == windowAttributes.width && _outputHeight == windowAttributes.height) return true;
-    _outputWidth = windowAttributes.width;
-    _outputHeight = windowAttributes.height;
-    XResizeWindow(_display, _window, _outputWidth, _outputHeight);
-    free();
-
-    _shmInfo.shmid = shmget(IPC_PRIVATE, _outputWidth * _outputHeight * sizeof(uint32_t), IPC_CREAT | 0777);
-    if(_shmInfo.shmid < 0) return false;
-
-    _shmInfo.shmaddr = (char*)shmat(_shmInfo.shmid, 0, 0);
-    _shmInfo.readOnly = False;
-    XShmAttach(_display, &_shmInfo);
-    _outputBuffer = (uint32_t*)_shmInfo.shmaddr;
-    _image = XShmCreateImage(_display, _visual, _depth, ZPixmap, _shmInfo.shmaddr, &_shmInfo, _outputWidth, _outputHeight);
 
     return true;
   }

ruby/video/xvideo.cpp

@@ -12,6 +12,7 @@ struct VideoXVideo : VideoDriver {
   ~VideoXVideo() { terminate(); }
 
   auto create() -> bool override {
+    super.setShader("Blur");
     return initialize();
   }