From fc83c355451a44a2c8a5ac1aaaea9f19c3fc8aeb Mon Sep 17 00:00:00 2001 From: twinaphex Date: Wed, 7 Jan 2015 18:34:37 +0100 Subject: [PATCH] Update rarchdb --- rarchdb/Makefile | 17 ++-- rarchdb/README.md | 24 +++++ rarchdb/dat_converter | 2 + rarchdb/dat_converter.lua | 127 +++++++++++++++++++++++++ rarchdb/lua_converter.c | 184 +++++++++++++++++++++++++++++++++++++ rarchdb/retro_endianness.h | 93 +++++++++++++++++++ rarchdb/retro_inline.h | 40 ++++++++ rarchdb/rmsgpack.c | 32 ++++--- 8 files changed, 500 insertions(+), 19 deletions(-) create mode 100755 rarchdb/dat_converter create mode 100644 rarchdb/dat_converter.lua create mode 100644 rarchdb/lua_converter.c create mode 100644 rarchdb/retro_endianness.h create mode 100644 rarchdb/retro_inline.h diff --git a/rarchdb/Makefile b/rarchdb/Makefile index 7f416fbbbd..4a5b5fcc19 100644 --- a/rarchdb/Makefile +++ b/rarchdb/Makefile @@ -1,12 +1,13 @@ CFLAGS = -g INCFLAGS = -I. -I../libretro-sdk/include -DAT_CONVERTER_OBJ = rmsgpack.o \ + +LUA_CONVERTER_OBJ = rmsgpack.o \ rmsgpack_dom.o \ rarchdb.o \ bintree.o \ - db_parser.o \ - dat_converter.o \ + lua_converter.o \ $(NULL) + RARCHDB_TOOL_OBJ = rmsgpack.o \ rmsgpack_dom.o \ db_parser.o \ @@ -15,13 +16,15 @@ RARCHDB_TOOL_OBJ = rmsgpack.o \ rarchdb.o \ $(NULL) -all: dat_converter rmsgpack_test rarchdb_tool +LUA_FLAGS = `pkg-config lua --libs` + +all: rmsgpack_test rarchdb_tool lua_converter %.o: %.c ${CC} $(INCFLAGS) $< -c ${CFLAGS} -o $@ -dat_converter: ${DAT_CONVERTER_OBJ} - ${CC} $(INCFLAGS) ${DAT_CONVERTER_OBJ} -o $@ +lua_converter: ${LUA_CONVERTER_OBJ} + ${CC} $(INCFLAGS) ${LUA_CONVERTER_OBJ} ${LUA_FLAGS} -o $@ rarchdb_tool: ${RARCHDB_TOOL_OBJ} ${CC} $(INCFLAGS) ${RARCHDB_TOOL_OBJ} -o $@ @@ -29,4 +32,4 @@ rarchdb_tool: ${RARCHDB_TOOL_OBJ} rmsgpack_test: ${CC} $(INCFLAGS) rmsgpack.c rmsgpack_test.c -g -o $@ clean: - rm -rf *.o rmsgpack_test dat_converter rarchdb_tool + rm -rf *.o rmsgpack_test lua_converter rarchdb_tool diff --git a/rarchdb/README.md b/rarchdb/README.md index 464996dbd7..1568dc476b 100644 --- a/rarchdb/README.md +++ b/rarchdb/README.md @@ -11,3 +11,27 @@ To find an entry with an index `rarchdb_tool find The util `mkdb.sh ` will create a db file with indexes for crc sha1 and md5 +# lua converters +In order to write you own converter you must have a lua file that implements the following functions: + +~~~.lua +-- this function gets called before the db is created and shuold validate the +-- arguments and set up the ground for db insertion +function init(...) + local args = {...} + local script_name = args[1] +end + +-- this is in iterator function. It is called before each insert. +-- the function should return a table for insertion or nil when there are no +-- more records to insert. +function get_value() + return { + key = "value", -- will be saved as string + num = 3, -- will be saved as int + bin = binary("some string"), -- will be saved as binary + unum = uint(3), -- will be saved as uint + some_bool = true, -- will be saved as bool + } +end +~~~ diff --git a/rarchdb/dat_converter b/rarchdb/dat_converter new file mode 100755 index 0000000000..d9f97d6b74 --- /dev/null +++ b/rarchdb/dat_converter @@ -0,0 +1,2 @@ +#!/bin/sh +./lua_converter "$2" dat_converter.lua "$1" diff --git a/rarchdb/dat_converter.lua b/rarchdb/dat_converter.lua new file mode 100644 index 0000000000..c79fa6fbb8 --- /dev/null +++ b/rarchdb/dat_converter.lua @@ -0,0 +1,127 @@ +local dat_obj = nil + +local function dat_lexer(f) + local line, err = f:read("*l") + return function() + local tok = nil + while not tok do + if not line then + return nil + end + tok, line = string.match(line, "^%s*(..-)(%s.*)") + if tok and string.match(tok, "^\"") then + tok, line = string.match(tok..line, "^\"([^\"]-)\"(.*)") + end + if not line then + line = f:read("*l") + end + end + return tok + end +end + +local function dat_parse_table(lexer) + local res = {} + local state = "key" + local key = nil + for tok in lexer do + if state == "key" then + if tok == ")" then + return res + else + key = tok + state = "value" + end + else + if tok == "(" then + res[key] = dat_parse_table(lexer) + else + res[key] = tok + end + state = "key" + end + end + return res +end + +local function dat_parser(lexer) + local res = {} + local state = "key" + local key = nil + local skip = true + for tok in lexer do + if state == "key" then + if tok == "game" then + skip = false + end + state = "value" + else + if tok == "(" then + local v = dat_parse_table(lexer) + if not skip then + table.insert(res, v) + skip = true + end + else + error("Expected '(' found '"..tok.."'") + end + state = "key" + end + end + return res +end + +local function unhex(s) + if not s then return nil end + return (s:gsub('.', function (c) + return string.format('%02X', string.byte(c)) + end)) +end + +function init(...) + local args = {...} + local dat_path = args[2] + assert(dat_path, "dat file argument is missing") + local dat_file, err = io.open(dat_path, "r") + if err then + error("could not open dat file '" .. dat_path .. "':" .. err) + end + + print("Parsing dat file '" .. dat_path .. "'...") + dat_obj = dat_parser(dat_lexer(dat_file)) + dat_file:close() +end + +function get_value() + local t = table.remove(dat_obj) + if not t then + return + else + return { + name = t.name, + description = t.description, + rom_name = t.rom.name, + size = uint(tonumber(t.rom.size)), + users = uint(tonumber(t.users)), + releasemonth = uint(tonumber(t.releasemonth)), + releaseyear = uint(tonumber(t.releaseyear)), + rumble = uint(tonumber(t.rumble)), + analog = uint(tonumber(t.analog)), + + esrb_rating = t.esrb_rating, + elspa_rating = t.elspa_rating, + pegi_rating = t.pegi_rating, + cero_rating = t.cero_rating, + + developers = t.developers, + publisher = t.publisher, + origin = t.origin, + + crc = binary(unhex(t.rom.crc)), + md5 = binary(unhex(t.rom.md5)), + sha1 = binary(unhex(t.rom.sha1)), + serial = binary(t.rom.serial), + } + end +end + diff --git a/rarchdb/lua_converter.c b/rarchdb/lua_converter.c new file mode 100644 index 0000000000..a51a632011 --- /dev/null +++ b/rarchdb/lua_converter.c @@ -0,0 +1,184 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "rarchdb.h" +int master_key = 1; + +const char *LUA_COMMON = " \ +function binary(s) if s ~= nil then return {binary = s} else return nil end end \ +function uint(s) if s ~= nil then return {uint = s} else return nil end end \ +"; + +static int call_init(lua_State *L, int argc, const char** argv) { + int rv = -1; + int i; + + lua_getglobal(L, "init"); + for (i = 0; i < argc; i++) { + lua_pushstring(L, argv[i]); + } + + if (lua_pcall(L, argc, 0, 0) != 0) { + printf( + "error running function `init': %s\n", + lua_tostring(L, -1) + ); + } + + return rv; +} + + +static int value_provider(void *ctx, struct rmsgpack_dom_value *out) { + lua_State *L = ctx; + + int rv = -1; + int i; + const char *tmp_string = NULL; + char *tmp_buff = NULL; + struct rmsgpack_dom_value *tmp_value; + const int key_idx = -2; + const int value_idx = -1; + const int MAX_FIELDS = 100; + size_t tmp_len; + lua_Number tmp_num; + + lua_getglobal(L, "get_value"); + + if (lua_pcall(L, 0, 1, 0) != 0) { + printf( + "error running function `get_value': %s\n", + lua_tostring(L, -1) + ); + } + + if (lua_isnil(L, -1)) { + rv = 1; + } else if (lua_istable(L, -1)) { + out->type = RDT_MAP; + out->map.len = 0; + out->map.items = calloc(MAX_FIELDS, sizeof(struct rmsgpack_dom_pair)); + lua_pushnil(L); + while (lua_next(L, -2) != 0) { + if (out->map.len > MAX_FIELDS) { + printf("skipping due to too many keys\n"); + } else if (!lua_isstring(L, key_idx)) { + printf("skipping non string key\n"); + } else if (lua_isnil(L, value_idx)) { + // Skipping nil value fields to save disk space + } else { + i = out->map.len; + tmp_buff = strdup(lua_tostring(L, key_idx)); + out->map.items[i].key.type = RDT_STRING; + out->map.items[i].key.string.len = strlen(tmp_buff); + out->map.items[i].key.string.buff = tmp_buff; + + tmp_value = &out->map.items[i].value; + switch(lua_type(L, value_idx)) { + case LUA_TNUMBER: + tmp_num = lua_tonumber(L, value_idx); + tmp_value->type = RDT_INT; + tmp_value->int_ = tmp_num; + break; + case LUA_TBOOLEAN: + tmp_value->type = RDT_BOOL; + tmp_value->bool_ = lua_toboolean(L, value_idx); + break; + case LUA_TSTRING: + tmp_buff = strdup(lua_tostring(L, key_idx)); + tmp_value->type = RDT_STRING; + tmp_value->string.len = strlen(tmp_buff); + tmp_value->string.buff = tmp_buff; + break; + case LUA_TTABLE: + lua_getfield(L, value_idx, "binary"); + if (!lua_isstring(L, -1)) { + lua_pop(L, 1); + lua_getfield(L, value_idx, "uint"); + if (!lua_isnumber(L, -1)) { + lua_pop(L, 1); + goto set_nil; + } else { + tmp_num = lua_tonumber(L, -1); + tmp_value->type = RDT_UINT; + tmp_value->uint_ = tmp_num; + lua_pop(L, 1); + } + } else { + tmp_string = lua_tolstring(L, -1, &tmp_len); + tmp_buff = malloc(tmp_len); + memcpy(tmp_buff, tmp_string, tmp_len); + tmp_value->type = RDT_BINARY; + tmp_value->binary.len = tmp_len; + tmp_value->binary.buff = tmp_buff; + lua_pop(L, 1); + } + break; + default: +set_nil: + tmp_value->type = RDT_NULL; + } + out->map.len++; + } + lua_pop(L, 1); + } + rv = 0; + } else { + printf("function `get_value' must return a table or nil\n"); + } + lua_pop(L, 1); + return rv; +} + +int main(int argc, char **argv) +{ + const char* db_file; + const char* lua_file; + int dst = -1; + int rv = 0; + + if (argc < 3) { + printf("usage:\n%s [args ...]\n", argv[0]); + return 1; + } + + db_file = argv[1]; + lua_file = argv[2]; + + lua_State *L = luaL_newstate(); + luaL_openlibs(L); + luaL_dostring(L, LUA_COMMON); + + if (luaL_dofile(L, lua_file) != 0) { + return 1; + } + + call_init(L, argc - 2, (const char**) argv + 2); + + dst = open(db_file, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + if (dst == -1) + { + printf("Could not open destination file '%s': %s\n", db_file, strerror(errno)); + rv = errno; + goto clean; + } + + rv = rarchdb_create(dst, &value_provider, L); + +clean: + lua_close(L); + if (dst != -1) { + close(dst); + } + return rv; +} + diff --git a/rarchdb/retro_endianness.h b/rarchdb/retro_endianness.h new file mode 100644 index 0000000000..baf98b74f8 --- /dev/null +++ b/rarchdb/retro_endianness.h @@ -0,0 +1,93 @@ +/* Copyright (C) 2010-2014 The RetroArch team + * + * --------------------------------------------------------------------------------------- + * The following license statement only applies to this file (retro_endianness.h). + * --------------------------------------------------------------------------------------- + * + * Permission is hereby granted, free of charge, + * to any person obtaining a copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef __LIBRETRO_SDK_ENDIANNESS_H +#define __LIBRETRO_SDK_ENDIANNESS_H + +#include +#include + +#define SWAP16(x) ((uint16_t)( \ + (((uint16_t)(x) & 0x00ff) << 8) | \ + (((uint16_t)(x) & 0xff00) >> 8) \ + )) + +#define SWAP32(x) ((uint32_t)( \ + (((uint32_t)(x) & 0x000000ff) << 24) | \ + (((uint32_t)(x) & 0x0000ff00) << 8) | \ + (((uint32_t)(x) & 0x00ff0000) >> 8) | \ + (((uint32_t)(x) & 0xff000000) >> 24) \ + )) + +static INLINE uint8_t is_little_endian(void) +{ + union + { + uint16_t x; + uint8_t y[2]; + } u; + + u.x = 1; + return u.y[0]; +} + +static INLINE uint32_t swap_if_big32(uint32_t val) +{ + if (is_little_endian()) + return val; + return (val >> 24) | ((val >> 8) & 0xFF00) | + ((val << 8) & 0xFF0000) | (val << 24); +} + +static INLINE uint32_t swap_if_little32(uint32_t val) +{ + if (is_little_endian()) + return (val >> 24) | ((val >> 8) & 0xFF00) | + ((val << 8) & 0xFF0000) | (val << 24); + return val; +} + +static INLINE uint16_t swap_if_big16(uint16_t val) +{ + if (is_little_endian()) + return val; + return (val >> 8) | (val << 8); +} + +static INLINE uint16_t swap_if_little16(uint16_t val) +{ + if (is_little_endian()) + return (val >> 8) | (val << 8); + return val; +} + +static INLINE void store32be(uint32_t *addr, uint32_t data) +{ + *addr = is_little_endian() ? SWAP32(data) : data; +} + +static INLINE uint32_t load32be(const uint32_t *addr) +{ + return is_little_endian() ? SWAP32(*addr) : *addr; +} + +#endif diff --git a/rarchdb/retro_inline.h b/rarchdb/retro_inline.h new file mode 100644 index 0000000000..aa58d917d3 --- /dev/null +++ b/rarchdb/retro_inline.h @@ -0,0 +1,40 @@ +/* Copyright (C) 2010-2014 The RetroArch team + * + * --------------------------------------------------------------------------------------- + * The following license statement only applies to this file (retro_inline.h). + * --------------------------------------------------------------------------------------- + * + * Permission is hereby granted, free of charge, + * to any person obtaining a copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef __LIBRETRO_SDK_INLINE_H +#define __LIBRETRO_SDK_INLINE_H + +#if !defined(__cplusplus) && defined(_WIN32) + +#ifndef INLINE +#define INLINE _inline +#endif + +#else + +#ifndef INLINE +#define INLINE inline +#endif + +#endif + +#endif diff --git a/rarchdb/rmsgpack.c b/rarchdb/rmsgpack.c index 4b6710ff77..e6768b0e8c 100644 --- a/rarchdb/rmsgpack.c +++ b/rarchdb/rmsgpack.c @@ -57,6 +57,8 @@ static const uint8_t MPF_NIL = 0xc0; int rmsgpack_write_array_header(int fd, uint32_t size) { + uint16_t tmp_i16; + uint32_t tmp_i32; if (size < 16) { size = (size | MPF_FIXARRAY); @@ -68,7 +70,8 @@ int rmsgpack_write_array_header(int fd, uint32_t size) { if (write(fd, &MPF_ARRAY16, sizeof(MPF_ARRAY16)) == -1) return -errno; - if (write(fd, (void *)(&size), sizeof(uint16_t)) == -1) + tmp_i16 = httobe16(size); + if (write(fd, (void *)(&tmp_i16), sizeof(uint16_t)) == -1) return -errno; return sizeof(int8_t) + sizeof(uint16_t); } @@ -76,7 +79,8 @@ int rmsgpack_write_array_header(int fd, uint32_t size) { if (write(fd, &MPF_ARRAY32, sizeof(MPF_ARRAY32)) == -1) return -errno; - if (write(fd, (void *)(&size), sizeof(uint32_t)) == -1) + tmp_i32 = httobe32(size); + if (write(fd, (void *)(&tmp_i32), sizeof(uint32_t)) == -1) return -errno; return sizeof(int8_t) + sizeof(uint32_t); } @@ -84,6 +88,8 @@ int rmsgpack_write_array_header(int fd, uint32_t size) int rmsgpack_write_map_header(int fd, uint32_t size) { + uint16_t tmp_i16; + uint32_t tmp_i32; if (size < 16) { size = (size | MPF_FIXMAP); @@ -95,15 +101,17 @@ int rmsgpack_write_map_header(int fd, uint32_t size) { if (write(fd, &MPF_MAP16, sizeof(MPF_MAP16)) == -1) return -errno; - if (write(fd, (void *)(&size), sizeof(uint16_t)) == -1) + tmp_i16 = httobe16(size); + if (write(fd, (void *)(&tmp_i16), sizeof(uint16_t)) == -1) return -errno; - return sizeof(int8_t) + sizeof(uint16_t); + return sizeof(uint8_t) + sizeof(uint16_t); } else { + tmp_i32 = httobe32(size); if (write(fd, &MPF_MAP32, sizeof(MPF_MAP32)) == -1) return -errno; - if (write(fd, (void *)(&size), sizeof(uint32_t)) == -1) + if (write(fd, (void *)(&tmp_i32), sizeof(uint32_t)) == -1) return -errno; return sizeof(int8_t) + sizeof(uint32_t); } @@ -131,13 +139,13 @@ int rmsgpack_write_string(int fd, const char *s, uint32_t len) } else if (len < 1<<16) { - if (write(fd, &MPF_STR16, sizeof(MPF_STR16)) == -1) - return -errno; - tmp_i16 = httobe16(len); - if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1) - return -errno; - written += sizeof(uint16_t); - } + if (write(fd, &MPF_STR16, sizeof(MPF_STR16)) == -1) + return -errno; + tmp_i16 = httobe16(len); + if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1) + return -errno; + written += sizeof(uint16_t); + } else { if (write(fd, &MPF_STR32, sizeof(MPF_STR32)) == -1)