From 7e4def44fd931008cc8fe110983a0b7645adc8f0 Mon Sep 17 00:00:00 2001 From: aliaspider Date: Sat, 23 Jan 2016 00:57:51 +0100 Subject: [PATCH 01/11] (libretro-db) prevent lua_converter from generating a random output each run. --- libretro-db/lua_common.c | 56 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/libretro-db/lua_common.c b/libretro-db/lua_common.c index 0334a571f6..1422e68813 100644 --- a/libretro-db/lua_common.c +++ b/libretro-db/lua_common.c @@ -94,6 +94,62 @@ set_nil: } lua_pop(L, 1); } +#if 1 + /* re-order to avoid random output each run */ + struct rmsgpack_dom_pair* ordered_pairs = calloc(out->val.map.len, sizeof(struct rmsgpack_dom_pair)); + struct rmsgpack_dom_pair* ordered_pairs_outp = ordered_pairs; + const char* ordered_keys[] = + { + "name", + "description", + "rom_name", + "size", + "users", + "releasemonth", + "releaseyear", + "rumble", + "analog", + + "famitsu_rating", + "edge_rating", + "edge_issue", + "edge_review", + + "enhancement_hw", + "barcode", + "esrb_rating", + "elspa_rating", + "pegi_rating", + "cero_rating", + "franchise", + + "developer", + "publisher", + "origin", + + "crc", + "md5", + "sha1", + "serial" + }; + for(i = 0; i < (sizeof(ordered_keys)/sizeof(char*)); i++) + { + int j; + for(j = 0; j < out->val.map.len; j++) + { + if(!strcmp(ordered_keys[i], out->val.map.items[j].key.val.string.buff)) + { + *ordered_pairs_outp++ = out->val.map.items[j]; + break; + } + } + } + + free(out->val.map.items); + out->val.map.items = ordered_pairs; + out->val.map.len = ordered_pairs_outp - ordered_pairs; +#endif + rv = 0; return rv; } From 3133b67da54d72aca8d1b51a2c062558c2c04ab2 Mon Sep 17 00:00:00 2001 From: aliaspider Date: Sat, 23 Jan 2016 01:07:35 +0100 Subject: [PATCH 02/11] (libretro-db) cleanup Makefile. --- libretro-db/Makefile | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/libretro-db/Makefile b/libretro-db/Makefile index b4382b034a..d2f860fb48 100644 --- a/libretro-db/Makefile +++ b/libretro-db/Makefile @@ -2,6 +2,8 @@ CFLAGS = -g -O2 LIBRETRO_COMMON_DIR := ../libretro-common INCFLAGS = -I. -I$(LIBRETRO_COMMON_DIR)/include +TARGETS = rmsgpack_test libretrodb_tool plain_converter lua_converter + LUA_CONVERTER_C = \ rmsgpack.c \ rmsgpack_dom.c \ @@ -14,7 +16,7 @@ LUA_CONVERTER_C = \ $(LIBRETRO_COMMON_DIR)/file/retro_file.c \ $(LIBRETRO_COMMON_DIR)/compat/compat_strl.c -LUA_CONVERTER_C_OBJS := $(LUA_CONVERTER_C:.c=.o) +LUA_CONVERTER_OBJS := $(LUA_CONVERTER_C:.c=.o) PLAIN_CONVERTER_C = \ lexer.c \ @@ -30,7 +32,7 @@ PLAIN_CONVERTER_C = \ $(LIBRETRO_COMMON_DIR)/file/retro_file.c \ $(LIBRETRO_COMMON_DIR)/compat/compat_strl.c -PLAIN_CONVERTER_C_OBJS := $(PLAIN_CONVERTER_C:.c=.o) +PLAIN_CONVERTER_OBJS := $(PLAIN_CONVERTER_C:.c=.o) RARCHDB_TOOL_C = \ rmsgpack.c \ @@ -66,35 +68,32 @@ RMSGPACK_C = \ RMSGPACK_OBJS := $(RMSGPACK_C:.c=.o) LUA_FLAGS = `pkg-config lua --libs` -TESTLIB_FLAGS = ${CFLAGS} ${LUA_FLAGS} -shared -fpic +TESTLIB_FLAGS = $(CFLAGS) $(LUA_FLAGS) -shared -fpic .PHONY: all clean check -all: rmsgpack_test libretrodb_tool plain_converter lua_converter +all: $(TARGETS) %.o: %.c - ${CC} $(INCFLAGS) $< -c ${CFLAGS} -o $@ + $(CC) $(INCFLAGS) $< -c $(CFLAGS) -o $@ -plain_converter: ${PLAIN_CONVERTER_C_OBJS} - ${CC} $(INCFLAGS) ${PLAIN_CONVERTER_C_OBJS} -o $@ +plain_converter: $(PLAIN_CONVERTER_OBJS) + $(CC) $(INCFLAGS) $(PLAIN_CONVERTER_OBJS) -o $@ -lua_converter: ${LUA_CONVERTER_C_OBJS} - ${CC} $(INCFLAGS) ${LUA_CONVERTER_C_OBJS} ${LUA_FLAGS} -o $@ +lua_converter: $(LUA_CONVERTER_OBJS) + $(CC) $(INCFLAGS) $(LUA_CONVERTER_OBJS) $(LUA_FLAGS) -o $@ -libretrodb_tool: ${RARCHDB_TOOL_OBJS} - ${CC} $(INCFLAGS) ${RARCHDB_TOOL_OBJS} -o $@ +libretrodb_tool: $(RARCHDB_TOOL_OBJS) + $(CC) $(INCFLAGS) $(RARCHDB_TOOL_OBJS) -o $@ rmsgpack_test: $(RMSGPACK_OBJS) - ${CC} $(INCFLAGS) ${RMSGPACK_OBJS} -g -o $@ + $(CC) $(INCFLAGS) $(RMSGPACK_OBJS) -g -o $@ -testlib.so: ${TESTLIB_OBJS} - ${CC} ${INCFLAGS} ${TESTLIB_FLAGS} ${TESTLIB_OBJS} -o $@ +testlib.so: $(TESTLIB_OBJS) + $(CC) $(INCFLAGS) $(TESTLIB_FLAGS) $(TESTLIB_OBJS) -o $@ check: testlib.so tests.lua lua ./tests.lua clean: - rm -rf $(LIBRETRO_COMMON_DIR)/*.o - rm -rf $(LIBRETRO_COMMON_DIR)/compat/*.o - rm -rf $(LIBRETRO_COMMON_DIR)/file/*.o - rm -rf *.o rmsgpack_test plain_converter libretrodb_tool testlib.so + rm -rf $(TARGETS) $(PLAIN_CONVERTER_OBJS) $(LUA_CONVERTER_OBJS) $(RARCHDB_TOOL_OBJS) $(RMSGPACK_OBJS) $(TESTLIB_OBJS) From 8207ab56c12c9f328041b93d8e8f1cccafbd10bd Mon Sep 17 00:00:00 2001 From: aliaspider Date: Sat, 23 Jan 2016 01:25:09 +0100 Subject: [PATCH 03/11] (libretro-db) plug memory leak. --- libretro-db/libretrodb.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libretro-db/libretrodb.c b/libretro-db/libretrodb.c index 103b2ad8b7..324b0762aa 100644 --- a/libretro-db/libretrodb.c +++ b/libretro-db/libretrodb.c @@ -130,6 +130,7 @@ int libretrodb_create(RFILE *fd, libretrodb_value_provider value_provider, retro_fseek(fd, sizeof(libretrodb_header_t), SEEK_CUR); + item.type = RDT_NULL; while ((rv = value_provider(ctx, &item)) == 0) { if ((rv = validate_document(&item)) < 0) @@ -138,6 +139,8 @@ int libretrodb_create(RFILE *fd, libretrodb_value_provider value_provider, if ((rv = rmsgpack_dom_write(fd, &item)) < 0) goto clean; + rmsgpack_dom_value_free(&item); + item.type = RDT_NULL; item_count++; } From 86907fb3b70f637501c9a8a01d3d0fc8a57b24bf Mon Sep 17 00:00:00 2001 From: aliaspider Date: Sat, 23 Jan 2016 01:33:44 +0100 Subject: [PATCH 04/11] (libretro-db) test-lib buildfix. --- libretro-db/rmsgpack_test.c | 2 +- libretro-db/testlib.c | 37 +++++++++++++++++++++++-------------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/libretro-db/rmsgpack_test.c b/libretro-db/rmsgpack_test.c index cabeb4e4bb..b0be32f2fb 100644 --- a/libretro-db/rmsgpack_test.c +++ b/libretro-db/rmsgpack_test.c @@ -99,7 +99,7 @@ static int stub_read_bin( void * data ){ stub_state_pre_print(data); - printf("b'%s'", s); + printf("b'%s'", (char*)s); stub_state_post_print(data); free(s); return 0; diff --git a/libretro-db/testlib.c b/libretro-db/testlib.c index 865576f598..5ee953cb16 100644 --- a/libretro-db/testlib.c +++ b/libretro-db/testlib.c @@ -12,6 +12,15 @@ #include "libretrodb.h" #include "lua_common.h" +struct libretrodb +{ + RFILE *fd; + uint64_t root; + uint64_t count; + uint64_t first_index_offset; + char path[1024]; +}; + static void push_rmsgpack_value(lua_State *L, struct rmsgpack_dom_value *value) { uint32_t i; @@ -19,38 +28,38 @@ static void push_rmsgpack_value(lua_State *L, struct rmsgpack_dom_value *value) switch(value->type) { case RDT_INT: - lua_pushnumber(L, value->int_); + lua_pushnumber(L, value->val.int_); break; case RDT_UINT: - lua_pushnumber(L, value->uint_); + lua_pushnumber(L, value->val.uint_); break; case RDT_BINARY: - lua_pushlstring(L, value->binary.buff, value->binary.len); + lua_pushlstring(L, value->val.binary.buff, value->val.binary.len); break; case RDT_BOOL: - lua_pushboolean(L, value->bool_); + lua_pushboolean(L, value->val.bool_); break; case RDT_NULL: lua_pushnil(L); break; case RDT_STRING: - lua_pushlstring(L, value->string.buff, value->binary.len); + lua_pushlstring(L, value->val.string.buff, value->val.binary.len); break; case RDT_MAP: - lua_createtable(L, 0, value->map.len); - for (i = 0; i < value->map.len; i++) + lua_createtable(L, 0, value->val.map.len); + for (i = 0; i < value->val.map.len; i++) { - push_rmsgpack_value(L, &value->map.items[i].key); - push_rmsgpack_value(L, &value->map.items[i].value); + push_rmsgpack_value(L, &value->val.map.items[i].key); + push_rmsgpack_value(L, &value->val.map.items[i].value); lua_settable(L, -3); } break; case RDT_ARRAY: - lua_createtable(L, value->array.len, 0); - for (i = 0; i < value->array.len; i++) + lua_createtable(L, value->val.array.len, 0); + for (i = 0; i < value->val.array.len; i++) { lua_pushnumber(L, i + 1); - push_rmsgpack_value(L, &value->array.items[i]); + push_rmsgpack_value(L, &value->val.array.items[i]); lua_settable(L, -3); } break; @@ -130,7 +139,7 @@ static int db_new(lua_State *L) return 2; } -static libretrodb *checkdb(lua_State *L) +static libretrodb_t *checkdb(lua_State *L) { void *ud = luaL_checkudata(L, 1, "RarchDB.DB"); luaL_argcheck(L, ud != NULL, 1, "`RarchDB.DB' expected"); @@ -200,7 +209,7 @@ static int db_query(lua_State *L) return 2; } -static libretrodb_cursor *checkcursor(lua_State *L) +static libretrodb_cursor_t *checkcursor(lua_State *L) { void *ud = luaL_checkudata(L, 1, "RarchDB.Cursor"); luaL_argcheck(L, ud != NULL, 1, "`RarchDB.Cursor' expected"); From f8273ba5ba92c916bbf89f3b102c849206ee1055 Mon Sep 17 00:00:00 2001 From: aliaspider Date: Sat, 23 Jan 2016 01:51:09 +0100 Subject: [PATCH 05/11] (libretro-db) add a dat->rdb converter in C. still lacks match key support for merging multiple dat files. --- libretro-db/Makefile | 23 +- libretro-db/c_converter.c | 650 ++++++++++++++++++++++++++++++++ libretro-db/c_converter_test.sh | 75 ++++ 3 files changed, 745 insertions(+), 3 deletions(-) create mode 100644 libretro-db/c_converter.c create mode 100755 libretro-db/c_converter_test.sh diff --git a/libretro-db/Makefile b/libretro-db/Makefile index d2f860fb48..e0c5b6eda0 100644 --- a/libretro-db/Makefile +++ b/libretro-db/Makefile @@ -1,8 +1,8 @@ -CFLAGS = -g -O2 +CFLAGS = -g -O2 -Wall LIBRETRO_COMMON_DIR := ../libretro-common INCFLAGS = -I. -I$(LIBRETRO_COMMON_DIR)/include -TARGETS = rmsgpack_test libretrodb_tool plain_converter lua_converter +TARGETS = rmsgpack_test libretrodb_tool plain_converter lua_converter c_converter LUA_CONVERTER_C = \ rmsgpack.c \ @@ -34,6 +34,20 @@ PLAIN_CONVERTER_C = \ PLAIN_CONVERTER_OBJS := $(PLAIN_CONVERTER_C:.c=.o) +C_CONVERTER_C = \ + rmsgpack.c \ + rmsgpack_dom.c \ + libretrodb.c \ + bintree.c \ + query.c \ + c_converter.c \ + $(LIBRETRO_COMMON_DIR)/hash/rhash.c \ + $(LIBRETRO_COMMON_DIR)/compat/compat_fnmatch.c \ + $(LIBRETRO_COMMON_DIR)/file/retro_file.c \ + $(LIBRETRO_COMMON_DIR)/compat/compat_strl.c + +C_CONVERTER_OBJS := $(C_CONVERTER_C:.c=.o) + RARCHDB_TOOL_C = \ rmsgpack.c \ rmsgpack_dom.c \ @@ -83,6 +97,9 @@ plain_converter: $(PLAIN_CONVERTER_OBJS) lua_converter: $(LUA_CONVERTER_OBJS) $(CC) $(INCFLAGS) $(LUA_CONVERTER_OBJS) $(LUA_FLAGS) -o $@ +c_converter: $(C_CONVERTER_OBJS) + $(CC) $(INCFLAGS) $(C_CONVERTER_OBJS) $(CFLAGS) -o $@ + libretrodb_tool: $(RARCHDB_TOOL_OBJS) $(CC) $(INCFLAGS) $(RARCHDB_TOOL_OBJS) -o $@ @@ -96,4 +113,4 @@ check: testlib.so tests.lua lua ./tests.lua clean: - rm -rf $(TARGETS) $(PLAIN_CONVERTER_OBJS) $(LUA_CONVERTER_OBJS) $(RARCHDB_TOOL_OBJS) $(RMSGPACK_OBJS) $(TESTLIB_OBJS) + rm -rf $(TARGETS) $(PLAIN_CONVERTER_OBJS) $(LUA_CONVERTER_OBJS) $(C_CONVERTER_OBJS) $(RARCHDB_TOOL_OBJS) $(RMSGPACK_OBJS) $(TESTLIB_OBJS) diff --git a/libretro-db/c_converter.c b/libretro-db/c_converter.c new file mode 100644 index 0000000000..8927f0e183 --- /dev/null +++ b/libretro-db/c_converter.c @@ -0,0 +1,650 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libretrodb.h" + +void dat_converter_exit(int rc) +{ + fflush(stdout); + exit(rc); +} + +typedef enum +{ + DAT_CONVERTER_STRING_MAP, + DAT_CONVERTER_LIST_MAP, +} dat_converter_map_enum; + +typedef enum +{ + DAT_CONVERTER_TOKEN_LIST, + DAT_CONVERTER_STRING_LIST, + DAT_CONVERTER_MAP_LIST, + DAT_CONVERTER_LIST_LIST, +} dat_converter_list_enum; + +typedef struct +{ + char* label; + int line_no; + int column; + const char* fname; +} dat_converter_token_t; + +typedef struct dat_converter_map_t dat_converter_map_t; +typedef struct dat_converter_list_t dat_converter_list_t; +typedef union dat_converter_list_item_t dat_converter_list_item_t; + +struct dat_converter_map_t +{ + char* key; + dat_converter_map_enum type; + union + { + char* string; + dat_converter_list_t* list; + } value; +}; + +struct dat_converter_list_t +{ + dat_converter_list_enum type; + dat_converter_list_item_t* values; + int count; + int capacity; +}; + +union dat_converter_list_item_t +{ + char* string; + dat_converter_map_t map; + dat_converter_token_t token; + dat_converter_list_t* list; +}; + +dat_converter_list_t* dat_converter_list_create(dat_converter_list_enum type) +{ + dat_converter_list_t* list = malloc(sizeof(*list)); + list->type = type; + list->count = 0; + list->capacity = (1 << 8); + list->values = (dat_converter_list_item_t*)malloc(sizeof(*list->values) * list->capacity); + return list; +} + +void dat_converter_list_append(dat_converter_list_t* dst, void* item) +{ + if (dst->count == dst->capacity) + { + dst->capacity <<= 1; + dst->values = realloc(dst->values, sizeof(*dst->values) * dst->capacity); + } + switch (dst->type) + { + case DAT_CONVERTER_TOKEN_LIST: + { + dat_converter_token_t* token = (dat_converter_token_t*) item; + dst->values[dst->count].token = *token; + break; + } + case DAT_CONVERTER_STRING_LIST: + { + char* str = (char*) item; + dst->values[dst->count].string = str; + break; + } + case DAT_CONVERTER_MAP_LIST: + { + dat_converter_map_t* map = (dat_converter_map_t*) item; + dst->values[dst->count].map = *map; + break; + } + case DAT_CONVERTER_LIST_LIST: + { + dat_converter_list_t* list = (dat_converter_list_t*) item; + dst->values[dst->count].list = list; + break; + } + default: + return; + } + dst->count++; +} + +void dat_converter_list_free(dat_converter_list_t* list) +{ + if(!list) + return; + switch (list->type) + { + case DAT_CONVERTER_LIST_LIST: + while (list->count--) + dat_converter_list_free(list->values[list->count].list); + break; + case DAT_CONVERTER_MAP_LIST: + while (list->count--) + { + if (list->values[list->count].map.type == DAT_CONVERTER_LIST_MAP) + dat_converter_list_free(list->values[list->count].map.value.list); + } + break; + default: + break; + } + + free(list->values); + free(list); +} + +void dat_converter_token_list_dump(dat_converter_list_t* list) +{ + int i; + + switch (list->type) + { + case DAT_CONVERTER_TOKEN_LIST: + for (i = 0; i < list->count; i++) + printf("\x1B[31m(%6i)@%s:%5i:%-5i\x1B[0m\"%s\"\n", i, + list->values[i].token.fname, + list->values[i].token.line_no, + list->values[i].token.column, + list->values[i].token.label); + break; + default: + return; + } + +} + +dat_converter_list_t* dat_converter_lexer(char* src, const char* dat_path) +{ + + + dat_converter_list_t* token_list = dat_converter_list_create(DAT_CONVERTER_TOKEN_LIST); + dat_converter_token_t token = {NULL, 1, 1, dat_path}; + + bool quoted_token = false; + + while (*src) + { + if ((!quoted_token && (*src == '\t' || *src == ' ')) || (*src == '\r')) + { + *src = '\0'; + src++; + token.column++; + token.label = NULL; + quoted_token = false; + continue; + } + + if (*src == '\n') + { + *src = '\0'; + src++; + token.column = 1; + token.line_no++; + token.label = NULL; + quoted_token = false; + continue; + } + + if (*src == '\"') + { + *src = '\0'; + src++; + token.column++; + quoted_token = !quoted_token; + token.label = NULL; + + if (quoted_token) + { + token.label = src; + dat_converter_list_append(token_list, &token); + } + + continue; + } + + if (!token.label) + { + token.label = src; + dat_converter_list_append(token_list, &token); + } + + src++; + token.column++; + } + *src = '\0'; + + token.label = NULL; + dat_converter_list_append(token_list, &token); + + return token_list; +} + +dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token) +{ + bool state_is_key = true; + dat_converter_list_t* parsed_table = dat_converter_list_create(DAT_CONVERTER_MAP_LIST); + char* key = NULL; + + dat_converter_list_item_t* current = *start_token; + + while (current->token.label) + { + + if (state_is_key) + { + if (!strcmp(current->token.label, ")")) + { + current++; + *start_token = current; + return parsed_table; + } + else if (!strcmp(current->token.label, "(")) + { + printf("%s:%d:%d: fatal error: Unexpected '(' instead of key\n", + current->token.fname, + current->token.line_no, + current->token.column); + dat_converter_exit(1); + } + else + { + key = current->token.label; + state_is_key = false; + current++; + } + } + else + { + if (!strcmp(current->token.label, "(")) + { + current++; + dat_converter_map_t map; + map.type = DAT_CONVERTER_LIST_MAP; + map.key = key; + map.value.list = dat_parser_table(¤t); + dat_converter_list_append(parsed_table, &map); + } + else if (!strcmp(current->token.label, ")")) + { + printf("%s:%d:%d: fatal error: Unexpected ')' instead of value\n", + current->token.fname, + current->token.line_no, + current->token.column); + // fflush(stdout); + dat_converter_exit(1); + } + else + { + dat_converter_map_t map; + map.type = DAT_CONVERTER_STRING_MAP; + map.key = key; + map.value.string = current->token.label; + dat_converter_list_append(parsed_table, &map); + current++; + } + state_is_key = true; + } + } + + printf("%s:%d:%d: fatal error: Missing ')' for '('\n", + (*start_token)->token.fname, + (*start_token)->token.line_no, + (*start_token)->token.column); + dat_converter_exit(1); + + /* unreached */ + dat_converter_list_free(parsed_table); + return NULL; +} + +dat_converter_list_t* dat_parser(dat_converter_list_t* lexer_list) +{ + bool state_is_key = true; + bool skip = true; + dat_converter_list_t* parsed_list = dat_converter_list_create(DAT_CONVERTER_LIST_LIST); + dat_converter_list_item_t* current = lexer_list->values; + dat_converter_list_append(parsed_list, NULL); + + while (current->token.label) + { + if (state_is_key) + { + if (!strcmp(current->token.label, "game")) + skip = false; + state_is_key = false; + current++; + } + else + { + if (!strcmp(current->token.label, "(")) + { + current++; + dat_converter_list_t* parsed_table = dat_parser_table(¤t); + if (!skip) + { + dat_converter_list_append(parsed_list, parsed_table); + skip = true; + } + else + dat_converter_list_free(parsed_table); + } + else + { + printf("%s:%d:%d: fatal error: Expected '(' found '%s'\n", + current->token.fname, + current->token.line_no, + current->token.column, + current->token.label); + dat_converter_exit(1); + } + state_is_key = true; + } + } + + return parsed_list; +} + + +static int value_provider(dat_converter_list_item_t** current_item, struct rmsgpack_dom_value* out) +{ + int i,j; +#if 1 + const char* ordered_keys[] = + { + "name", + "description", + "rom_name", + "size", + "users", + "releasemonth", + "releaseyear", + "rumble", + "analog", + + "famitsu_rating", + "edge_rating", + "edge_issue", + "edge_review", + + "enhancement_hw", + "barcode", + "esrb_rating", + "elspa_rating", + "pegi_rating", + "cero_rating", + "franchise", + + "developer", + "publisher", + "origin", + + "crc", + "md5", + "sha1", + "serial" + }; +#endif + out->type = RDT_MAP; + out->val.map.len = 0; + out->val.map.items = calloc(sizeof(ordered_keys)/sizeof(char*), sizeof(struct rmsgpack_dom_pair)); + + (*current_item)--; + dat_converter_list_t* list = (*current_item)->list; + if(!list) + return 1; + + if (list->type != DAT_CONVERTER_MAP_LIST) + dat_converter_exit(1); + + struct rmsgpack_dom_pair* current = out->val.map.items; + + for (i = 0; i < list->count; i++) + { + dat_converter_map_t* pair = &list->values[i].map; + + if (pair->type == DAT_CONVERTER_STRING_MAP) + { + current->key.type = RDT_STRING; + current->key.val.string.len = strlen(pair->key); + current->key.val.string.buff = strdup(pair->key); + + if (!strcmp(pair->key, "users") || + !strcmp(pair->key, "releasemonth") || + !strcmp(pair->key, "releaseyear") || + !strcmp(pair->key, "rumble") || + !strcmp(pair->key, "analog") || + !strcmp(pair->key, "famitsu_rating") || + !strcmp(pair->key, "edge_rating") || + !strcmp(pair->key, "edge_issue")) + { + current->value.type = RDT_UINT; + if (pair->value.string[strlen(pair->value.string) - 1] == '\?') + continue; + current->value.val.uint_ = (uint64_t)atoll(pair->value.string); + } + else if (!strcmp(pair->key, "serial")) + { + current->value.type = RDT_BINARY; + current->value.val.binary.len = strlen(pair->value.string); + current->value.val.binary.buff = strdup(pair->value.string); + } + else + { + current->value.type = RDT_STRING; + current->value.val.string.len = strlen(pair->value.string); + current->value.val.string.buff = strdup(pair->value.string); + } + + current++; + } + else if ((pair->type == DAT_CONVERTER_LIST_MAP) && (!strcmp(pair->key, "rom"))) + { + + if (i != (list->count - 1)) + continue; + + if (pair->value.list->type != DAT_CONVERTER_MAP_LIST) + dat_converter_exit(1); + + for (j = 0; j < pair->value.list->count; j++) + { + dat_converter_map_t* rom_pair = &pair->value.list->values[j].map; + if (rom_pair->type != DAT_CONVERTER_STRING_MAP) + dat_converter_exit(1); + + if (!strcmp(rom_pair->key, "name")) + { + current->key.type = RDT_STRING; + current->key.val.string.len = strlen("rom_name"); + current->key.val.string.buff = strdup("rom_name"); + + current->value.type = RDT_STRING; + current->value.val.string.len = strlen(rom_pair->value.string); + current->value.val.string.buff = strdup(rom_pair->value.string); + } + else + { + current->key.type = RDT_STRING; + current->key.val.string.len = strlen(rom_pair->key); + current->key.val.string.buff = strdup(rom_pair->key); + + if (!strcmp(rom_pair->key, "size")) + { + current->value.type = RDT_UINT; + current->value.val.uint_ = (uint64_t)atoll(rom_pair->value.string); + + } + else + { + current->value.type = RDT_BINARY; + current->value.val.binary.len = strlen(rom_pair->value.string); + current->value.val.binary.buff = strdup(rom_pair->value.string); + + if (!strcmp(rom_pair->key, "serial")) + { + current->value.val.binary.len = strlen(rom_pair->value.string); + current->value.val.binary.buff = strdup(rom_pair->value.string); + } + else + { + current->value.val.binary.len = strlen(rom_pair->value.string) / 2; + current->value.val.binary.buff = malloc(current->value.val.binary.len); + char* hex_char = rom_pair->value.string; + char* out_buff = current->value.val.binary.buff; + while (*hex_char && *(hex_char + 1)) + { + char val = 0; + if (*hex_char >= 'A' && *hex_char <= 'F') + val = *hex_char + 0xA - 'A'; + else if (*hex_char >= 'a' && *hex_char <= 'f') + val = *hex_char + 0xA - 'a'; + else if (*hex_char >= '0' && *hex_char <= '9') + val = *hex_char - '0'; + else + val = 0; + val <<= 4; + hex_char++; + if (*hex_char >= 'A' && *hex_char <= 'F') + val |= *hex_char + 0xA - 'A'; + else if (*hex_char >= 'a' && *hex_char <= 'f') + val |= *hex_char + 0xA - 'a'; + else if (*hex_char >= '0' && *hex_char <= '9') + val |= *hex_char - '0'; + + *out_buff++ = val; + hex_char++; + } + } + } + } + current++; + } + + } + else + exit(1); + } + + out->val.map.len = current - out->val.map.items; +#if 1 + /* re-order to match lua_converter */ + struct rmsgpack_dom_pair* ordered_pairs = calloc(out->val.map.len, sizeof(struct rmsgpack_dom_pair)); + struct rmsgpack_dom_pair* ordered_pairs_outp = ordered_pairs; + for(i = 0; i < (sizeof(ordered_keys)/sizeof(char*)); i++) + { + int j; + for(j = 0; j < out->val.map.len; j++) + { + if(!strcmp(ordered_keys[i], out->val.map.items[j].key.val.string.buff)) + { + *ordered_pairs_outp++ = out->val.map.items[j]; + break; + } + } + } + + free(out->val.map.items); + out->val.map.items = ordered_pairs; + out->val.map.len = ordered_pairs_outp - ordered_pairs; +#endif + return 0; +} + +int main(int argc, char** argv) +{ + const char* rdb_path; + const char* match_key; + RFILE* rdb_file; + int i; + + if (argc < 2) + { + printf("usage:\n%s [args ...]\n", *argv); + dat_converter_exit(1); + } + argc--; + argv++; + + rdb_path = *argv; + argc--; + argv++; + + if (argc > 1) + { + match_key = *argv; + argc--; + argv++; + } + + int dat_count = argc; + char** dat_buffers = (char**)malloc(dat_count * sizeof(*dat_buffers)); + dat_converter_list_t** dat_lexer_lists = (dat_converter_list_t**)malloc(dat_count * sizeof(*dat_lexer_lists)); + dat_converter_list_t** dat_parser_lists = (dat_converter_list_t**)malloc(dat_count * sizeof(*dat_parser_lists)); + + for (i = 0; i < dat_count; i++) + { + size_t dat_file_size; + FILE* dat_file = fopen(argv[i], "r"); + + if (!dat_file) + { + printf("could not open dat file '%s': %s\n", argv[i], strerror(errno)); + dat_converter_exit(1); + } + + fseek(dat_file, 0, SEEK_END); + dat_file_size = ftell(dat_file); + fseek(dat_file, 0, SEEK_SET); + dat_buffers[i] = (char*)malloc(dat_file_size + 1); + fread(dat_buffers[i], 1, dat_file_size, dat_file); + fclose(dat_file); + dat_buffers[i][dat_file_size] = '\0'; + + printf("Parsing dat file '%s'...\n", argv[i]); + dat_lexer_lists[i] = dat_converter_lexer(dat_buffers[i], argv[i]); + dat_parser_lists[i] = dat_parser(dat_lexer_lists[i]); + // dat_converter_token_list_dump(*dat_lexer_dst); + + } + + rdb_file = retro_fopen(rdb_path, RFILE_MODE_WRITE, -1); + + if (!rdb_file) + { + printf( + "Could not open destination file '%s': %s\n", + rdb_path, + strerror(errno) + ); + dat_converter_exit(1); + } + + dat_converter_list_item_t* current_item = &dat_parser_lists[0]->values[dat_parser_lists[0]->count]; + libretrodb_create(rdb_file, (libretrodb_value_provider)&value_provider, ¤t_item); + + retro_fclose(rdb_file); + + for (i = 0; i < dat_count; i++) + { + dat_converter_list_free(dat_lexer_lists[i]); + dat_converter_list_free(dat_parser_lists[i]); + free(dat_buffers[i]); + } + + free(dat_lexer_lists); + free(dat_parser_lists); + free(dat_buffers); + + return 0; +} + diff --git a/libretro-db/c_converter_test.sh b/libretro-db/c_converter_test.sh new file mode 100755 index 0000000000..c44ae773d2 --- /dev/null +++ b/libretro-db/c_converter_test.sh @@ -0,0 +1,75 @@ +#!/bin/sh + +DAT_dir=dat +lua_RDB_outdir=rdb_lua +c_RDB_outdir=rdb_c + +echo +echo "===========================================================" +echo "================== running LUA converter ==================" +echo "===========================================================" +echo + +rm -rf $lua_RDB_outdir +mkdir -p $lua_RDB_outdir + +for dat_file in $DAT_dir/*.dat ; do + name=`echo "$dat_file" | sed "s/${DAT_dir}\/*//"` + name=`echo "$name" | sed "s/\.dat//"` + ./lua_converter "$lua_RDB_outdir/$name.rdb" dat_converter.lua "$dat_file" +done + +echo +echo "===========================================================" +echo "=================== running C converter ===================" +echo "===========================================================" +echo + +rm -rf $c_RDB_outdir +mkdir -p $c_RDB_outdir + +for dat_file in $DAT_dir/*.dat ; do + name=`echo "$dat_file" | sed "s/${DAT_dir}\/*//"` + name=`echo "$name" | sed "s/\.dat//"` + ./c_converter "$c_RDB_outdir/$name.rdb" dat_converter.lua "$dat_file" +done + + +echo +echo "===========================================================" +echo "==================== comparing files =====================" +echo "===========================================================" +echo + +matches=0 +failed=0 + + +for lua_rdb_file in $lua_RDB_outdir/*.rdb ; do + name=`echo "$lua_rdb_file" | sed "s/${lua_RDB_outdir}\/*//"` + name=`echo "$name" | sed "s/\.rdb//"` + + files_differ=0 + diff "$c_RDB_outdir/$name.rdb" "$lua_RDB_outdir/$name.rdb" && files_differ=1 + if [ $files_differ = 0 ]; then + failed=$(( $failed + 1 )) + ls -la "$c_RDB_outdir/$name.rdb" + ls -la "$lua_RDB_outdir/$name.rdb" + else + matches=$(( $matches + 1 )) + fi +done + +echo +echo "===========================================================" +echo +echo "tested $(( $matches + $failed )) files: $matches match and $failed differ" +echo +if [ $failed = 0 ]; then + echo "test successful !!" +else + echo "test failed !!" +fi +echo +echo "===========================================================" +echo From 3f66270fcb341b3d2b1e1d6aa29a45de3791c4ab Mon Sep 17 00:00:00 2001 From: aliaspider Date: Sat, 23 Jan 2016 04:05:09 +0100 Subject: [PATCH 06/11] (libretro-db) c_converter: allow only one instance per key in a map list. --- libretro-db/c_converter.c | 106 ++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 50 deletions(-) diff --git a/libretro-db/c_converter.c b/libretro-db/c_converter.c index 8927f0e183..d4964f1574 100644 --- a/libretro-db/c_converter.c +++ b/libretro-db/c_converter.c @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include "libretrodb.h" @@ -79,45 +79,6 @@ dat_converter_list_t* dat_converter_list_create(dat_converter_list_enum type) return list; } -void dat_converter_list_append(dat_converter_list_t* dst, void* item) -{ - if (dst->count == dst->capacity) - { - dst->capacity <<= 1; - dst->values = realloc(dst->values, sizeof(*dst->values) * dst->capacity); - } - switch (dst->type) - { - case DAT_CONVERTER_TOKEN_LIST: - { - dat_converter_token_t* token = (dat_converter_token_t*) item; - dst->values[dst->count].token = *token; - break; - } - case DAT_CONVERTER_STRING_LIST: - { - char* str = (char*) item; - dst->values[dst->count].string = str; - break; - } - case DAT_CONVERTER_MAP_LIST: - { - dat_converter_map_t* map = (dat_converter_map_t*) item; - dst->values[dst->count].map = *map; - break; - } - case DAT_CONVERTER_LIST_LIST: - { - dat_converter_list_t* list = (dat_converter_list_t*) item; - dst->values[dst->count].list = list; - break; - } - default: - return; - } - dst->count++; -} - void dat_converter_list_free(dat_converter_list_t* list) { if(!list) @@ -143,6 +104,59 @@ void dat_converter_list_free(dat_converter_list_t* list) free(list); } +void dat_converter_list_append(dat_converter_list_t* dst, void* item) +{ + if (dst->count == dst->capacity) + { + dst->capacity <<= 1; + dst->values = realloc(dst->values, sizeof(*dst->values) * dst->capacity); + } + switch (dst->type) + { + case DAT_CONVERTER_TOKEN_LIST: + { + dat_converter_token_t* token = (dat_converter_token_t*) item; + dst->values[dst->count].token = *token; + break; + } + case DAT_CONVERTER_STRING_LIST: + { + char* str = (char*) item; + dst->values[dst->count].string = str; + break; + } + case DAT_CONVERTER_MAP_LIST: + { + int i; + dat_converter_map_t* map = (dat_converter_map_t*) item; + for(i = 0; i < dst->count; i++) + { + if(map->key && *map->key && !strcmp(dst->values[i].map.key, map->key)) + break; + } + if(i == dst->count) + dst->values[dst->count].map = *map; + else + { + if(dst->values[i].map.type == DAT_CONVERTER_LIST_MAP) + dat_converter_list_free(dst->values[i].map.value.list); + dst->values[i].map = *map; + return; + } + break; + } + case DAT_CONVERTER_LIST_LIST: + { + dat_converter_list_t* list = (dat_converter_list_t*) item; + dst->values[dst->count].list = list; + break; + } + default: + return; + } + dst->count++; +} + void dat_converter_token_list_dump(dat_converter_list_t* list) { int i; @@ -231,7 +245,6 @@ dat_converter_list_t* dat_converter_lexer(char* src, const char* dat_path) dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token) { - bool state_is_key = true; dat_converter_list_t* parsed_table = dat_converter_list_create(DAT_CONVERTER_MAP_LIST); char* key = NULL; @@ -240,7 +253,7 @@ dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token) while (current->token.label) { - if (state_is_key) + if (!key) { if (!strcmp(current->token.label, ")")) { @@ -259,7 +272,6 @@ dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token) else { key = current->token.label; - state_is_key = false; current++; } } @@ -292,7 +304,7 @@ dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token) dat_converter_list_append(parsed_table, &map); current++; } - state_is_key = true; + key = NULL; } } @@ -449,10 +461,6 @@ static int value_provider(dat_converter_list_item_t** current_item, struct rmsgp } else if ((pair->type == DAT_CONVERTER_LIST_MAP) && (!strcmp(pair->key, "rom"))) { - - if (i != (list->count - 1)) - continue; - if (pair->value.list->type != DAT_CONVERTER_MAP_LIST) dat_converter_exit(1); @@ -487,8 +495,6 @@ static int value_provider(dat_converter_list_item_t** current_item, struct rmsgp else { current->value.type = RDT_BINARY; - current->value.val.binary.len = strlen(rom_pair->value.string); - current->value.val.binary.buff = strdup(rom_pair->value.string); if (!strcmp(rom_pair->key, "serial")) { From 964f0f7926123c31292db140e5f1540db9d57174 Mon Sep 17 00:00:00 2001 From: aliaspider Date: Sat, 23 Jan 2016 04:10:19 +0100 Subject: [PATCH 07/11] (libretro-db) Makefile: add a DEBUG build option --- libretro-db/Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libretro-db/Makefile b/libretro-db/Makefile index e0c5b6eda0..75e9de48e9 100644 --- a/libretro-db/Makefile +++ b/libretro-db/Makefile @@ -1,9 +1,15 @@ -CFLAGS = -g -O2 -Wall +DEBUG = 0 LIBRETRO_COMMON_DIR := ../libretro-common INCFLAGS = -I. -I$(LIBRETRO_COMMON_DIR)/include TARGETS = rmsgpack_test libretrodb_tool plain_converter lua_converter c_converter +ifeq ($(DEBUG), 1) +CFLAGS = -g -O0 -Wall +else +CFLAGS = -g -O2 -Wall -DNDEBUG +endif + LUA_CONVERTER_C = \ rmsgpack.c \ rmsgpack_dom.c \ From 1fe10855e92e04355c6b40197f733d42390f3a45 Mon Sep 17 00:00:00 2001 From: aliaspider Date: Sat, 23 Jan 2016 04:26:19 +0100 Subject: [PATCH 08/11] nit. --- libretro-db/Makefile | 64 +++++++++++++-------------- libretro-db/lua_common.c | 96 ++++++++++++++++++++-------------------- 2 files changed, 80 insertions(+), 80 deletions(-) diff --git a/libretro-db/Makefile b/libretro-db/Makefile index 75e9de48e9..b81797d92f 100644 --- a/libretro-db/Makefile +++ b/libretro-db/Makefile @@ -12,12 +12,12 @@ endif LUA_CONVERTER_C = \ rmsgpack.c \ - rmsgpack_dom.c \ - lua_common.c \ - libretrodb.c \ - bintree.c \ - query.c \ - lua_converter.c \ + rmsgpack_dom.c \ + lua_common.c \ + libretrodb.c \ + bintree.c \ + query.c \ + lua_converter.c \ $(LIBRETRO_COMMON_DIR)/compat/compat_fnmatch.c \ $(LIBRETRO_COMMON_DIR)/file/retro_file.c \ $(LIBRETRO_COMMON_DIR)/compat/compat_strl.c @@ -28,10 +28,10 @@ PLAIN_CONVERTER_C = \ lexer.c \ parser.c \ rmsgpack.c \ - rmsgpack_dom.c \ - libretrodb.c \ - bintree.c \ - query.c \ + rmsgpack_dom.c \ + libretrodb.c \ + bintree.c \ + query.c \ plain_converter.c \ $(LIBRETRO_COMMON_DIR)/hash/rhash.c \ $(LIBRETRO_COMMON_DIR)/compat/compat_fnmatch.c \ @@ -42,10 +42,10 @@ PLAIN_CONVERTER_OBJS := $(PLAIN_CONVERTER_C:.c=.o) C_CONVERTER_C = \ rmsgpack.c \ - rmsgpack_dom.c \ - libretrodb.c \ - bintree.c \ - query.c \ + rmsgpack_dom.c \ + libretrodb.c \ + bintree.c \ + query.c \ c_converter.c \ $(LIBRETRO_COMMON_DIR)/hash/rhash.c \ $(LIBRETRO_COMMON_DIR)/compat/compat_fnmatch.c \ @@ -55,28 +55,28 @@ C_CONVERTER_C = \ C_CONVERTER_OBJS := $(C_CONVERTER_C:.c=.o) RARCHDB_TOOL_C = \ - rmsgpack.c \ - rmsgpack_dom.c \ - libretrodb_tool.c \ - bintree.c \ - query.c \ - libretrodb.c \ - $(LIBRETRO_COMMON_DIR)/compat/compat_fnmatch.c \ - $(LIBRETRO_COMMON_DIR)/file/retro_file.c \ - $(LIBRETRO_COMMON_DIR)/compat/compat_strl.c + rmsgpack.c \ + rmsgpack_dom.c \ + libretrodb_tool.c \ + bintree.c \ + query.c \ + libretrodb.c \ + $(LIBRETRO_COMMON_DIR)/compat/compat_fnmatch.c \ + $(LIBRETRO_COMMON_DIR)/file/retro_file.c \ + $(LIBRETRO_COMMON_DIR)/compat/compat_strl.c RARCHDB_TOOL_OBJS := $(RARCHDB_TOOL_C:.c=.o) TESTLIB_C = \ - testlib.c \ - query.c \ - libretrodb.c \ - bintree.c \ - rmsgpack.c \ - rmsgpack_dom.c \ - $(LIBRETRO_COMMON_DIR)/compat/compat_fnmatch.c \ - $(LIBRETRO_COMMON_DIR)/file/retro_file.c \ - $(LIBRETRO_COMMON_DIR)/compat/compat_strl.c + testlib.c \ + query.c \ + libretrodb.c \ + bintree.c \ + rmsgpack.c \ + rmsgpack_dom.c \ + $(LIBRETRO_COMMON_DIR)/compat/compat_fnmatch.c \ + $(LIBRETRO_COMMON_DIR)/file/retro_file.c \ + $(LIBRETRO_COMMON_DIR)/compat/compat_strl.c TESTLIB_OBJS := $(TESTLIB_C:.c=.o) diff --git a/libretro-db/lua_common.c b/libretro-db/lua_common.c index 1422e68813..d7e87bee75 100644 --- a/libretro-db/lua_common.c +++ b/libretro-db/lua_common.c @@ -95,59 +95,59 @@ set_nil: lua_pop(L, 1); } #if 1 - /* re-order to avoid random output each run */ - struct rmsgpack_dom_pair* ordered_pairs = calloc(out->val.map.len, sizeof(struct rmsgpack_dom_pair)); - struct rmsgpack_dom_pair* ordered_pairs_outp = ordered_pairs; - const char* ordered_keys[] = + /* re-order to avoid random output each run */ + struct rmsgpack_dom_pair* ordered_pairs = calloc(out->val.map.len, sizeof(struct rmsgpack_dom_pair)); + struct rmsgpack_dom_pair* ordered_pairs_outp = ordered_pairs; + const char* ordered_keys[] = + { + "name", + "description", + "rom_name", + "size", + "users", + "releasemonth", + "releaseyear", + "rumble", + "analog", + + "famitsu_rating", + "edge_rating", + "edge_issue", + "edge_review", + + "enhancement_hw", + "barcode", + "esrb_rating", + "elspa_rating", + "pegi_rating", + "cero_rating", + "franchise", + + "developer", + "publisher", + "origin", + + "crc", + "md5", + "sha1", + "serial" + }; + for(i = 0; i < (sizeof(ordered_keys)/sizeof(char*)); i++) + { + int j; + for(j = 0; j < out->val.map.len; j++) { - "name", - "description", - "rom_name", - "size", - "users", - "releasemonth", - "releaseyear", - "rumble", - "analog", - - "famitsu_rating", - "edge_rating", - "edge_issue", - "edge_review", - - "enhancement_hw", - "barcode", - "esrb_rating", - "elspa_rating", - "pegi_rating", - "cero_rating", - "franchise", - - "developer", - "publisher", - "origin", - - "crc", - "md5", - "sha1", - "serial" - }; - for(i = 0; i < (sizeof(ordered_keys)/sizeof(char*)); i++) - { - int j; - for(j = 0; j < out->val.map.len; j++) + if(!strcmp(ordered_keys[i], out->val.map.items[j].key.val.string.buff)) { - if(!strcmp(ordered_keys[i], out->val.map.items[j].key.val.string.buff)) - { - *ordered_pairs_outp++ = out->val.map.items[j]; - break; - } + *ordered_pairs_outp++ = out->val.map.items[j]; + break; } } + } - free(out->val.map.items); - out->val.map.items = ordered_pairs; - out->val.map.len = ordered_pairs_outp - ordered_pairs; + free(out->val.map.items); + out->val.map.items = ordered_pairs; + out->val.map.len = ordered_pairs_outp - ordered_pairs; #endif rv = 0; From c95205dbd320207bbf6dce0b5708826fd949c946 Mon Sep 17 00:00:00 2001 From: aliaspider Date: Sat, 23 Jan 2016 06:21:52 +0100 Subject: [PATCH 09/11] (libretro-db) c_converter: add support for merging multiple dat files into a single rdb file with a match-key. --- libretro-db/c_converter.c | 160 +++++++++++++++++++++++--------- libretro-db/c_converter_test.sh | 99 ++++++++++---------- 2 files changed, 165 insertions(+), 94 deletions(-) diff --git a/libretro-db/c_converter.c b/libretro-db/c_converter.c index d4964f1574..71d7dc6273 100644 --- a/libretro-db/c_converter.c +++ b/libretro-db/c_converter.c @@ -32,7 +32,7 @@ typedef enum typedef struct { - char* label; + const char* label; int line_no; int column; const char* fname; @@ -44,11 +44,11 @@ typedef union dat_converter_list_item_t dat_converter_list_item_t; struct dat_converter_map_t { - char* key; + const char* key; dat_converter_map_enum type; union { - char* string; + const char* string; dat_converter_list_t* list; } value; }; @@ -63,7 +63,7 @@ struct dat_converter_list_t union dat_converter_list_item_t { - char* string; + const char* string; dat_converter_map_t map; dat_converter_token_t token; dat_converter_list_t* list; @@ -74,14 +74,14 @@ dat_converter_list_t* dat_converter_list_create(dat_converter_list_enum type) dat_converter_list_t* list = malloc(sizeof(*list)); list->type = type; list->count = 0; - list->capacity = (1 << 8); + list->capacity = (1 << 2); list->values = (dat_converter_list_item_t*)malloc(sizeof(*list->values) * list->capacity); return list; } void dat_converter_list_free(dat_converter_list_t* list) { - if(!list) + if (!list) return; switch (list->type) { @@ -129,16 +129,21 @@ void dat_converter_list_append(dat_converter_list_t* dst, void* item) { int i; dat_converter_map_t* map = (dat_converter_map_t*) item; - for(i = 0; i < dst->count; i++) + if(!map->key) { - if(map->key && *map->key && !strcmp(dst->values[i].map.key, map->key)) + dst->values[dst->count].map = *map; + break; + } + for (i = 0; i < dst->count; i++) + { + if (map->key && dst->values[i].map.key && !strcmp(dst->values[i].map.key, map->key)) break; } - if(i == dst->count) + if (i == dst->count) dst->values[dst->count].map = *map; else { - if(dst->values[i].map.type == DAT_CONVERTER_LIST_MAP) + if (dst->values[i].map.type == DAT_CONVERTER_LIST_MAP) dat_converter_list_free(dst->values[i].map.value.list); dst->values[i].map = *map; return; @@ -246,7 +251,7 @@ dat_converter_list_t* dat_converter_lexer(char* src, const char* dat_path) dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token) { dat_converter_list_t* parsed_table = dat_converter_list_create(DAT_CONVERTER_MAP_LIST); - char* key = NULL; + const char* key = NULL; dat_converter_list_item_t* current = *start_token; @@ -292,7 +297,6 @@ dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token) current->token.fname, current->token.line_no, current->token.column); - // fflush(stdout); dat_converter_exit(1); } else @@ -319,21 +323,74 @@ dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token) return NULL; } -dat_converter_list_t* dat_parser(dat_converter_list_t* lexer_list) +const char* dat_converter_get_match_key(dat_converter_list_t* list, const char* match_key) { - bool state_is_key = true; + int i; + const char* res; + char* key; + char* dot; + + if (list->type != DAT_CONVERTER_MAP_LIST) + return NULL; + + key = strdup(match_key); + dot = strchr(key, '.'); + + if (dot) + { + *dot = '\0'; + for (i = 0; i < list->count; i++) + if (!strcmp(list->values[i].map.key, key)) + { + if (list->values[i].map.type == DAT_CONVERTER_LIST_MAP) + { + res = dat_converter_get_match_key(list->values[i].map.value.list, dot + 1); + free(key); + return res; + } + break; + } + } + else + { + for (i = 0; i < list->count; i++) + if (!strcmp(list->values[i].map.key, key)) + { + if ((list->values[i].map.type == DAT_CONVERTER_STRING_MAP)) + { + free(key); + return list->values[i].map.value.string; + } + break; + } + } + + return NULL; +} + +dat_converter_list_t* dat_parser(dat_converter_list_t* target, dat_converter_list_t* lexer_list, const char* match_key) +{ + const char* key = NULL; bool skip = true; - dat_converter_list_t* parsed_list = dat_converter_list_create(DAT_CONVERTER_LIST_LIST); dat_converter_list_item_t* current = lexer_list->values; - dat_converter_list_append(parsed_list, NULL); + + if (!target) + { + dat_converter_map_t map; + target = dat_converter_list_create(DAT_CONVERTER_MAP_LIST); + map.key = NULL; + map.type = DAT_CONVERTER_LIST_MAP; + map.value.list = NULL; + dat_converter_list_append(target, &map); + } while (current->token.label) { - if (state_is_key) + if (!key) { if (!strcmp(current->token.label, "game")) skip = false; - state_is_key = false; + key = current->token.label; current++; } else @@ -341,14 +398,28 @@ dat_converter_list_t* dat_parser(dat_converter_list_t* lexer_list) if (!strcmp(current->token.label, "(")) { current++; - dat_converter_list_t* parsed_table = dat_parser_table(¤t); + dat_converter_map_t map; + map.type = DAT_CONVERTER_LIST_MAP; + map.key = NULL; + map.value.list = dat_parser_table(¤t); if (!skip) { - dat_converter_list_append(parsed_list, parsed_table); + if (match_key) + { + map.key = dat_converter_get_match_key(map.value.list, match_key); + if(!map.key) + { + printf("missing match key '%s' in one of the entries\n", match_key); + dat_converter_exit(1); + } + } + dat_converter_list_append(target, &map); skip = true; } else - dat_converter_list_free(parsed_table); + dat_converter_list_free(map.value.list); + + key = NULL; } else { @@ -359,17 +430,15 @@ dat_converter_list_t* dat_parser(dat_converter_list_t* lexer_list) current->token.label); dat_converter_exit(1); } - state_is_key = true; } } - - return parsed_list; + return target; } static int value_provider(dat_converter_list_item_t** current_item, struct rmsgpack_dom_value* out) { - int i,j; + int i, j; #if 1 const char* ordered_keys[] = { @@ -408,15 +477,18 @@ static int value_provider(dat_converter_list_item_t** current_item, struct rmsgp #endif out->type = RDT_MAP; out->val.map.len = 0; - out->val.map.items = calloc(sizeof(ordered_keys)/sizeof(char*), sizeof(struct rmsgpack_dom_pair)); + out->val.map.items = calloc(sizeof(ordered_keys) / sizeof(char*), sizeof(struct rmsgpack_dom_pair)); (*current_item)--; - dat_converter_list_t* list = (*current_item)->list; - if(!list) + + assert((*current_item)->map.type == DAT_CONVERTER_LIST_MAP); + + dat_converter_list_t* list = (*current_item)->map.value.list; + + if (!list) return 1; - if (list->type != DAT_CONVERTER_MAP_LIST) - dat_converter_exit(1); + assert(list->type == DAT_CONVERTER_MAP_LIST); struct rmsgpack_dom_pair* current = out->val.map.items; @@ -461,14 +533,12 @@ static int value_provider(dat_converter_list_item_t** current_item, struct rmsgp } else if ((pair->type == DAT_CONVERTER_LIST_MAP) && (!strcmp(pair->key, "rom"))) { - if (pair->value.list->type != DAT_CONVERTER_MAP_LIST) - dat_converter_exit(1); + assert(pair->value.list->type == DAT_CONVERTER_MAP_LIST); for (j = 0; j < pair->value.list->count; j++) { dat_converter_map_t* rom_pair = &pair->value.list->values[j].map; - if (rom_pair->type != DAT_CONVERTER_STRING_MAP) - dat_converter_exit(1); + assert(rom_pair->type == DAT_CONVERTER_STRING_MAP); if (!strcmp(rom_pair->key, "name")) { @@ -505,7 +575,7 @@ static int value_provider(dat_converter_list_item_t** current_item, struct rmsgp { current->value.val.binary.len = strlen(rom_pair->value.string) / 2; current->value.val.binary.buff = malloc(current->value.val.binary.len); - char* hex_char = rom_pair->value.string; + const char* hex_char = rom_pair->value.string; char* out_buff = current->value.val.binary.buff; while (*hex_char && *(hex_char + 1)) { @@ -538,7 +608,7 @@ static int value_provider(dat_converter_list_item_t** current_item, struct rmsgp } else - exit(1); + assert(0); } out->val.map.len = current - out->val.map.items; @@ -546,12 +616,12 @@ static int value_provider(dat_converter_list_item_t** current_item, struct rmsgp /* re-order to match lua_converter */ struct rmsgpack_dom_pair* ordered_pairs = calloc(out->val.map.len, sizeof(struct rmsgpack_dom_pair)); struct rmsgpack_dom_pair* ordered_pairs_outp = ordered_pairs; - for(i = 0; i < (sizeof(ordered_keys)/sizeof(char*)); i++) + for (i = 0; i < (sizeof(ordered_keys) / sizeof(char*)); i++) { int j; - for(j = 0; j < out->val.map.len; j++) + for (j = 0; j < out->val.map.len; j++) { - if(!strcmp(ordered_keys[i], out->val.map.items[j].key.val.string.buff)) + if (!strcmp(ordered_keys[i], out->val.map.items[j].key.val.string.buff)) { *ordered_pairs_outp++ = out->val.map.items[j]; break; @@ -569,7 +639,7 @@ static int value_provider(dat_converter_list_item_t** current_item, struct rmsgp int main(int argc, char** argv) { const char* rdb_path; - const char* match_key; + const char* match_key = NULL; RFILE* rdb_file; int i; @@ -595,7 +665,7 @@ int main(int argc, char** argv) int dat_count = argc; char** dat_buffers = (char**)malloc(dat_count * sizeof(*dat_buffers)); dat_converter_list_t** dat_lexer_lists = (dat_converter_list_t**)malloc(dat_count * sizeof(*dat_lexer_lists)); - dat_converter_list_t** dat_parser_lists = (dat_converter_list_t**)malloc(dat_count * sizeof(*dat_parser_lists)); + dat_converter_list_t* dat_parser_list = NULL; for (i = 0; i < dat_count; i++) { @@ -618,7 +688,7 @@ int main(int argc, char** argv) printf("Parsing dat file '%s'...\n", argv[i]); dat_lexer_lists[i] = dat_converter_lexer(dat_buffers[i], argv[i]); - dat_parser_lists[i] = dat_parser(dat_lexer_lists[i]); + dat_parser_list = dat_parser(dat_parser_list, dat_lexer_lists[i], match_key); // dat_converter_token_list_dump(*dat_lexer_dst); } @@ -635,7 +705,7 @@ int main(int argc, char** argv) dat_converter_exit(1); } - dat_converter_list_item_t* current_item = &dat_parser_lists[0]->values[dat_parser_lists[0]->count]; + dat_converter_list_item_t* current_item = &dat_parser_list->values[dat_parser_list->count]; libretrodb_create(rdb_file, (libretrodb_value_provider)&value_provider, ¤t_item); retro_fclose(rdb_file); @@ -643,13 +713,13 @@ int main(int argc, char** argv) for (i = 0; i < dat_count; i++) { dat_converter_list_free(dat_lexer_lists[i]); - dat_converter_list_free(dat_parser_lists[i]); free(dat_buffers[i]); } free(dat_lexer_lists); - free(dat_parser_lists); free(dat_buffers); + dat_converter_list_free(dat_parser_list); + return 0; } diff --git a/libretro-db/c_converter_test.sh b/libretro-db/c_converter_test.sh index c44ae773d2..bc003215c5 100755 --- a/libretro-db/c_converter_test.sh +++ b/libretro-db/c_converter_test.sh @@ -4,20 +4,21 @@ DAT_dir=dat lua_RDB_outdir=rdb_lua c_RDB_outdir=rdb_c -echo -echo "===========================================================" -echo "================== running LUA converter ==================" -echo "===========================================================" -echo +#echo +#echo "===========================================================" +#echo "================== running LUA converter ==================" +#echo "===========================================================" +#echo -rm -rf $lua_RDB_outdir -mkdir -p $lua_RDB_outdir +#rm -rf $lua_RDB_outdir +#mkdir -p $lua_RDB_outdir -for dat_file in $DAT_dir/*.dat ; do - name=`echo "$dat_file" | sed "s/${DAT_dir}\/*//"` - name=`echo "$name" | sed "s/\.dat//"` - ./lua_converter "$lua_RDB_outdir/$name.rdb" dat_converter.lua "$dat_file" -done +#for dat_file in $DAT_dir/*.dat ; do +# name=`echo "$dat_file" | sed "s/${DAT_dir}\/*//"` +# name=`echo "$name" | sed "s/\.dat//"` +# ./lua_converter "$lua_RDB_outdir/$name.rdb" dat_converter.lua "$dat_file" +#done +#./lua_converter "$lua_RDB_outdir/merged.rdb" dat_converter.lua rom.crc $DAT_dir/N*.dat echo echo "===========================================================" @@ -31,45 +32,45 @@ mkdir -p $c_RDB_outdir for dat_file in $DAT_dir/*.dat ; do name=`echo "$dat_file" | sed "s/${DAT_dir}\/*//"` name=`echo "$name" | sed "s/\.dat//"` - ./c_converter "$c_RDB_outdir/$name.rdb" dat_converter.lua "$dat_file" + ./c_converter "$c_RDB_outdir/$name.rdb" "$dat_file" done +#./c_converter "$c_RDB_outdir/merged.rdb" rom.crc $DAT_dir/N*.dat + +#echo +#echo "===========================================================" +#echo "==================== comparing files =====================" +#echo "===========================================================" +#echo + +#matches=0 +#failed=0 -echo -echo "===========================================================" -echo "==================== comparing files =====================" -echo "===========================================================" -echo +#for lua_rdb_file in $lua_RDB_outdir/*.rdb ; do +# name=`echo "$lua_rdb_file" | sed "s/${lua_RDB_outdir}\/*//"` +# name=`echo "$name" | sed "s/\.rdb//"` -matches=0 -failed=0 +# files_differ=0 +# diff "$c_RDB_outdir/$name.rdb" "$lua_RDB_outdir/$name.rdb" && files_differ=1 +# if [ $files_differ = 0 ]; then +# failed=$(( $failed + 1 )) +# ls -la "$c_RDB_outdir/$name.rdb" +# ls -la "$lua_RDB_outdir/$name.rdb" +# else +# matches=$(( $matches + 1 )) +# fi +#done - -for lua_rdb_file in $lua_RDB_outdir/*.rdb ; do - name=`echo "$lua_rdb_file" | sed "s/${lua_RDB_outdir}\/*//"` - name=`echo "$name" | sed "s/\.rdb//"` - - files_differ=0 - diff "$c_RDB_outdir/$name.rdb" "$lua_RDB_outdir/$name.rdb" && files_differ=1 - if [ $files_differ = 0 ]; then - failed=$(( $failed + 1 )) - ls -la "$c_RDB_outdir/$name.rdb" - ls -la "$lua_RDB_outdir/$name.rdb" - else - matches=$(( $matches + 1 )) - fi -done - -echo -echo "===========================================================" -echo -echo "tested $(( $matches + $failed )) files: $matches match and $failed differ" -echo -if [ $failed = 0 ]; then - echo "test successful !!" -else - echo "test failed !!" -fi -echo -echo "===========================================================" -echo +#echo +#echo "===========================================================" +#echo +#echo "tested $(( $matches + $failed )) files: $matches match and $failed differ" +#echo +#if [ $failed = 0 ]; then +# echo "test successful !!" +#else +# echo "test failed !!" +#fi +#echo +#echo "===========================================================" +#echo From 8f8b123d065490bb1b002df4c354e34ee94f2dc1 Mon Sep 17 00:00:00 2001 From: aliaspider Date: Sat, 23 Jan 2016 06:42:40 +0100 Subject: [PATCH 10/11] (libretro-db) c_converter: re-enable tests --- libretro-db/c_converter_test.sh | 90 ++++++++++++++++----------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/libretro-db/c_converter_test.sh b/libretro-db/c_converter_test.sh index bc003215c5..9243327379 100755 --- a/libretro-db/c_converter_test.sh +++ b/libretro-db/c_converter_test.sh @@ -4,20 +4,20 @@ DAT_dir=dat lua_RDB_outdir=rdb_lua c_RDB_outdir=rdb_c -#echo -#echo "===========================================================" -#echo "================== running LUA converter ==================" -#echo "===========================================================" -#echo +echo +echo "===========================================================" +echo "================== running LUA converter ==================" +echo "===========================================================" +echo -#rm -rf $lua_RDB_outdir -#mkdir -p $lua_RDB_outdir +rm -rf $lua_RDB_outdir +mkdir -p $lua_RDB_outdir -#for dat_file in $DAT_dir/*.dat ; do -# name=`echo "$dat_file" | sed "s/${DAT_dir}\/*//"` -# name=`echo "$name" | sed "s/\.dat//"` -# ./lua_converter "$lua_RDB_outdir/$name.rdb" dat_converter.lua "$dat_file" -#done +for dat_file in $DAT_dir/*.dat ; do + name=`echo "$dat_file" | sed "s/${DAT_dir}\/*//"` + name=`echo "$name" | sed "s/\.dat//"` + ./lua_converter "$lua_RDB_outdir/$name.rdb" dat_converter.lua "$dat_file" +done #./lua_converter "$lua_RDB_outdir/merged.rdb" dat_converter.lua rom.crc $DAT_dir/N*.dat echo @@ -36,41 +36,41 @@ for dat_file in $DAT_dir/*.dat ; do done #./c_converter "$c_RDB_outdir/merged.rdb" rom.crc $DAT_dir/N*.dat -#echo -#echo "===========================================================" -#echo "==================== comparing files =====================" -#echo "===========================================================" -#echo +echo +echo "===========================================================" +echo "==================== comparing files =====================" +echo "===========================================================" +echo -#matches=0 -#failed=0 +matches=0 +failed=0 -#for lua_rdb_file in $lua_RDB_outdir/*.rdb ; do -# name=`echo "$lua_rdb_file" | sed "s/${lua_RDB_outdir}\/*//"` -# name=`echo "$name" | sed "s/\.rdb//"` +for lua_rdb_file in $lua_RDB_outdir/*.rdb ; do + name=`echo "$lua_rdb_file" | sed "s/${lua_RDB_outdir}\/*//"` + name=`echo "$name" | sed "s/\.rdb//"` -# files_differ=0 -# diff "$c_RDB_outdir/$name.rdb" "$lua_RDB_outdir/$name.rdb" && files_differ=1 -# if [ $files_differ = 0 ]; then -# failed=$(( $failed + 1 )) -# ls -la "$c_RDB_outdir/$name.rdb" -# ls -la "$lua_RDB_outdir/$name.rdb" -# else -# matches=$(( $matches + 1 )) -# fi -#done + files_differ=0 + diff "$c_RDB_outdir/$name.rdb" "$lua_RDB_outdir/$name.rdb" && files_differ=1 + if [ $files_differ = 0 ]; then + failed=$(( $failed + 1 )) + ls -la "$c_RDB_outdir/$name.rdb" + ls -la "$lua_RDB_outdir/$name.rdb" + else + matches=$(( $matches + 1 )) + fi +done -#echo -#echo "===========================================================" -#echo -#echo "tested $(( $matches + $failed )) files: $matches match and $failed differ" -#echo -#if [ $failed = 0 ]; then -# echo "test successful !!" -#else -# echo "test failed !!" -#fi -#echo -#echo "===========================================================" -#echo +echo +echo "===========================================================" +echo +echo "tested $(( $matches + $failed )) files: $matches match and $failed differ" +echo +if [ $failed = 0 ]; then + echo "test successful !!" +else + echo "test failed !!" +fi +echo +echo "===========================================================" +echo From f7c8304dc3fffc1b39db2a59a6a72064047911bb Mon Sep 17 00:00:00 2001 From: aliaspider Date: Sat, 23 Jan 2016 07:49:51 +0100 Subject: [PATCH 11/11] (libretro-db) Makefile: fix the testlib.so target --- libretro-db/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libretro-db/Makefile b/libretro-db/Makefile index b81797d92f..52147089b2 100644 --- a/libretro-db/Makefile +++ b/libretro-db/Makefile @@ -112,11 +112,12 @@ libretrodb_tool: $(RARCHDB_TOOL_OBJS) rmsgpack_test: $(RMSGPACK_OBJS) $(CC) $(INCFLAGS) $(RMSGPACK_OBJS) -g -o $@ -testlib.so: $(TESTLIB_OBJS) +testlib.so: CFLAGS += -fPIC +testlib.so: clean $(TESTLIB_OBJS) $(CC) $(INCFLAGS) $(TESTLIB_FLAGS) $(TESTLIB_OBJS) -o $@ check: testlib.so tests.lua lua ./tests.lua clean: - rm -rf $(TARGETS) $(PLAIN_CONVERTER_OBJS) $(LUA_CONVERTER_OBJS) $(C_CONVERTER_OBJS) $(RARCHDB_TOOL_OBJS) $(RMSGPACK_OBJS) $(TESTLIB_OBJS) + rm -rf $(TARGETS) $(PLAIN_CONVERTER_OBJS) $(LUA_CONVERTER_OBJS) $(C_CONVERTER_OBJS) $(RARCHDB_TOOL_OBJS) $(RMSGPACK_OBJS) $(TESTLIB_OBJS) testlib.so