ShuriZma
/
RetroArch
mirror of https://github.com/libretro/RetroArch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add rarchdb

This commit is contained in:
twinaphex 2014-12-30 21:02:15 +01:00
parent ce57697629
commit c16278f027
18 changed files with 2548 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
rarchdb/.ycm_extra_conf.py Normal file
View File

@ -0,0 +1,115 @@
import os
import ycm_core
# Defaults, if no database can be found.
defaultsc = [
'gcc',
'-Wno-long-long',
'-Wno-variadic-macros',
'-pthread'
'-std=c99',
]
defaultscpp = [
'c'
'-Wno-long-long',
'-Wno-variadic-macros',
'-pthread'
'-std=c99',
]
# Things that must be included.
entered_flags = ['-fdelayed-template-parsing']
#
def DirectoryOfThisScript():
return os.path.dirname( os.path.abspath( __file__ ) )
# Find all compilation databases in the build directory and load them.
def MakeRelativePathsInFlagsAbsolute( flags, working_directory ):
if not working_directory:
return list( flags )
new_flags = []
make_next_absolute = False
path_flags = [ '-isystem', '-I', '-iquote', '--sysroot=' ]
for flag in flags:
new_flag = flag
if make_next_absolute:
make_next_absolute = False
if not flag.startswith( '/' ):
new_flag = os.path.join( working_directory, flag )
for path_flag in path_flags:
if flag == path_flag:
make_next_absolute = True
break
if flag.startswith( path_flag ):
path = flag[ len( path_flag ): ]
new_flag = path_flag + os.path.join( working_directory, path )
break
if new_flag:
new_flags.append( new_flag )
return new_flags
def FlagsForFile( filename ):
# Search through all parent directories for a directory named 'build'
databases = []
path_focus = os.path.dirname(filename)
while len(path_focus) > 1:
for f in os.listdir(path_focus):
compilation_database_folder = path_focus + "/" + f
for r,d,f in os.walk(compilation_database_folder):
for files in f:
if files == 'compile_commands.json':
databases += [ycm_core.CompilationDatabase( r )]
path_focus = os.path.dirname(os.path.dirname(path_focus))
# Use a header's source file database for completion.
filetype_flags = []
if filename.endswith(".h"):
for f in os.listdir(os.path.dirname(filename)):
if filename.replace(".h",".cpp").find(f) != -1:
filename = filename.replace(".h",".cpp")
break
if filename.replace(".h",".c").find(f) != -1:
filename = filename.replace(".h",".c")
break
elif filename.endswith(".hpp"):
for f in os.listdir(os.path.dirname(filename)):
if filename.replace(".hpp",".cpp").find(f) != -1:
filename = filename.replace(".hpp",".cpp")
break
# Get the compile commands
final_flags = []
# If possible, from the database.
if len(databases) > 0:
for database in databases:
compilation_info = database.GetCompilationInfoForFile( filename )
fromfile_flags = MakeRelativePathsInFlagsAbsolute(
compilation_info.compiler_flags_,
compilation_info.compiler_working_dir_ )
final_flags += fromfile_flags
# If not, set some sane defaults.
else:
relative_to = DirectoryOfThisScript()
final_flags += MakeRelativePathsInFlagsAbsolute( flags, relative_to )
if not final_flags:
if filename.endswith(".c"):
final_flags = defaultsc
elif filename.endswith(".cpp"):
final_flags = defaultscpp
# This allows header files to be parsed according to their parent source
final_flags = filetype_flags + final_flags
# For things that must be included regardless:
final_flags += entered_flags
return {
'flags': final_flags,
'do_cache': True
}

31
rarchdb/Makefile Normal file
View File

@ -0,0 +1,31 @@
CFLAGS=-std=c99 -g
DAT_CONVERTER_OBJ = rmsgpack.o \
rmsgpack_dom.o \
rarchdb.o \
bintree.o \
db_parser.o \
dat_converter.o \
$(NULL)
RARCHDB_TOOL_OBJ = rmsgpack.o \
rmsgpack_dom.o \
db_parser.o \
rarchdb_tool.o \
bintree.o \
rarchdb.o \
$(NULL)
all: dat_converter rmsgpack_test rarchdb_tool
%.o: %.c
${CC} $< -c ${CFLAGS} -o $@
dat_converter: ${DAT_CONVERTER_OBJ}
${CC} ${DAT_CONVERTER_OBJ} -o $@
rarchdb_tool: ${RARCHDB_TOOL_OBJ}
${CC} ${RARCHDB_TOOL_OBJ} -o $@
rmsgpack_test:
gcc rmsgpack.c rmsgpack_test.c -std=c99 -g -o $@
clean:
rm -rf *.o rmsgpack_test dat_converter rarchdb_tool

13
rarchdb/README.md Normal file
View File

@ -0,0 +1,13 @@
# rarchdb
A small read only database
Mainly to be used by retroarch
# Usage
To convert a dat file use `dat_converter <dat file> <db file>`
To list out the content of a db `rarchdb_tool <db file> list`
To create an index `rarchdb_tool <db file> create-index <index name> <field name>`
To find an entry with an index `rarchdb_tool <db file> find <index name> <value>`
The util `mkdb.sh <dat file> <db file>` will create a db file with indexes for crc sha1 and md5

119
rarchdb/bintree.c Normal file
View File

@ -0,0 +1,119 @@
#include <stdlib.h>
#include <errno.h>
#include <stdio.h>
#include "bintree.h"
static int NIL_VALUE = 1;
static void* NIL_NODE = &NIL_NODE;
static struct bintree_node *new_nil_node(struct bintree_node *parent);
void bintree_new(struct bintree *t, bintree_cmp_func cmp, void *ctx)
{
t->root = new_nil_node(NULL);
t->cmp = cmp;
t->ctx = ctx;
}
static struct bintree_node *new_nil_node(struct bintree_node *parent)
{
struct bintree_node *node = calloc(1, sizeof(struct bintree_node));
if (!node)
return NULL;
node->value = NIL_NODE;
node->parent = parent;
node->left = NULL;
node->right = NULL;
return node;
}
static inline int is_nil(const struct bintree_node *node)
{
return node == NULL || node->value == NIL_NODE;
}
static int insert(struct bintree *t, struct bintree_node *root, void *value)
{
int cmp_res = 0;
if (is_nil(root))
{
root->left = new_nil_node(root);
root->right = new_nil_node(root);
if (!root->left || !root->right)
{
if (root->left)
{
free(root->left);
root->left = NULL;
}
if (root->right)
{
free(root->right);
root->right = NULL;
}
return -ENOMEM;
}
root->value = value;
}
else
{
cmp_res = t->cmp(root->value, value, t->ctx);
if (cmp_res > 0)
return insert(t, root->left, value);
else if (cmp_res < 0)
return insert(t, root->right, value);
else
return -EINVAL;
}
return 0;
}
int bintree_insert(struct bintree *t, void *value)
{
return insert(t, t->root, value);
}
static int _bintree_iterate(struct bintree_node *n, bintree_iter_cb cb, void *ctx)
{
int rv;
if (is_nil(n))
return 0;
else
{
if((rv = _bintree_iterate(n->left, cb, ctx)) != 0)
return rv;
if ((rv = cb(n->value, ctx)) != 0)
return rv;
if((rv = _bintree_iterate(n->right, cb, ctx)) != 0)
return rv;
}
return 0;
}
int bintree_iterate(const struct bintree *t, bintree_iter_cb cb, void *ctx)
{
return _bintree_iterate(t->root, cb, ctx);
}
static void bintree_free_node(struct bintree_node *n)
{
if (n == NULL)
return;
if (n->value == NIL_NODE)
{
free(n);
return;
}
n->value = NULL;
bintree_free_node(n->left);
bintree_free_node(n->right);
free(n);
}
void bintree_free(struct bintree *t)
{
bintree_free_node(t->root);
}

27
rarchdb/bintree.h Normal file
View File

@ -0,0 +1,27 @@
#ifndef __RARCHDB_BINTREE_H__
#define __RARCHDB_BINTREE_H__
typedef int(*bintree_cmp_func)(const void *a, const void *b, void* ctx);
typedef int(*bintree_iter_cb)(void *value, void* ctx);
struct bintree_node {
void* value;
struct bintree_node *parent;
struct bintree_node *left;
struct bintree_node *right;
};
struct bintree {
struct bintree_node *root;
bintree_cmp_func cmp;
void *ctx;
};
void bintree_new(struct bintree *t, bintree_cmp_func cmp, void *ctx);
int bintree_insert(struct bintree *t, void *value);
int bintree_iterate(const struct bintree *t, bintree_iter_cb cb, void* ctx);
void bintree_free(struct bintree *t);
#endif

291
rarchdb/dat_converter.c Normal file
View File

@ -0,0 +1,291 @@
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include "rarchdb.h"
#include "db_parser.h"
#define MAX_TOKEN 256
static char *strndup_(const char *s, size_t n)
{
char* buff = calloc(n, sizeof(char));
if (!buff)
return 0;
strncpy(buff, s, n);
return buff;
}
static struct rmsgpack_dom_value *get_map_value(const struct rmsgpack_dom_value *m, char* key)
{
struct rmsgpack_dom_value v;
v.type = RDT_STRING;
v.string.len = strlen(key);
v.string.buff = key;
return rmsgpack_dom_value_map_value(m, &v);
}
static int load_string(int fd, struct rmsgpack_dom_value *out)
{
char tok[MAX_TOKEN];
ssize_t tok_size;
if ((tok_size = get_token(fd, tok, MAX_TOKEN)) < 0)
return tok_size;
out->type = RDT_STRING;
out->string.len = tok_size;
out->string.buff = strndup_(tok, tok_size);
return 0;
}
static int load_uint(int fd, struct rmsgpack_dom_value *out)
{
char tok[MAX_TOKEN], *c;
ssize_t tok_size;
uint64_t value = 0;
if ((tok_size = get_token(fd, tok, MAX_TOKEN)) < 0)
return tok_size;
for (c = tok; c < tok + tok_size; c++)
{
value *= 10;
value += *c - '0';
}
out->type = RDT_UINT;
out->uint_ = value;
return 0;
}
static int load_bin(int fd, struct rmsgpack_dom_value *out)
{
char tok[MAX_TOKEN];
ssize_t tok_size;
uint8_t h;
uint8_t l;
int i;
if ((tok_size = get_token(fd, tok, MAX_TOKEN)) < 0)
return tok_size;
out->type = RDT_BINARY;
out->binary.len = tok_size / 2;
for (i = 0; i < tok_size; i += 2)
{
if (tok[i] <= '9')
h = tok[i] - '0';
else
h = (tok[i] - 'A') + 10;
if (tok[i+1] <= '9')
l = tok[i+1] - '0';
else
l = (tok[i+1] - 'A') + 10;
tok[i/2] = h * 16 + l;
}
out->binary.buff = malloc(out->binary.len);
memcpy(out->binary.buff, tok, out->binary.len);
return 0;
}
static int dat_value_provider(void *ctx, struct rmsgpack_dom_value *out)
{
int rv, i;
static const int field_count = 22;
int fd = *((int*)ctx);
char* key;
out->type = RDT_MAP;
out->map.len = field_count;
out->map.items = calloc(field_count, sizeof(struct rmsgpack_dom_pair));
if (find_token(fd, "game") < 0)
return 1;
for (i = 0; i < field_count; i++)
{
if ((rv = load_string(fd, &out->map.items[i].key)) < 0)
goto failed;
key = out->map.items[i].key.string.buff;
if (strncmp(key, "name", sizeof("name")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "description", sizeof("description")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "users", sizeof("users")) == 0)
{
if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "releasemonth", sizeof("releasemonth")) == 0)
{
if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "releaseyear", sizeof("releaseyear")) == 0)
{
if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "rumble", sizeof("rumble")) == 0)
{
if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "analog", sizeof("analog")) == 0)
{
if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "serial", sizeof("serial")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "esrb_rating", sizeof("esrb_rating")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "elspa_rating", sizeof("elspa_rating")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "pegi_rating", sizeof("pegi_rating")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "cero_rating", sizeof("cero_rating")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "developers", sizeof("developers")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "publisher", sizeof("publisher")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "origin", sizeof("origin")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "rom", sizeof("rom")) == 0)
{
if (find_token(fd, "name") < 0)
goto failed;
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "size", sizeof("size")) == 0)
{
if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "sha1", sizeof("sha1")) == 0)
{
if ((rv = load_bin(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "crc", sizeof("crc")) == 0)
{
if ((rv = load_bin(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "md5", sizeof("md5")) == 0)
{
if ((rv = load_bin(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "serial", sizeof("serial")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, ")", sizeof(")")) == 0)
{
rmsgpack_dom_value_free(&out->map.items[i].key);
out->map.len = i;
printf("Couldn't find all fields for item\n");
break;
}
else
{
rmsgpack_dom_value_free(&out->map.items[i].key);
i--;
}
}
printf("Inserting '%s' (%02X%02X%02X%02X)...\n",
get_map_value(out, "name")->string.buff,
(unsigned char)get_map_value(out, "crc")->binary.buff[0],
(unsigned char)get_map_value(out, "crc")->binary.buff[1],
(unsigned char)get_map_value(out, "crc")->binary.buff[2],
(unsigned char)get_map_value(out, "crc")->binary.buff[3]
);
return 0;
failed:
rmsgpack_dom_value_free(out);
out->type = RDT_NULL;
return rv;
}
int main(int argc, char **argv)
{
int rv = 0;
int src = -1;
int dst = -1;
if (argc != 3)
printf("Usage: %s <dat file> <output file>\n", argv[0]);
src = open(argv[1], O_RDONLY);
if (src == -1)
{
printf("Could not open source file '%s': %s\n", argv[1], strerror(errno));
rv = errno;
goto clean;
}
dst = open(argv[2], O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
if (dst == -1)
{
printf("Could not open destination file '%s': %s\n", argv[1], strerror(errno));
rv = errno;
goto clean;
}
rv = rarchdb_create(dst, &dat_value_provider, &src);
clean:
if (src != -1)
close(src);
if (dst != -1)
close(dst);
return rv;
}

80
rarchdb/db_parser.c Normal file
View File

@ -0,0 +1,80 @@
#include "db_parser.h"
#include <errno.h>
#include <string.h>
#include <stdlib.h>
ssize_t get_token(int fd, char *token, size_t max_len)
{
char *c = token;
int rv;
ssize_t len = 0;
int in_string = 0;
while (1)
{
rv = read(fd, c, 1);
if (rv == 0)
return 0;
else if (rv < 1)
{
switch (errno)
{
case EINTR:
case EAGAIN:
continue;
default:
return -errno;
}
}
switch (*c)
{
case ' ':
case '\t':
case '\r':
case '\n':
if (c == token)
continue;
if (!in_string)
{
*c = '\0';
return len;
}
break;
case '\"':
if (c == token)
{
in_string = 1;
continue;
}
*c = '\0';
return len;
}
len++;
c++;
if (len == (ssize_t)max_len)
{
*c = '\0';
return len;
}
}
}
int find_token(int fd, const char *token)
{
int tmp_len = strlen(token);
char *tmp_token = (char*)calloc(tmp_len, 1);
if (!tmp_token)
return -1;
while (strncmp(tmp_token, token, tmp_len) != 0)
{
if (get_token(fd, tmp_token, tmp_len) <= 0)
return -1;
}
return 0;
}

15
rarchdb/db_parser.h Normal file
View File

@ -0,0 +1,15 @@
#ifndef __RARCHDB_PARSER_H
#define __RARCHDB_PARSER_H
#ifdef _WIN32
#include <io.h>
#else
#include <unistd.h>
#endif
#define MAX_TOKEN_LEN 255
ssize_t get_token(int fd, char* token, size_t max_len);
int find_token(int fd, const char* token);
#endif

3
rarchdb/mkdb.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/sh
./dat_converter "$1" "$2" && ./rarchdb_tool "$2" create-index crc crc && ./rarchdb_tool "$2" create-index md5 md5 && ./rarchdb_tool "$2" create-index sha1 sha1

372
rarchdb/rarchdb.c Normal file
View File

@ -0,0 +1,372 @@
#include "rarchdb.h"
#include <sys/types.h>
#include <unistd.h>
#include <string.h>
#include <errno.h>
#include <sys/stat.h>
#include <stdlib.h>
#include <fcntl.h>
#include <stdio.h>
#include "rmsgpack_dom.h"
#include "rmsgpack.h"
#include "bintree.h"
#include "rarchdb_endian.h"
#define MAGIC_NUMBER "RARCHDB"
struct rarchdb_header
{
char magic_number[7];
uint64_t metadata_offset;
};
struct rarchdb_metadata
{
uint64_t count;
};
struct rarchdb_index
{
char name[50];
uint64_t key_size;
uint64_t next;
};
struct node_iter_ctx
{
struct rarchdb *db;
struct rarchdb_index *idx;
};
static struct rmsgpack_dom_value sentinal;
static int rarchdb_read_metadata(int fd, struct rarchdb_metadata *md)
{
return rmsgpack_dom_read_into(fd, "count", &md->count, NULL);
}
static int rarchdb_write_metadata(int fd, struct rarchdb_metadata *md)
{
rmsgpack_write_map_header(fd, 1);
rmsgpack_write_string(fd, "count", strlen("count"));
return rmsgpack_write_uint(fd, md->count);
}
int rarchdb_create(int fd, rarchdb_value_provider value_provider, void *ctx)
{
int rv;
uint64_t item_count = 0;
struct rmsgpack_dom_value item = {};
struct rarchdb_header header = {};
struct rarchdb_metadata md;
strcpy(header.magic_number, MAGIC_NUMBER);
off_t root = lseek(fd, 0, SEEK_CUR);
// We write the header in the end because we need to know the size of
// the db first
lseek(fd, sizeof(struct rarchdb_header), SEEK_CUR);
while ((rv = value_provider(ctx, &item)) == 0)
{
if((rv = rmsgpack_dom_write(fd, &item)) < 0)
return rv;
rmsgpack_dom_value_free(&item);
item_count++;
}
if (rv < 0)
return rv;
if((rv = rmsgpack_dom_write(fd, &sentinal)) < 0)
return rv;
header.metadata_offset = httobe64(lseek(fd, 0, SEEK_CUR));
md.count = item_count;
rarchdb_write_metadata(fd, &md);
lseek(fd, root, SEEK_SET);
write(fd, &header, sizeof(header));
printf("Created DB with %lu entries\n", item_count);
return 0;
}
static int rarchdb_read_index_header(int fd, struct rarchdb_index *idx)
{
uint64_t name_len = 50;
return rmsgpack_dom_read_into(fd,
"name", idx->name, &name_len,
"key_size", &idx->key_size,
"next", &idx->next,
NULL);
}
static void rarchdb_write_index_header(int fd, struct rarchdb_index *idx)
{
rmsgpack_write_map_header(fd, 3);
rmsgpack_write_string(fd, "name", strlen("name"));
rmsgpack_write_string(fd, idx->name, strlen(idx->name));
rmsgpack_write_string(fd, "key_size", strlen("key_size"));
rmsgpack_write_uint(fd, idx->key_size);
rmsgpack_write_string(fd, "next", strlen("next"));
rmsgpack_write_uint(fd, idx->next);
}
void rarchdb_close(struct rarchdb *db)
{
close(db->fd);
db->fd = -1;
}
int rarchdb_open(const char *path, struct rarchdb *db)
{
struct rarchdb_header header;
struct rarchdb_metadata md;
int rv;
int fd = open(path, O_RDWR);
if (fd == -1)
return -errno;
db->root = lseek(fd, 0, SEEK_CUR);
if ((rv = read(fd, &header, sizeof(header))) == -1)
{
rv = -errno;
goto error;
}
if (strncmp(header.magic_number, MAGIC_NUMBER, sizeof(MAGIC_NUMBER)) != 0)
{
rv = -EINVAL;
goto error;
}
header.metadata_offset = betoht64(header.metadata_offset);
lseek(fd, header.metadata_offset, SEEK_SET);
if (rarchdb_read_metadata(fd, &md) < 0)
{
rv = -EINVAL;
goto error;
}
db->count = md.count;
db->first_index_offset = lseek(fd, 0, SEEK_CUR);
db->fd = fd;
rarchdb_read_reset(db);
return 0;
error:
close(fd);
return rv;
}
static int rarchdb_find_index(struct rarchdb *db, const char *index_name, struct rarchdb_index *idx)
{
off_t eof = lseek(db->fd, 0, SEEK_END);
off_t offset = lseek(db->fd, db->first_index_offset, SEEK_SET);
while (offset < eof)
{
rarchdb_read_index_header(db->fd, idx);
if (strncmp(index_name, idx->name, strlen(idx->name)) == 0)
return 0;
offset = lseek(db->fd, idx->next, SEEK_CUR);
}
return -1;
}
static int node_compare(const void *a, const void* b, void *ctx)
{
return memcmp(a, b, *(uint8_t*)ctx);
}
static int binsearch(const void *buff, const void *item, uint64_t count, uint8_t field_size, uint64_t *offset)
{
int mid = count / 2;
int item_size = field_size + sizeof(uint64_t);
const void *current = buff + (mid * item_size);
int rv = node_compare(current, item, &field_size);
if (rv == 0)
{
*offset = *(uint64_t*)(current + field_size);
return 0;
}
else if (count == 0)
return -1;
else if (rv > 0)
return binsearch(buff, item, mid, field_size, offset);
return binsearch(current + item_size, item, count - mid, field_size, offset);
}
int rarchdb_find_entry(struct rarchdb *db, const char *index_name, const void *key)
{
struct rarchdb_index idx;
uint64_t offset;
ssize_t nread = 0;
int rv;
if (rarchdb_find_index(db, index_name, &idx) < 0)
{
rarchdb_read_reset(db);
return -1;
}
size_t bufflen = idx.next;
void* buff = malloc(bufflen);
if (!buff)
return -ENOMEM;
while (nread < bufflen)
{
rv = read(db->fd, buff + nread, bufflen - nread);
if (rv <= 0)
{
free(buff);
return -errno;
}
nread += rv;
}
rv = binsearch(buff, key, db->count, idx.key_size, &offset);
free(buff);
rarchdb_read_reset(db);
if (rv == 0)
lseek(db->fd, offset, SEEK_SET);
return rv;
}
int rarchdb_read_reset(struct rarchdb *db)
{
db->eof = 0;
return lseek(db->fd, db->root + sizeof(struct rarchdb_header), SEEK_SET);
}
int rarchdb_read_item(struct rarchdb *db, struct rmsgpack_dom_value *out)
{
int rv;
if (db->eof)
return EOF;
rv = rmsgpack_dom_read(db->fd, out);
if (rv < 0)
return rv;
if (out->type == RDT_NULL)
{
db->eof = 1;
return EOF;
}
return 0;
}
static int node_iter(void *value, void *ctx)
{
struct node_iter_ctx *nictx = ctx;
if (write(nictx->db->fd, value, nictx->idx->key_size + sizeof(uint64_t)) > 0)
return 0;
return -1;
}
uint64_t rarchdb_tell(struct rarchdb *db)
{
return lseek(db->fd, 0, SEEK_CUR);
}
int rarchdb_create_index(struct rarchdb *db, const char* name, const char *field_name)
{
int rv;
struct rmsgpack_dom_value key;
struct rarchdb_index idx;
struct rmsgpack_dom_value item;
struct rmsgpack_dom_value *field;
void* buff = NULL;
uint8_t field_size = 0;
struct bintree tree;
uint64_t item_loc = rarchdb_tell(db);
uint64_t idx_header_offset;
bintree_new(&tree, node_compare, &field_size);
rarchdb_read_reset(db);
key.type = RDT_STRING;
key.string.len = strlen(field_name);
// We know we aren't going to change it
key.string.buff = (char *) field_name;
while(rarchdb_read_item(db, &item) == 0)
{
if (item.type != RDT_MAP)
{
rv = -EINVAL;
printf("Only map keys are supported\n");
goto clean;
}
field = rmsgpack_dom_value_map_value(&item, &key);
if (!field)
{
rv = -EINVAL;
printf("field not found in item\n");
goto clean;
}
if (field->type != RDT_BINARY)
{
rv = -EINVAL;
printf("field is not binary\n");
goto clean;
}
if (field->binary.len == 0)
{
rv = -EINVAL;
printf("field is empty\n");
goto clean;
}
if (field_size == 0)
field_size = field->binary.len;
else if (field->binary.len != field_size)
{
rv = -EINVAL;
printf("field is not of correct size\n");
goto clean;
}
buff = malloc(field_size + sizeof(uint64_t));
if(!buff)
{
rv = -ENOMEM;
goto clean;
}
memcpy(buff, field->binary.buff, field_size);
memcpy(buff + field_size, &item_loc, sizeof(uint64_t));
if (bintree_insert(&tree, buff) != 0)
{
printf("Value is not unique: ");
rmsgpack_dom_value_print(field);
printf("\n");
rv = -EINVAL;
goto clean;
}
buff = NULL;
rmsgpack_dom_value_free(&item);
item_loc = rarchdb_tell(db);
}
idx_header_offset = lseek(db->fd, 0, SEEK_END);
strncpy(idx.name, name, 50);
idx.name[49] = '\0';
idx.key_size = field_size;
idx.next = db->count * (field_size + sizeof(uint64_t));
rarchdb_write_index_header(db->fd, &idx);
struct node_iter_ctx nictx;
nictx.db = db;
nictx.idx = &idx;
bintree_iterate(&tree, node_iter, &nictx);
bintree_free(&tree);
clean:
rmsgpack_dom_value_free(&item);
if (buff)
free(buff);
rarchdb_read_reset(db);
return 0;
}

28
rarchdb/rarchdb.h Normal file
View File

@ -0,0 +1,28 @@
#ifndef __RARCHDB_H__
#define __RARCHDB_H__
#include <stdint.h>
#include "rmsgpack_dom.h"
struct rarchdb {
int fd;
int eof;
uint64_t root;
uint64_t count;
uint64_t first_index_offset;
};
typedef int(*rarchdb_value_provider)(void *ctx, struct rmsgpack_dom_value *out);
int rarchdb_create(int fd, rarchdb_value_provider value_provider, void *ctx);
void rarchdb_close(struct rarchdb *db);
int rarchdb_open(const char *path, struct rarchdb *db);
int rarchdb_read_item(struct rarchdb *db, struct rmsgpack_dom_value *out);
int rarchdb_read_reset(struct rarchdb *db);
int rarchdb_create_index(struct rarchdb *db, const char* name, const char *field_name);
int rarchdb_find_entry(struct rarchdb *db, const char *index_name, const void *key);
#endif

51
rarchdb/rarchdb_endian.h Normal file
View File

@ -0,0 +1,51 @@
#ifndef __RARCHDB_MSGPACK_ENDIAN_H
#define __RARCHDB_MSGPACK_ENDIAN_H
#include <stdint.h>
#ifndef INLINE
#define INLINE inline
#endif
static INLINE uint8_t is_little_endian(void)
{
union
{
uint16_t x;
uint8_t y[2];
} u;
u.x = 1;
return u.y[0];
}
#define swap16(val) \
((((uint16_t)(val) & 0x00ff) << 8) \
| (((uint16_t)(val) & 0xff00) >> 8))
#define swap32(val) \
((((uint32_t)(val) & 0x000000ff) << 24) \
| (((uint32_t)(val) & 0x0000ff00) << 8) \
| (((uint32_t)(val) & 0x00ff0000) >> 8) \
| (((uint32_t)(val) & 0xff000000) >> 24))
#define swap64(val) \
((((uint64_t)(val) & 0x00000000000000ffULL) << 56) \
| (((uint64_t)(val) & 0x000000000000ff00ULL) << 40) \
| (((uint64_t)(val) & 0x0000000000ff0000ULL) << 24) \
| (((uint64_t)(val) & 0x00000000ff000000ULL) << 8) \
| (((uint64_t)(val) & 0x000000ff00000000ULL) >> 8) \
| (((uint64_t)(val) & 0x0000ff0000000000ULL) >> 24) \
| (((uint64_t)(val) & 0x00ff000000000000ULL) >> 40) \
| (((uint64_t)(val) & 0xff00000000000000ULL) >> 56))
#define httobe64(x) (is_little_endian() ? swap64(x) : (x))
#define httobe32(x) (is_little_endian() ? swap32(x) : (x))
#define httobe16(x) (is_little_endian() ? swap16(x) : (x))
#define betoht16(x) httobe16(x)
#define betoht32(x) httobe32(x)
#define betoht64(x) httobe64(x)
#endif

116
rarchdb/rarchdb_tool.c Normal file
View File

@ -0,0 +1,116 @@
#include <stdio.h>
#include <string.h>
#include "rarchdb.h"
#include "rmsgpack_dom.h"
static int list_db(void)
{
return 0;
}
int main(int argc, char** argv)
{
int rv;
struct rarchdb db;
struct rmsgpack_dom_value item;
if (argc < 3)
{
printf("Usage: %s <db file> <command> [extra args...]\n", argv[0]);
printf("Available Commands:\n");
printf("\tlist\n");
printf("\tcreate-index <index name> <field name>\n");
printf("\tfind <index name> <value>\n");
return 1;
}
const char* command = argv[2];
const char* path = argv[1];
if ((rv = rarchdb_open(path, &db)) != 0)
{
printf("Could not open db file '%s': %s\n", path, strerror(-rv));
return 1;
}
if (strcmp(command, "list") == 0)
{
if (argc != 3)
{
printf("Usage: %s <db file> list\n", argv[0]);
return 1;
}
while(rarchdb_read_item(&db, &item) == 0)
{
rmsgpack_dom_value_print(&item);
printf("\n");
rmsgpack_dom_value_free(&item);
}
}
else if (strcmp(command, "create-index") == 0)
{
const char *index_name, *field_name;
if (argc != 5)
{
printf("Usage: %s <db file> create-index <index name> <field name>\n", argv[0]);
return 1;
}
index_name = argv[3];
field_name = argv[4];
rarchdb_create_index(&db, index_name, field_name);
}
else if (strcmp(command, "find") == 0)
{
int i;
const char *index_name, *value;
char bin_value[256];
uint8_t h;
uint8_t l;
if (argc != 5)
{
printf("Usage: %s <db file> create-index <index name> <field name>\n", argv[0]);
return 1;
}
index_name = argv[3];
value = argv[4];
for (i = 0; i < strlen(value); i += 2)
{
if (value[i] <= '9')
h = value[i] - '0';
else
h = (value[i] - 'A') + 10;
if (value[i+1] <= '9')
l = value[i+1] - '0';
else
l = (value[i+1] - 'A') + 10;
bin_value[i/2] = h * 16 + l;
}
if (rarchdb_find_entry(&db, index_name, bin_value) != 0)
{
printf("Could not find item\n");
return 1;
}
if (rarchdb_read_item(&db, &item) == 0)
{
rmsgpack_dom_value_print(&item);
printf("\n");
rmsgpack_dom_value_free(&item);
}
}
else
{
printf("Unkown command %s\n", argv[1]);
return 1;
}
rarchdb_close(&db);
}

548
rarchdb/rmsgpack.c Normal file
View File

@ -0,0 +1,548 @@
/*
* An almost complete implementation of msgpack by
* Saggi Mizrahi <ficoos@gmail.com>
*
* TODO:
* - float types
* - ext types
*
* For more information http://msgpack.org/
*/
#include "rmsgpack.h"
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <stdint.h>
#include <string.h>
#include "rarchdb_endian.h"
static const uint8_t MPF_FIXMAP = 0x80;
static const uint8_t MPF_MAP16 = 0xde;
static const uint8_t MPF_MAP32 = 0xdf;
static const uint8_t MPF_FIXARRAY = 0x90;
static const uint8_t MPF_ARRAY16 = 0xdc;
static const uint8_t MPF_ARRAY32 = 0xdd;
static const uint8_t MPF_FIXSTR = 0xa0;
static const uint8_t MPF_STR8 = 0xd9;
static const uint8_t MPF_STR16 = 0xda;
static const uint8_t MPF_STR32 = 0xdb;
static const uint8_t MPF_BIN8 = 0xc4;
static const uint8_t MPF_BIN16 = 0xc5;
static const uint8_t MPF_BIN32 = 0xc6;
static const uint8_t MPF_FALSE = 0xc2;
static const uint8_t MPF_TRUE = 0xc3;
static const uint8_t MPF_INT8 = 0xd0;
static const uint8_t MPF_INT16 = 0xd1;
static const uint8_t MPF_INT32 = 0xd2;
static const uint8_t MPF_INT64 = 0xd3;
static const uint8_t MPF_UINT8 = 0xcc;
static const uint8_t MPF_UINT16 = 0xcd;
static const uint8_t MPF_UINT32 = 0xce;
static const uint8_t MPF_UINT64 = 0xcf;
static const uint8_t MPF_NIL = 0xc0;
int rmsgpack_write_array_header(int fd, uint32_t size)
{
if (size < 16)
{
size = (size | MPF_FIXARRAY);
if (write(fd, &size, sizeof(int8_t)) == -1)
return -errno;
return sizeof(int8_t);
}
else if (size == (uint16_t)size)
{
if (write(fd, &MPF_ARRAY16, sizeof(MPF_ARRAY16)) == -1)
return -errno;
if (write(fd, (void *)(&size), sizeof(uint16_t)) == -1)
return -errno;
return sizeof(int8_t) + sizeof(uint16_t);
}
else
{
if (write(fd, &MPF_ARRAY32, sizeof(MPF_ARRAY32)) == -1)
return -errno;
if (write(fd, (void *)(&size), sizeof(uint32_t)) == -1)
return -errno;
return sizeof(int8_t) + sizeof(uint32_t);
}
}
int rmsgpack_write_map_header(int fd, uint32_t size)
{
if (size < 16)
{
size = (size | MPF_FIXMAP);
if (write(fd, &size, sizeof(int8_t)) == -1)
return -errno;
return sizeof(int8_t);
}
else if (size < (uint16_t)size)
{
if (write(fd, &MPF_MAP16, sizeof(MPF_MAP16)) == -1)
return -errno;
if (write(fd, (void *)(&size), sizeof(uint16_t)) == -1)
return -errno;
return sizeof(int8_t) + sizeof(uint16_t);
}
else
{
if (write(fd, &MPF_MAP32, sizeof(MPF_MAP32)) == -1)
return -errno;
if (write(fd, (void *)(&size), sizeof(uint32_t)) == -1)
return -errno;
return sizeof(int8_t) + sizeof(uint32_t);
}
}
int rmsgpack_write_string(int fd, const char *s, uint32_t len)
{
int8_t fixlen = 0;
uint16_t tmp_i16;
uint32_t tmp_i32;
int written = sizeof(int8_t);
if (len < 32)
{
fixlen = len | MPF_FIXSTR;
if (write(fd, &fixlen, sizeof(int8_t)) == -1)
return -errno;
}
else if (len < 1<<8)
{
if (write(fd, &MPF_STR8, sizeof(MPF_STR8)) == -1)
return -errno;
if (write(fd, &len, sizeof(uint8_t)) == -1)
return -errno;
written += sizeof(uint8_t);
}
else if (len < 1<<16)
{
if (write(fd, &MPF_STR16, sizeof(MPF_STR16)) == -1)
return -errno;
tmp_i16 = httobe16(len);
if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1)
return -errno;
written += sizeof(uint16_t);
}
else
{
if (write(fd, &MPF_STR32, sizeof(MPF_STR32)) == -1)
return -errno;
tmp_i32 = httobe32(len);
if (write(fd, &tmp_i32, sizeof(uint32_t)) == -1)
return -errno;
written += sizeof(uint32_t);
}
if (write(fd, s, len) == -1)
return -errno;
written += len;
return written;
}
int rmsgpack_write_bin(int fd, const void *s, uint32_t len)
{
uint16_t tmp_i16;
uint32_t tmp_i32;
int written = sizeof(int8_t);
if (len == (uint8_t)len)
{
if (write(fd, &MPF_BIN8, sizeof(MPF_BIN8)) == -1)
return -errno;
if (write(fd, &len, sizeof(uint8_t)) == -1)
return -errno;
written += sizeof(uint8_t);
}
else if (len == (uint16_t)len)
{
if (write(fd, &MPF_BIN16, sizeof(MPF_BIN16)) == -1)
return -errno;
tmp_i16 = httobe16(len);
if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1)
return -errno;
written += sizeof(uint16_t);
}
else
{
if (write(fd, &MPF_BIN32, sizeof(MPF_BIN32)) == -1)
return -errno;
tmp_i32 = httobe32(len);
if (write(fd, &tmp_i32, sizeof(uint32_t)) == -1)
return -errno;
written += sizeof(uint32_t);
}
if (write(fd, s, len) == -1)
return -errno;
written += len;
return 0;
}
int rmsgpack_write_nil(int fd)
{
if (write(fd, &MPF_NIL, sizeof(MPF_NIL)) == -1)
return -errno;
return sizeof(uint8_t);
}
int rmsgpack_write_bool(int fd, int value)
{
if (value)
{
if (write(fd, &MPF_TRUE, sizeof(MPF_TRUE)) == -1)
return -errno;
}
else
{
if (write(fd, &MPF_FALSE, sizeof(MPF_FALSE)) == -1)
return -errno;
}
return sizeof(uint8_t);
}
int rmsgpack_write_int(int fd, int64_t value)
{
uint8_t tmpval = 0;
int16_t tmp_i16;
int32_t tmp_i32;
int written = sizeof(uint8_t);
if (value >=0 && value < 128)
{
if (write(fd, &value, sizeof(int8_t)) == -1)
return -errno;
}
else if (value < 0 && value > -32)
{
tmpval = (value) | 0xe0;
if (write(fd, &tmpval, sizeof(uint8_t)) == -1)
return -errno;
}
else if (value == (int8_t)value)
{
if (write(fd, &MPF_INT8, sizeof(MPF_INT8)) == -1)
return -errno;
if (write(fd, &value, sizeof(int8_t)) == -1)
return -errno;
written += sizeof(int8_t);
}
else if (value == (int16_t)value)
{
if (write(fd, &MPF_INT16, sizeof(MPF_INT16)) == -1)
return -errno;
tmp_i16 = httobe16(value);
if (write(fd, &tmp_i16, sizeof(int16_t)) == -1)
return -errno;
written += sizeof(int16_t);
}
else if (value == (int32_t)value)
{
if (write(fd, &MPF_INT32, sizeof(MPF_INT32)) == -1)
return -errno;
tmp_i32 = httobe32(value);
if (write(fd, &tmp_i32, sizeof(int32_t)) == -1)
return -errno;
written += sizeof(int32_t);
}
else
{
if (write(fd, &MPF_INT64, sizeof(MPF_INT64)) == -1)
return -errno;
value = httobe64(value);
if (write(fd, &value, sizeof(int64_t)) == -1)
return -errno;
written += sizeof(int64_t);
}
return written;
}
int rmsgpack_write_uint(int fd, uint64_t value)
{
uint16_t tmp_i16;
uint32_t tmp_i32;
int written = sizeof(uint8_t);
if (value == (uint8_t)value)
{
if (write(fd, &MPF_UINT8, sizeof(MPF_UINT8)) == -1)
return -errno;
if (write(fd, &value, sizeof(uint8_t)) == -1)
return -errno;
written += sizeof(uint8_t);
}
else if (value == (uint16_t)value)
{
if (write(fd, &MPF_UINT16, sizeof(MPF_UINT16)) == -1)
return -errno;
tmp_i16 = httobe16(value);
if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1)
return -errno;
written += sizeof(uint16_t);
}
else if (value == (uint32_t)value)
{
if (write(fd, &MPF_UINT32, sizeof(MPF_UINT32)) == -1)
return -errno;
tmp_i32 = httobe32(value);
if (write(fd, &tmp_i32, sizeof(uint32_t)) == -1)
return -errno;
written += sizeof(uint32_t);
}
else
{
if (write(fd, &MPF_UINT64, sizeof(MPF_UINT64)) == -1)
return -errno;
value = httobe64(value);
if (write(fd, &value, sizeof(uint64_t)) == -1)
return -errno;
written += sizeof(uint64_t);
}
return written;
}
static int read_uint(int fd, uint64_t *out, size_t size)
{
uint64_t tmp;
if(read(fd, &tmp, size) == -1)
return -errno;
switch(size)
{
case 1:
*out = *(uint8_t*)(&tmp);
break;
case 2:
*out = betoht16(tmp);
break;
case 4:
*out = betoht32(tmp);
break;
case 8:
*out = betoht64(tmp);
break;
}
return 0;
}
static int read_int(int fd, int64_t *out, size_t size)
{
uint64_t tmp;
if(read(fd, &tmp, size) == -1)
return -errno;
switch(size)
{
case 1:
*out = *((int8_t*)(&tmp));
break;
case 2:
tmp = betoht16(tmp);
*out = *((int16_t*)(&tmp));
break;
case 4:
tmp = betoht32(tmp);
*out = *((int32_t*)(&tmp));
break;
case 8:
tmp = betoht64(tmp);
*out = *((int64_t*)(&tmp));
break;
}
return 0;
}
static int read_buff(int fd, size_t size, char** pbuff, uint64_t *len)
{
uint64_t tmp_len;
if(read_uint(fd, &tmp_len, size) == -1)
return -errno;
*pbuff = calloc(tmp_len + 1, sizeof(char));
if (read(fd, *pbuff, tmp_len) == -1)
{
free(*pbuff);
return -errno;
}
*len = tmp_len;
return 0;
}
static int read_map(int fd, uint32_t len, struct rmsgpack_read_callbacks *callbacks, void *data)
{
int rv, i;
if (
callbacks->read_map_start &&
(rv = callbacks->read_map_start(len, data)) < 0)
return rv;
for (i = 0; i < len; i++)
{
if((rv = rmsgpack_read(fd, callbacks, data)) < 0)
return rv;
if((rv = rmsgpack_read(fd, callbacks, data)) < 0)
return rv;
}
return 0;
}
static int read_array(int fd, uint32_t len, struct rmsgpack_read_callbacks *callbacks, void *data)
{
int rv, i;
if (
callbacks->read_array_start &&
(rv = callbacks->read_array_start(len, data)) < 0)
return rv;
for (i = 0; i < len; i++)
{
if((rv = rmsgpack_read(fd, callbacks, data)) < 0)
return rv;
}
return 0;
}
int rmsgpack_read(int fd, struct rmsgpack_read_callbacks *callbacks, void *data)
{
int rv;
uint64_t tmp_len = 0;
uint64_t tmp_uint = 0;
int64_t tmp_int = 0;
uint8_t type = 0;
char* buff;
if (read(fd, &type, sizeof(uint8_t)) == -1)
return -errno;
if (type < MPF_FIXMAP)
{
if (!callbacks->read_int)
return 0;
return callbacks->read_int(type, data);
}
else if (type < MPF_FIXARRAY)
{
tmp_len = type - MPF_FIXMAP;
return read_map(fd, tmp_len, callbacks, data);
}
else if (type < MPF_FIXSTR)
{
tmp_len = type - MPF_FIXARRAY;
return read_array(fd, tmp_len, callbacks, data);
}
else if (type < MPF_NIL)
{
tmp_len = type - MPF_FIXSTR;
buff = calloc(tmp_len + 1, sizeof(char));
if (!buff)
return -ENOMEM;
if(read(fd, buff, tmp_len) == -1)
{
free(buff);
return -errno;
}
buff[tmp_len] = '\0';
if (!callbacks->read_string)
{
free(buff);
return 0;
}
return callbacks->read_string(buff, tmp_len, data);
}
else if (type > MPF_MAP32)
{
if (!callbacks->read_int)
return 0;
return callbacks->read_int(type - 0xff - 1, data);
}
switch(type)
{
case 0xc0:
if(callbacks->read_nil)
return callbacks->read_nil(data);
break;
case 0xc2:
if(callbacks->read_bool)
return callbacks->read_bool(0, data);
break;
case 0xc3:
if(callbacks->read_bool)
return callbacks->read_bool(1, data);
break;
case 0xc4:
case 0xc5:
case 0xc6:
if((rv = read_buff(fd, 1<<(type - 0xc4), &buff, &tmp_len)) < 0)
return rv;
if (callbacks->read_bin)
return callbacks->read_bin(buff, tmp_len, data);
break;
case 0xcc:
case 0xcd:
case 0xce:
case 0xcf:
tmp_len = 1<<(type - 0xcc);
tmp_uint = 0;
if(read_uint(fd, &tmp_uint, tmp_len) == -1)
return -errno;
if (callbacks->read_uint)
return callbacks->read_uint(tmp_uint, data);
break;
case 0xd0:
case 0xd1:
case 0xd2:
case 0xd3:
tmp_len = 1<<(type - 0xd0);
tmp_int = 0;
if(read_int(fd, &tmp_int, tmp_len) == -1)
return -errno;
if (callbacks->read_int)
return callbacks->read_int(tmp_int, data);
break;
case 0xd9:
case 0xda:
case 0xdb:
if((rv = read_buff(fd, 1<<(type - 0xd9), &buff, &tmp_len)) < 0)
return rv;
if (callbacks->read_string)
return callbacks->read_string(buff, tmp_len, data);
break;
case 0xdc:
case 0xdd:
if(read_uint(fd, &tmp_len, 2<<(type - 0xdc)) == -1)
return -errno;
return read_array(fd, tmp_len, callbacks, data);
break;
case 0xde:
case 0xdf:
if(read_uint(fd, &tmp_len, 2<<(type - 0xde)) == -1)
return -errno;
return read_map(fd, tmp_len, callbacks, data);
break;
}
return 0;
}

30
rarchdb/rmsgpack.h Normal file
View File

@ -0,0 +1,30 @@
#ifndef __RARCHDB_MSGPACK_H__
#define __RARCHDB_MSGPACK_H__
#include <stdint.h>
struct rmsgpack_read_callbacks {
int(*read_nil)(void*);
int(*read_bool)(int, void*);
int(*read_int)(int64_t, void*);
int(*read_uint)(uint64_t, void*);
int(*read_string)(char *, uint32_t, void*);
int(*read_bin)(void *, uint32_t, void*);
int(*read_map_start)(uint32_t, void*);
int(*read_array_start)(uint32_t, void*);
};
int rmsgpack_write_array_header(int fd, uint32_t size);
int rmsgpack_write_map_header(int fd, uint32_t size);
int rmsgpack_write_string(int fd, const char *s, uint32_t len);
int rmsgpack_write_bin(int fd, const void *s, uint32_t len);
int rmsgpack_write_nil(int fd);
int rmsgpack_write_bool(int fd, int value);
int rmsgpack_write_int(int fd, int64_t value);
int rmsgpack_write_uint(int fd, uint64_t value);
int rmsgpack_read(int fd, struct rmsgpack_read_callbacks *callbacks, void *data);
#endif

462
rarchdb/rmsgpack_dom.c Normal file
View File

@ -0,0 +1,462 @@
#include "rmsgpack_dom.h"
#include <stdint.h>
#include <stdio.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <stdarg.h>
#include "rmsgpack.h"
#define MAX_DEPTH 128
struct dom_reader_state
{
int i;
struct rmsgpack_dom_value* stack[MAX_DEPTH];
};
static struct rmsgpack_dom_value *dom_reader_state_pop(struct dom_reader_state *s)
{
struct rmsgpack_dom_value *v = s->stack[s->i];
s->i--;
return v;
}
static int dom_reader_state_push(struct dom_reader_state *s, struct rmsgpack_dom_value *v)
{
//TODO: Verify we don't overflow
s->i++;
s->stack[s->i] = v;
return 0;
}
static int dom_read_nil(void *data)
{
struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
v->type = RDT_NULL;
return 0;
}
static int dom_read_bool(int value, void *data)
{
struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
v->type = RDT_BOOL;
v->bool_ = value;
return 0;
}
static int dom_read_int(int64_t value, void *data)
{
struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
v->type = RDT_INT;
v->int_ = value;
return 0;
}
static int dom_read_uint(uint64_t value, void *data)
{
struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
v->type = RDT_UINT;
v->uint_ = value;
return 0;
}
static int dom_read_string(char* value, uint32_t len, void *data)
{
struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
v->type = RDT_STRING;
v->string.len = len;
v->string.buff = value;
return 0;
}
static int dom_read_bin(void* value, uint32_t len, void *data)
{
struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
v->type = RDT_BINARY;
v->binary.len = len;
v->binary.buff = value;
return 0;
}
static int dom_read_map_start(uint32_t len, void *data)
{
int i;
struct rmsgpack_dom_pair *items;
struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
v->type = RDT_MAP;
v->map.len = len;
v->map.items = NULL;
items = (struct rmsgpack_dom_pair *)calloc(len, sizeof(struct rmsgpack_dom_pair));
if (!items)
return -ENOMEM;
v->map.items = items;
for (i = 0; i < len; i++)
{
if (dom_reader_state_push(data, &items[i].value) < 0)
return -ENOMEM;
if (dom_reader_state_push(data, &items[i].key) < 0)
return -ENOMEM;
}
return 0;
}
static int dom_read_array_start(uint32_t len, void *data)
{
int i;
struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
struct rmsgpack_dom_value *items;
v->type = RDT_ARRAY;
v->array.len = len;
v->array.items = NULL;
items = (struct rmsgpack_dom_value*)calloc(len, sizeof(struct rmsgpack_dom_pair));
if (!items)
return -ENOMEM;
v->array.items = items;
for (i = 0; i < len; i++)
{
if (dom_reader_state_push(data, &items[i]) < 0)
return -ENOMEM;
}
return 0;
}
static struct rmsgpack_read_callbacks dom_reader_callbacks = {
dom_read_nil,
dom_read_bool,
dom_read_int,
dom_read_uint,
dom_read_string,
dom_read_bin,
dom_read_map_start,
dom_read_array_start
};
void rmsgpack_dom_value_free(struct rmsgpack_dom_value *v)
{
int i;
switch (v->type)
{
case RDT_STRING:
free(v->string.buff);
break;
case RDT_BINARY:
free(v->binary.buff);
break;
case RDT_MAP:
for (i = 0; i < v->map.len; i++)
{
rmsgpack_dom_value_free(&v->map.items[i].key);
rmsgpack_dom_value_free(&v->map.items[i].value);
}
free(v->map.items);
break;
case RDT_ARRAY:
for (i = 0; i < v->array.len; i++)
rmsgpack_dom_value_free(&v->array.items[i]);
free(v->array.items);
break;
case RDT_NULL:
case RDT_INT:
case RDT_BOOL:
case RDT_UINT:
// Do nothing
break;
}
}
struct rmsgpack_dom_value *rmsgpack_dom_value_map_value(
const struct rmsgpack_dom_value *map, const struct rmsgpack_dom_value *key)
{
int i;
if (map->type != RDT_MAP)
return NULL;
for (i = 0; i < map->map.len; i++)
{
if (rmsgpack_dom_value_cmp(key, &map->map.items[i].key) == 0)
return &map->map.items[i].value;
}
return NULL;
}
int rmsgpack_dom_value_cmp(
const struct rmsgpack_dom_value *a, const struct rmsgpack_dom_value *b)
{
int rv, i;
if (a == b)
return 1;
if (a->type != b->type)
return 1;
switch (a->type)
{
case RDT_NULL:
return 0;
break;
case RDT_BOOL:
return a->bool_ == b->bool_ ? 1 : 0;
break;
case RDT_INT:
return a->int_ == b->int_ ? 1 : 0;
break;
case RDT_UINT:
return a->uint_ == b->uint_ ? 1 : 0;
break;
case RDT_STRING:
if (a->string.len != b->string.len)
return 1;
return strncmp(a->string.buff, b->string.buff, a->string.len);
break;
case RDT_BINARY:
if (a->binary.len != b->binary.len)
return 1;
return memcmp(a->binary.buff, b->binary.buff, a->binary.len);
break;
case RDT_MAP:
if (a->map.len != b->map.len)
return 1;
for (i = 0; i < a->map.len; i++)
{
if((rv = rmsgpack_dom_value_cmp(&a->map.items[i].key, &b->map.items[i].key)) != 0)
return rv;
if((rv = rmsgpack_dom_value_cmp(&a->map.items[i].value, &b->map.items[i].value)) != 0)
return rv;
}
break;
case RDT_ARRAY:
if (a->array.len != b->array.len)
return 1;
for (i = 0; i < a->array.len; i++)
{
if((rv = rmsgpack_dom_value_cmp(&a->array.items[i], &b->array.items[i])) != 0)
return rv;
}
break;
}
return 1;
}
void rmsgpack_dom_value_print(struct rmsgpack_dom_value *obj)
{
int i;
switch (obj->type)
{
case RDT_NULL:
printf("nil");
break;
case RDT_BOOL:
if (obj->bool_)
printf("true");
else
printf("false");
break;
case RDT_INT:
printf("%ld", obj->int_);
break;
case RDT_UINT:
printf("%lu", obj->uint_);
break;
case RDT_STRING:
printf("\"%s\"", obj->string.buff);
break;
case RDT_BINARY:
printf("\"");
for (i = 0; i < obj->binary.len; i++)
printf("%02X", (unsigned char) obj->binary.buff[i]);
printf("\"");
break;
case RDT_MAP:
printf("{");
for (i = 0; i < obj->map.len; i++)
{
rmsgpack_dom_value_print(&obj->map.items[i].key);
printf(": ");
rmsgpack_dom_value_print(&obj->map.items[i].value);
if (i < (obj->map.len - 1))
printf(", ");
}
printf("}");
break;
case RDT_ARRAY:
printf("[");
for (i = 0; i < obj->array.len; i++)
{
rmsgpack_dom_value_print(&obj->array.items[i]);
if (i < (obj->array.len - 1))
printf(", ");
}
printf("]");
}
}
int rmsgpack_dom_write(int fd, const struct rmsgpack_dom_value *obj)
{
int i;
int rv = 0;
int written = 0;
switch (obj->type)
{
case RDT_NULL:
return rmsgpack_write_nil(fd);
break;
case RDT_BOOL:
return rmsgpack_write_bool(fd, obj->bool_);
break;
case RDT_INT:
return rmsgpack_write_int(fd, obj->int_);
break;
case RDT_UINT:
return rmsgpack_write_uint(fd, obj->uint_);
break;
case RDT_STRING:
return rmsgpack_write_string(fd, obj->string.buff, obj->string.len);
break;
case RDT_BINARY:
return rmsgpack_write_bin(fd, obj->binary.buff, obj->binary.len);
break;
case RDT_MAP:
if ((rv = rmsgpack_write_map_header(fd, obj->map.len)) < 0)
return rv;
written += rv;
for (i = 0; i < obj->map.len; i++)
{
if((rv = rmsgpack_dom_write(fd, &obj->map.items[i].key)) < 0)
return rv;
written += rv;
if((rv = rmsgpack_dom_write(fd, &obj->map.items[i].value)) < 0)
return rv;
written += rv;
}
break;
case RDT_ARRAY:
if ((rv = rmsgpack_write_array_header(fd, obj->array.len)) < 0)
return rv;
written += rv;
for (i = 0; i < obj->array.len; i++)
{
if((rv = rmsgpack_dom_write(fd, &obj->array.items[i])) < 0)
return rv;
written += rv;
}
}
return written;
}
int rmsgpack_dom_read(int fd, struct rmsgpack_dom_value *out)
{
int rv = 0;
struct dom_reader_state s;
s.i = 0;
s.stack[0] = out;
rv = rmsgpack_read(fd, &dom_reader_callbacks, &s);
if (rv < 0)
rmsgpack_dom_value_free(out);
return rv;
}
int rmsgpack_dom_read_into(int fd, ...)
{
va_list ap;
struct rmsgpack_dom_value map;
int rv;
const char* key_name;
int value_type;
struct rmsgpack_dom_value key;
struct rmsgpack_dom_value *value;
int64_t *int_value;
uint64_t *uint_value;
int *bool_value;
char *buff_value;
uint64_t min_len;
va_start(ap, fd);
rv = rmsgpack_dom_read(fd, &map);
if (rv < 0)
return rv;
if (map.type != RDT_MAP)
{
rv = -EINVAL;
goto clean;
}
while (1)
{
key_name = va_arg(ap, const char*);
if (key_name == NULL)
{
rv = 0;
goto clean;
}
key.type = RDT_STRING;
key.string.len = strlen(key_name);
key.string.buff = (char *) key_name;
value = rmsgpack_dom_value_map_value(&map, &key);
switch(value->type)
{
case RDT_INT:
int_value = va_arg(ap, int64_t*);
*int_value = value->int_;
break;
case RDT_BOOL:
bool_value = va_arg(ap, int*);
*bool_value = value->bool_;
break;
case RDT_UINT:
uint_value = va_arg(ap, uint64_t*);
*uint_value = value->uint_;
break;
case RDT_BINARY:
buff_value = va_arg(ap, char*);
uint_value = va_arg(ap, uint64_t*);
*uint_value = value->binary.len;
min_len = value->binary.len > *uint_value ? *uint_value : value->binary.len;
memcpy(
buff_value,
value->binary.buff,
min_len
);
break;
case RDT_STRING:
buff_value = va_arg(ap, char*);
uint_value = va_arg(ap, uint64_t*);
min_len = value->string.len + 1 > *uint_value ? *uint_value : value->string.len + 1;
*uint_value = min_len;
memcpy(
buff_value,
value->string.buff,
min_len
);
break;
default:
rv = -1;
goto clean;
}
}
clean:
va_end(ap);
rmsgpack_dom_value_free(&map);
return 0;
}

56
rarchdb/rmsgpack_dom.h Normal file
View File

@ -0,0 +1,56 @@
#ifndef __RARCHDB_MSGPACK_DOM_H__
#define __RARCHDB_MSGPACK_DOM_H__
#include <stdint.h>
enum rmsgpack_dom_type {
RDT_NULL = 0,
RDT_BOOL,
RDT_UINT,
RDT_INT,
RDT_STRING,
RDT_BINARY,
RDT_MAP,
RDT_ARRAY
};
struct rmsgpack_dom_value {
enum rmsgpack_dom_type type;
union {
uint64_t uint_;
int64_t int_;
struct {
uint32_t len;
char* buff;
} string;
struct {
uint32_t len;
char* buff;
} binary;
int bool_;
struct {
uint32_t len;
struct rmsgpack_dom_pair *items;
} map;
struct {
uint32_t len;
struct rmsgpack_dom_value *items;
} array;
};
};
struct rmsgpack_dom_pair {
struct rmsgpack_dom_value key;
struct rmsgpack_dom_value value;
};
void rmsgpack_dom_value_print(struct rmsgpack_dom_value *obj);
void rmsgpack_dom_value_free(struct rmsgpack_dom_value *v);
int rmsgpack_dom_value_cmp(const struct rmsgpack_dom_value *a, const struct rmsgpack_dom_value *b);
struct rmsgpack_dom_value *rmsgpack_dom_value_map_value(const struct rmsgpack_dom_value *map, const struct rmsgpack_dom_value *key);
int rmsgpack_dom_read(int fd, struct rmsgpack_dom_value *out);
int rmsgpack_dom_write(int fd, const struct rmsgpack_dom_value *obj);
int rmsgpack_dom_read_into(int fd, ...);
#endif

191
rarchdb/rmsgpack_test.c Normal file
View File

@ -0,0 +1,191 @@
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdlib.h>
#include <unistd.h>
#include "rmsgpack.h"
struct stub_state
{
int i;
uint64_t stack[256];
};
static void stub_state_push_map(struct stub_state *s, uint32_t size)
{
s->i++;
s->stack[s->i] = 1;
s->i++;
s->stack[s->i] = size * 2;
printf("{");
}
static void stub_state_push_array(struct stub_state *s, uint32_t size)
{
s->i++;
s->stack[s->i] = 2;
s->i++;
s->stack[s->i] = size;
printf("[");
}
static void stub_state_pre_print(struct stub_state *s)
{
}
static void stub_state_post_print(struct stub_state *s)
{
switch(s->stack[s->i - 1])
{
case 1:
if (s->stack[s->i] % 2 == 0)
{
printf(": ");
s->stack[s->i]--;
}
else if (s->stack[s->i] == 1)
{
printf("}");
s->i -= 2;
stub_state_post_print(s);
}
else
{
printf(", ");
s->stack[s->i]--;
}
break;
case 2:
if (s->stack[s->i] == 1)
{
printf("]");
s->i -= 2;
stub_state_post_print(s);
}
else
{
printf(", ");
s->stack[s->i]--;
}
break;
}
}
static int stub_read_map_start(uint32_t size, void* data)
{
stub_state_push_map(data, size);
return 0;
}
static int stub_read_array_start(uint32_t size, void* data)
{
stub_state_push_array(data, size);
return 0;
}
static int stub_read_string(char *s, uint32_t len, void* data)
{
stub_state_pre_print(data);
printf("'%s'", s);
stub_state_post_print(data);
free(s);
return 0;
}
static int stub_read_bin(void *s, uint32_t len, void* data)
{
stub_state_pre_print(data);
printf("b'%s'", s);
stub_state_post_print(data);
free(s);
return 0;
}
static int stub_read_uint(uint64_t value, void* data)
{
stub_state_pre_print(data);
printf("%lu", value);
stub_state_post_print(data);
return 0;
}
static int stub_read_nil(void* data)
{
stub_state_pre_print(data);
printf("nil");
stub_state_post_print(data);
return 0;
}
static int stub_read_int(int64_t value, void* data)
{
stub_state_pre_print(data);
printf("%ld", value);
stub_state_post_print(data);
return 0;
}
static int stub_read_bool(int value, void* data)
{
stub_state_pre_print(data);
if (value)
printf("true");
else
printf("false");
stub_state_post_print(data);
return 0;
}
static struct rmsgpack_read_callbacks stub_callbacks = {
stub_read_nil,
stub_read_bool,
stub_read_int,
stub_read_uint,
stub_read_string,
stub_read_bin,
stub_read_map_start,
stub_read_array_start
};
int main(void)
{
int fd;
/*
int fd = open("test.msgpack", O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
int rv = 0;
if (fd == -1)
{
printf("Could not open file: %s", strerror(errno));
return errno;
}
rmsgpack_write_map_header(fd, 2);
rmsgpack_write_string(fd, "compact", strlen("compact"));
rmsgpack_write_bool(fd, 1);
rmsgpack_write_string(fd, "schema", strlen("schema"));
rmsgpack_write_array_header(fd, 10);
rmsgpack_write_string(fd, "schema", strlen("schema"));
rmsgpack_write_uint(fd, 1<<17);
rmsgpack_write_int(fd, (1<<17) + 1);
rmsgpack_write_int(fd, 4);
rmsgpack_write_int(fd, -3);
rmsgpack_write_int(fd, -22);
rmsgpack_write_int(fd, -35);
rmsgpack_write_int(fd, -421421412);
rmsgpack_write_int(fd, 4214);
rmsgpack_write_int(fd, -4214);
rmsgpack_write_uint(fd, 1<<17);
close(fd);
*/
struct stub_state state;
state.i = 0;
state.stack[0] = 0;
fd = open("test.msgpack", O_RDONLY);
rmsgpack_read(fd, &stub_callbacks, &state);
printf("\n");
close(fd);
return 0;
}