From d2eb49ccbc5b7291d5fa9d1c73427d79b4da4387 Mon Sep 17 00:00:00 2001 From: Eric Warmenhoven Date: Wed, 22 Jan 2025 19:29:47 -0500 Subject: [PATCH] net_http refactor (#17460) * net_http: refactor net_http_new The goal is to move calls to getaddrinfo() and connect() into net_http_update(). This will make it possible for them to be replaced with non-blocking alternatives later. The net_http calling pattern right now allows callers to create the http_connection_t, call net_http_new() which creates the http_t from the http_connection_t, free the http_connection_t, and then start calling net_http_update(). In order to preserve that, the http_t needs to copy the values out of the http_connection_t on create. This also preserves the http_connection_t values instead of freeing them, so the connection would be able to be used later. * net_http: implement dns cache * net_http: separate out address resolution, connect, and request send * net_http: perform getaddrinfo on separate thread * net_http: implement basic connection pool * net_http: refactor receive calls to read faster, do fewer reallocs * net_http: build fix for platforms without SSL * net_http: build fix for non-griffin builds * net_http: build fix for non-threaded platforms --- libretro-common/net/net_http.c | 1510 ++++++++++++++++++++------------ 1 file changed, 952 insertions(+), 558 deletions(-) diff --git a/libretro-common/net/net_http.c b/libretro-common/net/net_http.c index 486459af9b..35834cce66 100644 --- a/libretro-common/net/net_http.c +++ b/libretro-common/net/net_http.c @@ -31,66 +31,116 @@ #include #endif #include +#include #include #include #include #include #include +#ifdef HAVE_THREADS +#include +#endif -enum +enum response_part { P_HEADER_TOP = 0, P_HEADER, P_BODY, P_BODY_CHUNKLEN, - P_DONE, - P_ERROR + P_DONE }; -enum +enum bodytype { T_FULL = 0, T_LEN, T_CHUNK }; -struct http_socket_state_t +struct conn_pool_entry { - void *ssl_ctx; + char *domain; + int port; int fd; + void *ssl_ctx; bool ssl; + bool connected; + bool in_use; + struct conn_pool_entry *next; }; +static struct conn_pool_entry *conn_pool = NULL; +#ifdef HAVE_THREADS +static slock_t *conn_pool_lock = NULL; +#endif + struct http_t { - char *data; - struct string_list *headers; - struct http_socket_state_t sock_state; /* ptr alignment */ - size_t pos; - size_t len; - size_t buflen; - int status; - char part; - char bodytype; bool error; + + struct conn_pool_entry *conn; + bool ssl; + bool request_sent; + + struct request + { + char *domain; + char *path; + char *method; + char *contenttype; + void *postdata; + char *useragent; + char *headers; + size_t contentlength; + int port; + } request; + + struct response + { + char *data; + struct string_list *headers; + size_t pos; + size_t len; + size_t buflen; + int status; + enum response_part part; + enum bodytype bodytype; + } response; }; struct http_connection_t { char *domain; - char *location; - char *urlcopy; + char *path; + char *url; char *scan; - char *methodcopy; - char *contenttypecopy; - void *postdatacopy; - char *useragentcopy; - char *headerscopy; - struct http_socket_state_t sock_state; /* ptr alignment */ - size_t contentlength; + char *method; + char *contenttype; + void *postdata; + char *useragent; + char *headers; + size_t contentlength; /* ptr alignment */ int port; + bool ssl; }; +struct dns_cache_entry +{ + char *domain; + int port; + struct addrinfo *addr; + retro_time_t timestamp; + bool valid; + struct dns_cache_entry *next; +}; + +static struct dns_cache_entry *dns_cache = NULL; +/* 5 min timeout, in usec */ +static const retro_time_t dns_cache_timeout = 1000 /* usec/ms */ * 1000 /* ms/s */ * 60 /* s/min */ * 5 /* min */; +#ifdef HAVE_THREADS +static slock_t *dns_cache_lock = NULL; +#endif + /** * net_http_urlencode: * @@ -424,85 +474,6 @@ void net_http_urlencode_full(char *s, const char *source, size_t len) free(tmp); } -static int net_http_new_socket(struct http_connection_t *conn) -{ - struct addrinfo *addr = NULL, *next_addr = NULL; - int fd = socket_init( - (void**)&addr, conn->port, conn->domain, SOCKET_TYPE_STREAM, 0); -#ifdef HAVE_SSL - if (conn->sock_state.ssl) - { - if (fd < 0) - goto done; - - if (!(conn->sock_state.ssl_ctx = ssl_socket_init(fd, conn->domain))) - { - socket_close(fd); - fd = -1; - goto done; - } - - /* TODO: Properly figure out what's going wrong when the newer - timeout/poll code interacts with mbed and winsock - https://github.com/libretro/RetroArch/issues/14742 */ - - /* Temp fix, don't use new timeout/poll code for cheevos http requests */ - bool timeout = true; -#ifdef __WIN32 - if (!strcmp(conn->domain, "retroachievements.org")) - timeout = false; -#endif - - if (ssl_socket_connect(conn->sock_state.ssl_ctx, addr, timeout, true) - < 0) - { - fd = -1; - goto done; - } - } - else -#endif - for (next_addr = addr; fd >= 0; fd = socket_next((void**)&next_addr)) - { - if (socket_connect_with_timeout(fd, next_addr, 5000)) - break; - - socket_close(fd); - } - -#ifdef HAVE_SSL -done: -#endif - if (addr) - freeaddrinfo_retro(addr); - - conn->sock_state.fd = fd; - - return fd; -} - -static void net_http_send_str( - struct http_socket_state_t *sock_state, bool *error, - const char *text, size_t text_size) -{ - if (*error) - return; -#ifdef HAVE_SSL - if (sock_state->ssl) - { - if (!ssl_socket_send_all_blocking( - sock_state->ssl_ctx, text, text_size, true)) - *error = true; - } - else -#endif - { - if (!socket_send_all_blocking( - sock_state->fd, text, text_size, true)) - *error = true; - } -} - struct http_connection_t *net_http_connection_new(const char *url, const char *method, const char *data) { @@ -510,42 +481,27 @@ struct http_connection_t *net_http_connection_new(const char *url, if (!url) return NULL; - if (!(conn = (struct http_connection_t*)malloc( - sizeof(*conn)))) + if (!(conn = (struct http_connection_t*)calloc(1, sizeof(*conn)))) return NULL; - conn->domain = NULL; - conn->location = NULL; - conn->urlcopy = NULL; - conn->scan = NULL; - conn->methodcopy = NULL; - conn->contenttypecopy = NULL; - conn->postdatacopy = NULL; - conn->useragentcopy = NULL; - conn->headerscopy = NULL; - conn->port = 0; - conn->sock_state.fd = 0; - conn->sock_state.ssl = false; - conn->sock_state.ssl_ctx= NULL; - if (method) - conn->methodcopy = strdup(method); + conn->method = strdup(method); if (data) { - conn->postdatacopy = strdup(data); + conn->postdata = strdup(data); conn->contentlength = strlen(data); } - if (!(conn->urlcopy = strdup(url))) + if (!(conn->url = strdup(url))) goto error; if (!strncmp(url, "http://", STRLEN_CONST("http://"))) - conn->scan = conn->urlcopy + STRLEN_CONST("http://"); + conn->scan = conn->url + STRLEN_CONST("http://"); else if (!strncmp(url, "https://", STRLEN_CONST("https://"))) { - conn->scan = conn->urlcopy + STRLEN_CONST("https://"); - conn->sock_state.ssl = true; + conn->scan = conn->url + STRLEN_CONST("https://"); + conn->ssl = true; } else goto error; @@ -558,15 +514,12 @@ struct http_connection_t *net_http_connection_new(const char *url, return conn; error: - if (conn->urlcopy) - free(conn->urlcopy); - if (conn->methodcopy) - free(conn->methodcopy); - if (conn->postdatacopy) - free(conn->postdatacopy); - conn->urlcopy = NULL; - conn->methodcopy = NULL; - conn->postdatacopy = NULL; + if (conn->url) + free(conn->url); + if (conn->method) + free(conn->method); + if (conn->postdata) + free(conn->postdata); free(conn); return NULL; } @@ -608,7 +561,7 @@ bool net_http_connection_done(struct http_connection_t *conn) else if (conn->port == 0) { /* port not specified, default to standard HTTP or HTTPS port */ - if (conn->sock_state.ssl) + if (conn->ssl) conn->port = 443; else conn->port = 80; @@ -616,39 +569,39 @@ bool net_http_connection_done(struct http_connection_t *conn) if (*conn->scan == '/') { - /* domain followed by location - split off the location */ + /* domain followed by path - split off the path */ /* site.com/path.html or site.com:80/path.html */ *conn->scan = '\0'; - conn->location = conn->scan + 1; + conn->path = conn->scan + 1; return true; } else if (!*conn->scan) { - /* domain with no location - point location at empty string */ + /* domain with no path - point path at empty string */ /* site.com or site.com:80 */ - conn->location = conn->scan; + conn->path = conn->scan; return true; } else if (*conn->scan == '?') { - /* domain with no location, but still has query parms - point location at the query parms */ + /* domain with no path, but still has query parms - point path at the query parms */ /* site.com?param=3 or site.com:80?param=3 */ if (!has_port) { /* if there wasn't a port, we have to expand the urlcopy so we can separate the two parts */ size_t domain_len = strlen(conn->domain); - size_t location_len = strlen(conn->scan); - char* urlcopy = (char*)malloc(domain_len + location_len + 2); + size_t path_len = strlen(conn->scan); + char* urlcopy = (char*)malloc(domain_len + path_len + 2); memcpy(urlcopy, conn->domain, domain_len); urlcopy[domain_len] = '\0'; - memcpy(urlcopy + domain_len + 1, conn->scan, location_len + 1); + memcpy(urlcopy + domain_len + 1, conn->scan, path_len + 1); - free(conn->urlcopy); - conn->domain = conn->urlcopy = urlcopy; - conn->location = conn->scan = urlcopy + domain_len + 1; + free(conn->url); + conn->domain = conn->url = urlcopy; + conn->path = conn->scan = urlcopy + domain_len + 1; } else /* There was a port, so overwriting the : will terminate the domain and we can just point at the ? */ - conn->location = conn->scan; + conn->path = conn->scan; return true; } @@ -662,30 +615,23 @@ void net_http_connection_free(struct http_connection_t *conn) if (!conn) return; - if (conn->urlcopy) - free(conn->urlcopy); + if (conn->url) + free(conn->url); - if (conn->methodcopy) - free(conn->methodcopy); + if (conn->method) + free(conn->method); - if (conn->contenttypecopy) - free(conn->contenttypecopy); + if (conn->contenttype) + free(conn->contenttype); - if (conn->postdatacopy) - free(conn->postdatacopy); + if (conn->postdata) + free(conn->postdata); - if (conn->useragentcopy) - free(conn->useragentcopy); + if (conn->useragent) + free(conn->useragent); - if (conn->headerscopy) - free(conn->headerscopy); - - conn->urlcopy = NULL; - conn->methodcopy = NULL; - conn->contenttypecopy = NULL; - conn->postdatacopy = NULL; - conn->useragentcopy = NULL; - conn->headerscopy = NULL; + if (conn->headers) + free(conn->headers); free(conn); } @@ -693,19 +639,19 @@ void net_http_connection_free(struct http_connection_t *conn) void net_http_connection_set_user_agent( struct http_connection_t *conn, const char *user_agent) { - if (conn->useragentcopy) - free(conn->useragentcopy); + if (conn->useragent) + free(conn->useragent); - conn->useragentcopy = user_agent ? strdup(user_agent) : NULL; + conn->useragent = user_agent ? strdup(user_agent) : NULL; } void net_http_connection_set_headers( struct http_connection_t *conn, const char *headers) { - if (conn->headerscopy) - free(conn->headerscopy); + if (conn->headers) + free(conn->headers); - conn->headerscopy = headers ? strdup(headers) : NULL; + conn->headers = headers ? strdup(headers) : NULL; } void net_http_connection_set_content( @@ -713,116 +659,582 @@ void net_http_connection_set_content( size_t content_length, const void *content) { - if (conn->contenttypecopy) - free(conn->contenttypecopy); - if (conn->postdatacopy) - free(conn->postdatacopy); + if (conn->contenttype) + free(conn->contenttype); + if (conn->postdata) + free(conn->postdata); - conn->contenttypecopy = content_type ? strdup(content_type) : NULL; + conn->contenttype = content_type ? strdup(content_type) : NULL; conn->contentlength = content_length; if (content_length) { - conn->postdatacopy = malloc(content_length); - memcpy(conn->postdatacopy, content, content_length); + conn->postdata = malloc(content_length); + memcpy(conn->postdata, content, content_length); } } const char *net_http_connection_url(struct http_connection_t *conn) { - return conn->urlcopy; + return conn->url; } const char* net_http_connection_method(struct http_connection_t* conn) { - return conn->methodcopy; + return conn->method; +} + +static void net_http_dns_cache_remove_expired(void) +{ + struct dns_cache_entry *entry = dns_cache; + struct dns_cache_entry *prev = NULL; + while (entry) + { + if (entry->timestamp + dns_cache_timeout < cpu_features_get_time_usec()) + { + if (prev) + prev->next = entry->next; + else + dns_cache = entry->next; + freeaddrinfo_retro(entry->addr); + free(entry->domain); + free(entry); + entry = prev ? prev->next : dns_cache; + } + else + { + prev = entry; + entry = entry->next; + } + } +} + +static struct dns_cache_entry *net_http_dns_cache_find(const char *domain, int port) +{ + struct dns_cache_entry *entry; + + net_http_dns_cache_remove_expired(); + + entry = dns_cache; + while (entry) + { + if (port == entry->port && string_is_equal(entry->domain, domain)) + { + entry->timestamp = cpu_features_get_time_usec(); + return entry; + } + entry = entry->next; + } + return NULL; +} + +static struct dns_cache_entry *net_http_dns_cache_add(const char *domain, int port, struct addrinfo *addr) +{ + struct dns_cache_entry *entry = (struct dns_cache_entry*)calloc(1, sizeof(*entry)); + if (!entry) + return NULL; + entry->domain = strdup(domain); + entry->port = port; + entry->addr = addr; + entry->timestamp = cpu_features_get_time_usec(); + entry->valid = (addr != NULL); + entry->next = dns_cache; + dns_cache = entry; + return entry; +} + +static void net_http_conn_pool_free(struct conn_pool_entry *entry) +{ +#ifdef HAVE_SSL + if (entry->ssl && entry->ssl_ctx) + { + ssl_socket_close(entry->ssl_ctx); + ssl_socket_free(entry->ssl_ctx); + } +#endif + if (entry->fd >= 0) + socket_close(entry->fd); + free(entry->domain); + free(entry); +} + +static void net_http_conn_pool_remove(struct conn_pool_entry *entry) +{ + struct conn_pool_entry *prev = NULL; + struct conn_pool_entry *current; + if (!entry) + return; +#ifdef HAVE_THREADS + slock_lock(conn_pool_lock); +#endif + current = conn_pool; + while (current) + { + if (current == entry) + { + if (prev) + prev->next = current->next; + else + conn_pool = current->next; + net_http_conn_pool_free(current); +#ifdef HAVE_THREADS + slock_unlock(conn_pool_lock); +#endif + return; + } + prev = current; + current = current->next; + } +#ifdef HAVE_THREADS + slock_unlock(conn_pool_lock); +#endif +} + +/* *NOT* thread safe, caller must lock */ +static void net_http_conn_pool_remove_expired(void) +{ + struct conn_pool_entry *entry = conn_pool; + struct conn_pool_entry *prev = NULL; + struct timeval tv = { 0 }; + int max = 0; + fd_set fds; + FD_ZERO(&fds); + entry = conn_pool; + while (entry) + { + if (!entry->in_use) + { + FD_SET(entry->fd, &fds); + if (entry->fd >= max) + max = entry->fd + 1; + } + entry = entry->next; + } + if (select(max, &fds, NULL, NULL, &tv) <= 0) + return; + entry = conn_pool; + while (entry) + { + if (!entry->in_use && FD_ISSET(entry->fd, &fds)) + { + char buf[4096]; + bool error = false; +#ifdef HAVE_SSL + if (entry->ssl && entry->ssl_ctx) + ssl_socket_receive_all_nonblocking(entry->ssl_ctx, &error, buf, sizeof(buf)); + else +#endif + socket_receive_all_nonblocking(entry->fd, &error, buf, sizeof(buf)); + + if (!error) + continue; + + if (prev) + prev->next = entry->next; + else + conn_pool = entry->next; + /* if it's not in use and it's reaadable we assume that means it's closed without checking recv */ + net_http_conn_pool_free(entry); + entry = prev ? prev->next : conn_pool; + } + else + { + prev = entry; + entry = entry->next; + } + } +} + +/* if it's not already in the pool, will add to end. + *NOT* thread safe, caller must lock */ +static void net_http_conn_pool_move_to_end(struct conn_pool_entry *entry) +{ + struct conn_pool_entry *prev = NULL; + struct conn_pool_entry *current = conn_pool; + /* 0 items in pool */ + if (!conn_pool) + { + conn_pool = entry; + entry->next = NULL; + return; + } + /* already only item in pool */ + if (conn_pool == entry && !conn_pool->next) + return; + while (current) + { + if (current != entry) + prev = current; + else + { + /* need to remove current */ + if (prev) + prev->next = current->next; + else + conn_pool = current->next; + } + current = current->next; + } + prev->next = entry; + entry->next = NULL; +} + +static struct conn_pool_entry *net_http_conn_pool_find(const char *domain, int port) +{ + struct conn_pool_entry *entry; + +#ifdef HAVE_THREADS + slock_lock(conn_pool_lock); +#endif + + net_http_conn_pool_remove_expired(); + + entry = conn_pool; + while (entry) + { + if (!entry->in_use && port == entry->port && string_is_equal(entry->domain, domain)) + { + entry->in_use = true; + net_http_conn_pool_move_to_end(entry); +#ifdef HAVE_THREADS + slock_unlock(conn_pool_lock); +#endif + return entry; + } + entry = entry->next; + } +#ifdef HAVE_THREADS + slock_unlock(conn_pool_lock); +#endif + return NULL; +} + +static struct conn_pool_entry *net_http_conn_pool_add(const char *domain, int port, int fd, bool ssl) +{ + struct conn_pool_entry *entry = (struct conn_pool_entry*)calloc(1, sizeof(*entry)); + if (!entry) + return NULL; + entry->domain = strdup(domain); + entry->port = port; + entry->fd = fd; + entry->in_use = true; + entry->ssl = ssl; + entry->connected = false; +#ifdef HAVE_THREADS + slock_lock(conn_pool_lock); +#endif + net_http_conn_pool_move_to_end(entry); +#ifdef HAVE_THREADS + slock_unlock(conn_pool_lock); +#endif + return entry; } struct http_t *net_http_new(struct http_connection_t *conn) { - bool error = false; -#ifdef HAVE_SSL - if (!conn || (net_http_new_socket(conn)) < 0) + struct http_t *state; + + if (!conn) return NULL; + + state = (struct http_t*)calloc(1, sizeof(struct http_t)); + if (!state) + return NULL; + + state->ssl = conn->ssl; + state->conn = NULL; + + state->request.domain = strdup(conn->domain); + state->request.path = strdup(conn->path); + state->request.method = strdup(conn->method); + state->request.contenttype = conn->contenttype ? strdup(conn->contenttype) : NULL; + state->request.contentlength = conn->contentlength; + if (conn->postdata && conn->contentlength) + { + state->request.postdata = malloc(conn->contentlength); + memcpy(state->request.postdata, conn->postdata, conn->contentlength); + } + state->request.useragent = conn->useragent ? strdup(conn->useragent) : NULL; + state->request.headers = conn->headers ? strdup(conn->headers) : NULL; + state->request.port = conn->port; + + state->response.status = -1; + state->response.buflen = 16 * 1024; + state->response.data = (char*)malloc(state->response.buflen); + state->response.headers = string_list_new(); + string_list_initialize(state->response.headers); + + return state; +} + +static void net_http_resolve(void *data) +{ + struct dns_cache_entry *entry = (struct dns_cache_entry*)data; + struct addrinfo hints = {0}; + struct addrinfo *addr = NULL; + char *domain; + int port; + char port_buf[6]; +#if defined(HAVE_SOCKET_LEGACY) || defined(WIIU) + int family = AF_INET; #else - int fd = -1; - if (!conn || (fd = net_http_new_socket(conn)) < 0) - return NULL; + int family = AF_UNSPEC; #endif - /* This is a bit lazy, but it works. */ - if (conn->methodcopy) + hints.ai_family = family; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags |= AI_NUMERICSERV; + +#ifdef HAVE_THREADS + slock_lock(dns_cache_lock); +#endif + domain = strdup(entry->domain); + port = entry->port; +#ifdef HAVE_THREADS + slock_unlock(dns_cache_lock); +#endif + + if (!network_init()) { - net_http_send_str(&conn->sock_state, &error, conn->methodcopy, - strlen(conn->methodcopy)); - net_http_send_str(&conn->sock_state, &error, " /", - STRLEN_CONST(" /")); +#ifdef HAVE_THREADS + slock_lock(dns_cache_lock); +#endif + entry->valid = true; + entry->addr = NULL; +#ifdef HAVE_THREADS + slock_unlock(dns_cache_lock); +#endif + free(domain); + return; + } + + snprintf(port_buf, sizeof(port_buf), "%hu", (unsigned short)port); + + getaddrinfo_retro(domain, port_buf, &hints, &addr); + free(domain); + +#ifdef HAVE_THREADS + slock_lock(dns_cache_lock); +#endif + entry->valid = true; + entry->addr = addr; +#ifdef HAVE_THREADS + slock_unlock(dns_cache_lock); +#endif +} + +static bool net_http_new_socket(struct http_t *state) +{ + struct addrinfo *addr = NULL; + struct dns_cache_entry *entry; + +#ifdef HAVE_THREADS + sthread_t *thread; + + if (!dns_cache_lock) + dns_cache_lock = slock_new(); + slock_lock(dns_cache_lock); +#endif + + entry = net_http_dns_cache_find(state->request.domain, state->request.port); + if (entry) + { + if (entry->valid) + { + int fd; + if (!entry->addr) + { +#ifdef HAVE_THREADS + slock_unlock(dns_cache_lock); +#endif + return false; + } + addr = entry->addr; + fd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol); + if (fd >= 0) + state->conn = net_http_conn_pool_add(state->request.domain, state->request.port, fd, state->ssl); +#ifdef HAVE_THREADS + /* still waiting on thread */ + slock_unlock(dns_cache_lock); +#endif + return (fd >= 0); + } + else + { +#ifdef HAVE_THREADS + /* still waiting on thread */ + slock_unlock(dns_cache_lock); +#endif + return true; + } } else { - net_http_send_str(&conn->sock_state, &error, "GET /", - STRLEN_CONST("GET /")); + entry = net_http_dns_cache_add(state->request.domain, state->request.port, NULL); +#ifdef HAVE_THREADS + /* create the entry for it as an indicator that the request is underway */ + thread = sthread_create(net_http_resolve, entry); + sthread_detach(thread); +#else + net_http_resolve(state); +#endif } - net_http_send_str(&conn->sock_state, &error, conn->location, - strlen(conn->location)); - net_http_send_str(&conn->sock_state, &error, " HTTP/1.1\r\n", - STRLEN_CONST(" HTTP/1.1\r\n")); +#ifdef HAVE_THREADS + slock_unlock(dns_cache_lock); +#endif - net_http_send_str(&conn->sock_state, &error, "Host: ", - STRLEN_CONST("Host: ")); - net_http_send_str(&conn->sock_state, &error, conn->domain, - strlen(conn->domain)); + return true; +} - if (conn->port) +static bool net_http_connect(struct http_t *state) +{ + struct addrinfo *addr = NULL, *next_addr = NULL; + struct conn_pool_entry *conn = state->conn; + struct dns_cache_entry *dns_entry = net_http_dns_cache_find(state->request.domain, state->request.port); + /* we just used/added this in _new_socket above, if it's not there it's a big bug */ + addr = dns_entry->addr; + +#ifdef HAVE_SSL + if (state->ssl) + { + if (!conn || conn->fd < 0) + return false; + + if (!(conn->ssl_ctx = ssl_socket_init(conn->fd, state->request.domain))) + { + net_http_conn_pool_remove(conn); + state->conn = NULL; + state->error = true; + return false; + } + + /* TODO: Properly figure out what's going wrong when the newer + timeout/poll code interacts with mbed and winsock + https://github.com/libretro/RetroArch/issues/14742 */ + + /* Temp fix, don't use new timeout/poll code for cheevos http requests */ + bool timeout = true; +#ifdef __WIN32 + if (!strcmp(state->request.domain, "retroachievements.org")) + timeout = false; +#endif + + if (ssl_socket_connect(conn->ssl_ctx, addr, timeout, true) < 0) + { + net_http_conn_pool_remove(conn); + state->conn = NULL; + state->error = true; + return false; + } + else + { + conn->connected = true; + return true; + } + } + else +#endif + { + for (next_addr = addr; conn->fd >= 0; conn->fd = socket_next((void**)&next_addr)) + { + if (socket_connect_with_timeout(conn->fd, next_addr, 5000)) + { + conn->connected = true; + return true; + } + + socket_close(conn->fd); + } + conn->fd = -1; /* already closed */ + net_http_conn_pool_remove(conn); + state->conn = NULL; + state->error = true; + return false; + } +} + +static void net_http_send_str( + struct http_t *state, const char *text, size_t text_size) +{ + if (state->error) + return; +#ifdef HAVE_SSL + if (state->ssl) + { + if (!ssl_socket_send_all_blocking( + state->conn->ssl_ctx, text, text_size, true)) + state->error = true; + } + else +#endif + { + if (!socket_send_all_blocking( + state->conn->fd, text, text_size, true)) + state->error = true; + } +} + +static bool net_http_send_request(struct http_t *state) +{ + struct request *request = &state->request; + + /* This is a bit lazy, but it works. */ + if (request->method) + { + net_http_send_str(state, request->method, strlen(request->method)); + net_http_send_str(state, " /", STRLEN_CONST(" /")); + } + else + net_http_send_str(state, "GET /", STRLEN_CONST("GET /")); + + net_http_send_str(state, request->path, strlen(request->path)); + net_http_send_str(state, " HTTP/1.1\r\n", STRLEN_CONST(" HTTP/1.1\r\n")); + + net_http_send_str(state, "Host: ", STRLEN_CONST("Host: ")); + net_http_send_str(state, request->domain, strlen(request->domain)); + + if (request->port) { char portstr[16]; size_t _len = 0; portstr[ _len] = ':'; portstr[++_len] = '\0'; _len += snprintf(portstr + _len, sizeof(portstr) - _len, - "%i", conn->port); - net_http_send_str(&conn->sock_state, &error, portstr, _len); + "%i", request->port); + net_http_send_str(state, portstr, _len); } - net_http_send_str(&conn->sock_state, &error, "\r\n", - STRLEN_CONST("\r\n")); + net_http_send_str(state, "\r\n", STRLEN_CONST("\r\n")); /* Pre-formatted headers */ - if (conn->headerscopy) - net_http_send_str(&conn->sock_state, &error, conn->headerscopy, - strlen(conn->headerscopy)); - if (conn->contenttypecopy) + if (request->headers) + net_http_send_str(state, request->headers, strlen(request->headers)); + if (request->contenttype) { - net_http_send_str(&conn->sock_state, &error, "Content-Type: ", - STRLEN_CONST("Content-Type: ")); - net_http_send_str(&conn->sock_state, &error, - conn->contenttypecopy, strlen(conn->contenttypecopy)); - net_http_send_str(&conn->sock_state, &error, "\r\n", - STRLEN_CONST("\r\n")); + net_http_send_str(state, "Content-Type: ", STRLEN_CONST("Content-Type: ")); + net_http_send_str(state, request->contenttype, strlen(request->contenttype)); + net_http_send_str(state, "\r\n", STRLEN_CONST("\r\n")); } - if (conn->methodcopy && (string_is_equal(conn->methodcopy, "POST") || string_is_equal(conn->methodcopy, "PUT"))) + if (request->method && (string_is_equal(request->method, "POST") || string_is_equal(request->method, "PUT"))) { size_t post_len, len; char *len_str = NULL; - if (!conn->postdatacopy && !string_is_equal(conn->methodcopy, "PUT")) - goto err; - - if (!conn->headerscopy) + if (!request->postdata && !string_is_equal(request->method, "PUT")) { - if (!conn->contenttypecopy) - net_http_send_str(&conn->sock_state, &error, - "Content-Type: application/x-www-form-urlencoded\r\n", - STRLEN_CONST( - "Content-Type: application/x-www-form-urlencoded\r\n" - )); + state->error = true; + return true; } - net_http_send_str(&conn->sock_state, &error, "Content-Length: ", - STRLEN_CONST("Content-Length: ")); + if (!request->headers && !request->contenttype) + net_http_send_str(state, + "Content-Type: application/x-www-form-urlencoded\r\n", + STRLEN_CONST("Content-Type: application/x-www-form-urlencoded\r\n")); - post_len = conn->contentlength; + net_http_send_str(state, "Content-Length: ", STRLEN_CONST("Content-Length: ")); + + post_len = request->contentlength; #ifdef _WIN32 len = snprintf(NULL, 0, "%" PRIuPTR, post_len); len_str = (char*)malloc(len + 1); @@ -835,82 +1247,26 @@ struct http_t *net_http_new(struct http_connection_t *conn) len_str[len] = '\0'; - net_http_send_str(&conn->sock_state, &error, len_str, - strlen(len_str)); - net_http_send_str(&conn->sock_state, &error, "\r\n", - STRLEN_CONST("\r\n")); + net_http_send_str(state, len_str, strlen(len_str)); + net_http_send_str(state, "\r\n", STRLEN_CONST("\r\n")); free(len_str); } - net_http_send_str(&conn->sock_state, &error, "User-Agent: ", - STRLEN_CONST("User-Agent: ")); - if (conn->useragentcopy) - net_http_send_str(&conn->sock_state, &error, - conn->useragentcopy, strlen(conn->useragentcopy)); + net_http_send_str(state, "User-Agent: ", STRLEN_CONST("User-Agent: ")); + if (request->useragent) + net_http_send_str(state, request->useragent, strlen(request->useragent)); else - net_http_send_str(&conn->sock_state, &error, "libretro", - STRLEN_CONST("libretro")); - net_http_send_str(&conn->sock_state, &error, "\r\n", - STRLEN_CONST("\r\n")); + net_http_send_str(state, "libretro", STRLEN_CONST("libretro")); + net_http_send_str(state, "\r\n", STRLEN_CONST("\r\n")); - net_http_send_str(&conn->sock_state, &error, - "Connection: close\r\n", STRLEN_CONST("Connection: close\r\n")); - net_http_send_str(&conn->sock_state, &error, "\r\n", - STRLEN_CONST("\r\n")); + net_http_send_str(state, "\r\n", STRLEN_CONST("\r\n")); - if (conn->postdatacopy && conn->contentlength) - net_http_send_str(&conn->sock_state, &error, conn->postdatacopy, - conn->contentlength); + if (request->postdata && request->contentlength) + net_http_send_str(state, request->postdata, request->contentlength); - if (!error) - { - struct http_t *state = (struct http_t*)malloc(sizeof(struct http_t)); - state->sock_state = conn->sock_state; - state->status = -1; - state->data = NULL; - state->part = P_HEADER_TOP; - state->bodytype = T_FULL; - state->error = false; - state->pos = 0; - state->len = 0; - state->buflen = 512; - - if ((state->data = (char*)malloc(state->buflen))) - { - if ((state->headers = string_list_new()) != NULL) - return state; - } - free(state); - } - -err: - if (conn) - { - if (conn->methodcopy) - free(conn->methodcopy); - if (conn->contenttypecopy) - free(conn->contenttypecopy); - if (conn->postdatacopy) - free(conn->postdatacopy); - conn->methodcopy = NULL; - conn->contenttypecopy = NULL; - conn->postdatacopy = NULL; - -#ifdef HAVE_SSL - if (conn->sock_state.ssl_ctx) - { - ssl_socket_close(conn->sock_state.ssl_ctx); - ssl_socket_free(conn->sock_state.ssl_ctx); - conn->sock_state.ssl_ctx = NULL; - } -#endif - } - -#ifndef HAVE_SSL - socket_close(fd); -#endif - return NULL; + state->request_sent = true; + return state->error; } /** @@ -923,9 +1279,198 @@ err: **/ int net_http_fd(struct http_t *state) { - if (!state) + if (!state || !state->conn) return -1; - return state->sock_state.fd; + return state->conn->fd; +} + +static ssize_t net_http_receive_header(struct http_t *state, ssize_t newlen) +{ + struct response *response = &state->response; + + response->pos += newlen; + + while (response->part < P_BODY) + { + char *dataend = response->data + response->pos; + char *lineend = (char*)memchr(response->data, '\n', response->pos); + + if (!lineend) + break; + + *lineend = '\0'; + + if (lineend != response->data && lineend[-1]=='\r') + lineend[-1] = '\0'; + + if (response->part == P_HEADER_TOP) + { + if (strncmp(response->data, "HTTP/1.", STRLEN_CONST("HTTP/1."))!=0) + { + response->part = P_DONE; + state->error = true; + return -1; + } + response->status = (int)strtoul(response->data + + STRLEN_CONST("HTTP/1.1 "), NULL, 10); + response->part = P_HEADER; + } + else + { + if (string_starts_with_case_insensitive(response->data, "Content-Length:")) + { + char* ptr = response->data + STRLEN_CONST("Content-Length:"); + while (ISSPACE(*ptr)) + ++ptr; + + response->bodytype = T_LEN; + response->len = strtol(ptr, NULL, 10); + } + else if (string_is_equal_case_insensitive(response->data, "Transfer-Encoding: chunked")) + response->bodytype = T_CHUNK; + + if (response->data[0]=='\0') + { + if (response->status == 100) + { + response->part = P_HEADER_TOP; + } + else + { + response->part = P_BODY; + if (response->bodytype == T_CHUNK) + response->part = P_BODY_CHUNKLEN; + } + } + else + { + union string_list_elem_attr attr; + attr.i = 0; + string_list_append(response->headers, response->data, attr); + } + } + + memmove(response->data, lineend + 1, dataend-(lineend+1)); + response->pos = (dataend-(lineend + 1)); + } + + if (response->part >= P_BODY) + { + newlen = response->pos; + response->pos = 0; + if (response->bodytype == T_LEN) + { + response->buflen = response->len; + response->data = (char*)realloc(response->data, response->buflen); + } + } + else + { + if (response->pos >= response->buflen - 64) + { + response->buflen *= 2; + response->data = (char*)realloc(response->data, response->buflen); + } + } + return newlen; +} + +static bool net_http_receive_body(struct http_t *state, ssize_t newlen) +{ + struct response *response = &state->response; + + if (newlen < 0 || state->error) + { + if (response->bodytype != T_FULL) + return false; + response->part = P_DONE; + if (response->buflen != response->len) + response->data = (char*)realloc(response->data, response->len); + return true; + } + +parse_again: + if (response->bodytype == T_CHUNK) + { + if (response->part == P_BODY_CHUNKLEN) + { + response->pos += newlen; + + if (response->pos - response->len >= 2) + { + /* + * len=start of chunk including \r\n + * pos=end of data + */ + + char *fullend = response->data + response->pos; + char *end = (char*)memchr(response->data + response->len + 2, '\n', + response->pos - response->len - 2); + + if (end) + { + size_t chunklen = strtoul(response->data + response->len, NULL, 16); + response->pos = response->len; + end++; + + memmove(response->data + response->len, end, fullend-end); + + response->len = chunklen; + newlen = (fullend - end); + + /* + len=num bytes + newlen=unparsed bytes after \n + pos=start of chunk including \r\n + */ + + response->part = P_BODY; + if (response->len == 0) + { + response->part = P_DONE; + response->len = response->pos; + response->data = (char*)realloc(response->data, response->len); + return true; + } + goto parse_again; + } + } + } + else if (response->part == P_BODY) + { + if ((size_t)newlen >= response->len) + { + response->pos += response->len; + newlen -= response->len; + response->len = response->pos; + response->part = P_BODY_CHUNKLEN; + goto parse_again; + } + response->pos += newlen; + response->len -= newlen; + } + } + else + { + response->pos += newlen; + + if (response->pos > response->len) + return false; + else if (response->pos == response->len) + { + response->part = P_DONE; + if (response->buflen != response->len) + response->data = (char*)realloc(response->data, response->len); + return true; + } + } + + if (response->pos >= response->buflen) + { + response->buflen *= 2; + response->data = (char*)realloc(response->data, response->buflen); + } + return true; } /** @@ -936,249 +1481,95 @@ int net_http_fd(struct http_t *state) **/ bool net_http_update(struct http_t *state, size_t* progress, size_t* total) { - if (state) + struct response *response; + ssize_t newlen = 0; + + if (!state) + return true; + + if (state->error) + return true; + + if (!state->conn) { - ssize_t newlen = 0; - if (state->error) - goto error; - - if (state->part < P_BODY) + state->conn = net_http_conn_pool_find(state->request.domain, state->request.port); + if (!state->conn) { - if (state->error) - goto error; - -#ifdef HAVE_SSL - if (state->sock_state.ssl && state->sock_state.ssl_ctx) - newlen = ssl_socket_receive_all_nonblocking(state->sock_state.ssl_ctx, &state->error, - (uint8_t*)state->data + state->pos, - state->buflen - state->pos); - else -#endif - newlen = socket_receive_all_nonblocking(state->sock_state.fd, &state->error, - (uint8_t*)state->data + state->pos, - state->buflen - state->pos); - - if (newlen < 0) - { - state->error = true; - goto error; - } - - if (state->pos + newlen >= state->buflen - 64) - { - state->buflen *= 2; - state->data = (char*)realloc(state->data, state->buflen); - } - state->pos += newlen; - - while (state->part < P_BODY) - { - char *dataend = state->data + state->pos; - char *lineend = (char*)memchr(state->data, '\n', state->pos); - - if (!lineend) - break; - - *lineend = '\0'; - - if (lineend != state->data && lineend[-1]=='\r') - lineend[-1] = '\0'; - - if (state->part == P_HEADER_TOP) - { - if (strncmp(state->data, "HTTP/1.", STRLEN_CONST("HTTP/1."))!=0) - { - state->error = true; - goto error; - } - state->status = (int)strtoul(state->data - + STRLEN_CONST("HTTP/1.1 "), NULL, 10); - state->part = P_HEADER; - } - else - { - if (string_starts_with_case_insensitive(state->data, "Content-Length:")) - { - char* ptr = state->data + STRLEN_CONST("Content-Length:"); - while (ISSPACE(*ptr)) - ++ptr; - - state->bodytype = T_LEN; - state->len = strtol(ptr, NULL, 10); - } - if (string_is_equal_case_insensitive(state->data, "Transfer-Encoding: chunked")) - state->bodytype = T_CHUNK; - - if (state->data[0]=='\0') - { - if (state->status == 100) - { - state->part = P_HEADER_TOP; - } - else - { - state->part = P_BODY; - if (state->bodytype == T_CHUNK) - state->part = P_BODY_CHUNKLEN; - } - } - else - { - union string_list_elem_attr attr; - attr.i = 0; - string_list_append(state->headers, state->data, attr); - } - } - - memmove(state->data, lineend + 1, dataend-(lineend+1)); - state->pos = (dataend-(lineend + 1)); - } - - if (state->part >= P_BODY) - { - newlen = state->pos; - state->pos = 0; - } + if (!net_http_new_socket(state)) + state->error = true; + return state->error; } - - if (state->part >= P_BODY && state->part < P_DONE) - { - if (!newlen) - { - if (state->error) - newlen = -1; - else if (state->len) - { -#ifdef HAVE_SSL - if (state->sock_state.ssl && state->sock_state.ssl_ctx) - newlen = ssl_socket_receive_all_nonblocking( - state->sock_state.ssl_ctx, - &state->error, - (uint8_t*)state->data + state->pos, - state->buflen - state->pos); - else -#endif - newlen = socket_receive_all_nonblocking( - state->sock_state.fd, - &state->error, - (uint8_t*)state->data + state->pos, - state->buflen - state->pos); - } - - if (newlen < 0) - { - if (state->bodytype != T_FULL) - { - state->error = true; - goto error; - } - state->part = P_DONE; - state->data = (char*)realloc(state->data, state->len); - newlen = 0; - } - - if (state->pos + newlen >= state->buflen - 64) - { - state->buflen *= 2; - state->data = (char*)realloc(state->data, state->buflen); - } - } - -parse_again: - if (state->bodytype == T_CHUNK) - { - if (state->part == P_BODY_CHUNKLEN) - { - state->pos += newlen; - - if (state->pos - state->len >= 2) - { - /* - * len=start of chunk including \r\n - * pos=end of data - */ - - char *fullend = state->data + state->pos; - char *end = (char*)memchr(state->data + state->len + 2, '\n', - state->pos - state->len - 2); - - if (end) - { - size_t chunklen = strtoul(state->data+state->len, NULL, 16); - state->pos = state->len; - end++; - - memmove(state->data+state->len, end, fullend-end); - - state->len = chunklen; - newlen = (fullend - end); - - /* - len=num bytes - newlen=unparsed bytes after \n - pos=start of chunk including \r\n - */ - - state->part = P_BODY; - if (state->len == 0) - { - state->part = P_DONE; - state->len = state->pos; - state->data = (char*)realloc(state->data, state->len); - } - goto parse_again; - } - } - } - else if (state->part == P_BODY) - { - if ((size_t)newlen >= state->len) - { - state->pos += state->len; - newlen -= state->len; - state->len = state->pos; - state->part = P_BODY_CHUNKLEN; - goto parse_again; - } - state->pos += newlen; - state->len -= newlen; - } - } - else - { - state->pos += newlen; - - if (state->pos > state->len) - { - state->error = true; - goto error; - } - else if (state->pos == state->len) - { - state->part = P_DONE; - state->data = (char*)realloc(state->data, state->len); - } - } - } - - if (progress) - *progress = state->pos; - - if (total) - { - if (state->bodytype == T_LEN) - *total = state->len; - else - *total = 0; - } - - if (state->part != P_DONE) - return false; } + + if (!state->conn->connected) + { + if (!net_http_connect(state)) + state->error = true; + return state->error; + } + + if (!state->request_sent) + return net_http_send_request(state); + + response = &state->response; + +#ifdef HAVE_SSL + if (state->ssl && state->conn->ssl_ctx) + newlen = ssl_socket_receive_all_nonblocking(state->conn->ssl_ctx, &state->error, + (uint8_t*)response->data + response->pos, + response->buflen - response->pos); + else +#endif + newlen = socket_receive_all_nonblocking(state->conn->fd, &state->error, + (uint8_t*)response->data + response->pos, + response->buflen - response->pos); + + if (response->part < P_BODY) + { + if (newlen < 0 || state->error) + goto error; + newlen = net_http_receive_header(state, newlen); + } + + if (response->part >= P_BODY && response->part < P_DONE) + { + if (!net_http_receive_body(state, newlen)) + goto error; + } + + if (progress) + *progress = response->pos; + + if (total) + { + if (response->bodytype == T_LEN) + *total = response->len; + else + *total = 0; + } + + if (response->part != P_DONE) + return false; + + for (newlen = 0; (size_t)newlen < response->headers->size; newlen++) + { + if (string_is_equal_case_insensitive(response->headers->elems[newlen].data, "connection: close")) + { + net_http_conn_pool_remove(state->conn); + state->conn = NULL; + return true; + } + } + + state->conn->in_use = false; + state->conn = NULL; return true; + error: - state->part = P_ERROR; - state->status = -1; + net_http_conn_pool_remove(state->conn); + state->error = true; + response->part = P_DONE; + response->status = -1; return true; } @@ -1195,7 +1586,7 @@ int net_http_status(struct http_t *state) { if (!state) return -1; - return state->status; + return state->response.status; } /** @@ -1215,7 +1606,7 @@ struct string_list *net_http_headers(struct http_t *state) if (state->error) return NULL; - return state->headers; + return state->response.headers; } /** @@ -1232,7 +1623,7 @@ uint8_t* net_http_data(struct http_t *state, size_t* len, bool accept_error) if (!state) return NULL; - if (!accept_error && (state->error || state->status < 200 || state->status > 299)) + if (!accept_error && (state->error || state->response.status < 200 || state->response.status > 299)) { if (len) *len = 0; @@ -1240,9 +1631,9 @@ uint8_t* net_http_data(struct http_t *state, size_t* len, bool accept_error) } if (len) - *len = state->len; + *len = state->response.len; - return (uint8_t*)state->data; + return (uint8_t*)state->response.data; } /** @@ -1255,19 +1646,22 @@ void net_http_delete(struct http_t *state) if (!state) return; - if (state->sock_state.fd >= 0) - { -#ifdef HAVE_SSL - if (state->sock_state.ssl && state->sock_state.ssl_ctx) - { - ssl_socket_close(state->sock_state.ssl_ctx); - ssl_socket_free(state->sock_state.ssl_ctx); - state->sock_state.ssl_ctx = NULL; - } - else -#endif - socket_close(state->sock_state.fd); - } + if (state->conn) + net_http_conn_pool_remove(state->conn); + if (state->request.domain) + free(state->request.domain); + if (state->request.path) + free(state->request.path); + if (state->request.method) + free(state->request.method); + if (state->request.contenttype) + free(state->request.contenttype); + if (state->request.postdata) + free(state->request.postdata); + if (state->request.useragent) + free(state->request.useragent); + if (state->request.headers) + free(state->request.headers); free(state); } @@ -1278,5 +1672,5 @@ void net_http_delete(struct http_t *state) **/ bool net_http_error(struct http_t *state) { - return (state->error || state->status < 200 || state->status > 299); + return (state->error || state->response.status < 200 || state->response.status > 299); }