From ff523df873c1f3ba1b1d0e777e9517d1246c4ef9 Mon Sep 17 00:00:00 2001
From: ac-stef <ac_stef@users.sf.net>
Date: Mon, 31 Aug 2015 13:06:40 +0200
Subject: [PATCH] new http get function

* based on the enet socket layer
* supports redirects, range requests and gzip encoding
* automatically tries to re-use existing connections
* easy multithreading
* writes output to file or ram buffer
---
 source/codeblocks/AssaultCube.cbp       |   4 +
 source/codeblocks/AssaultCube_Linux.cbp |   3 +
 source/src/Makefile                     |   3 +
 source/src/http.cpp                     | 330 ++++++++++++++++++++++++
 source/src/protos.h                     |  35 +++
 5 files changed, 375 insertions(+)
 create mode 100644 source/src/http.cpp

diff --git a/source/codeblocks/AssaultCube.cbp b/source/codeblocks/AssaultCube.cbp
index 73019b6a..d1eed2ad 100755
--- a/source/codeblocks/AssaultCube.cbp
+++ b/source/codeblocks/AssaultCube.cbp
@@ -427,6 +427,10 @@
 			<Option target="default" />
 			<Option target="debug" />
 		</Unit>
+		<Unit filename="../src/http.cpp">
+			<Option target="default" />
+			<Option target="debug" />
+		</Unit>
 		<Unit filename="../src/serverchecks.h">
 			<Option target="default" />
 			<Option target="debug" />
diff --git a/source/codeblocks/AssaultCube_Linux.cbp b/source/codeblocks/AssaultCube_Linux.cbp
index c41ecfd8..85a2a496 100644
--- a/source/codeblocks/AssaultCube_Linux.cbp
+++ b/source/codeblocks/AssaultCube_Linux.cbp
@@ -284,6 +284,9 @@
 		<Unit filename="../src/serverbrowser.cpp">
 			<Option target="&lt;{~None~}&gt;" />
 		</Unit>
+		<Unit filename="../src/http.cpp">
+			<Option target="&lt;{~None~}&gt;" />
+		</Unit>
 		<Unit filename="../src/serverchecks.h">
 			<Option target="&lt;{~None~}&gt;" />
 		</Unit>
diff --git a/source/src/Makefile b/source/src/Makefile
index cf78d90f..406ee449 100644
--- a/source/src/Makefile
+++ b/source/src/Makefile
@@ -64,6 +64,7 @@ CLIENT_OBJS= \
 	docs.o \
 	editing.o \
 	entities.o \
+	http.o \
 	log.o \
 	main.o \
 	menus.o \
@@ -249,6 +250,8 @@ editing.o: console.h protos.h
 entities.o: cube.h platform.h tools.h geom.h model.h protocol.h sound.h
 entities.o: weapon.h entity.h world.h command.h varray.h vote.h
 entities.o: console.h protos.h
+http.o: cube.h platform.h tools.h geom.h model.h protocol.h sound.h weapon.h
+http.o: entity.h world.h command.h varray.h vote.h console.h protos.h
 log.o: cube.h platform.h tools.h geom.h model.h protocol.h sound.h weapon.h
 log.o: entity.h world.h command.h varray.h vote.h console.h protos.h
 main.o: cube.h platform.h tools.h geom.h model.h protocol.h sound.h weapon.h
diff --git a/source/src/http.cpp b/source/src/http.cpp
new file mode 100644
index 00000000..e0743d39
--- /dev/null
+++ b/source/src/http.cpp
@@ -0,0 +1,330 @@
+// very simple http get function
+
+#include "cube.h"
+#include <errno.h>
+
+#define CR 13
+#define LF 10
+#define TCP_TIMEOUT 5678      // only re-use quite fresh connections
+#define ERROR(msg) { err = msg; return -1; }
+
+const char lfcrlf[] = { LF, CR, LF, 0x00 }, *crlf = lfcrlf + 1;
+
+void httpget::reset(int keep) // keep == 0: delete all, keep == 1: keep connection, keep == 2: keep connection and url
+{
+    if(keep < 1)
+    { // reset connection
+        DELSTRING(hostname);
+        ip.host = ENET_HOST_ANY;
+        ip.port = 80;         // change manually, if needed
+        maxredirects = 3;
+        maxtransfer = maxsize = 1<<20; // 1 MB (caps transfer size and unzipped size)
+        disconnect();
+    }
+    err = NULL;
+    response = chunked = gzipped = contentlength = offset = elapsedtime = traffic = 0;
+    if(keep < 2) DELSTRING(url);
+    DELSTRING(header);
+    rawsnd.setsize(0);
+    rawrec.setsize(0);
+    datarec.setsize(0);
+}
+
+bool httpget::set_host(const char *newhostname) // set and resolve host
+{
+    if(!hostname || strcmp(hostname, newhostname))
+    {
+        DELSTRING(hostname);
+        disconnect(); // new or different host name -> close connection (if still open)
+
+        ENetAddress newhost;
+        if(enet_address_set_host(&newhost, newhostname) < 0)
+        {
+            err = "failed to resolve hostname";
+            return false;
+        }
+        ip.host = newhost.host;
+        hostname = newstring(newhostname);
+    }
+    return ip.host != ENET_HOST_ANY;
+}
+
+bool httpget::execcallback(float progress)
+{
+    if(callbackfunc && (*callbackfunc)(callbackdata, progress))
+    {
+        err = "manual abort";
+        return true;
+    }
+    return false;
+}
+
+void httpget::disconnect()
+{
+    if(tcp != ENET_SOCKET_NULL)
+    {
+        enet_socket_destroy(tcp);
+        tcp = ENET_SOCKET_NULL;
+    }
+}
+
+bool httpget::connect(bool force)
+{
+    if(force || tcp_age.elapsed() > TCP_TIMEOUT) disconnect();
+    if(tcp == ENET_SOCKET_NULL)
+    {
+        ENetSocket sock = enet_socket_create(ENET_SOCKET_TYPE_STREAM);
+        if(sock == ENET_SOCKET_NULL || enet_socket_connect(sock, &ip) < 0)
+        {
+            err = sock == ENET_SOCKET_NULL ? "could not open socket" : "could not connect";
+            if(sock != ENET_SOCKET_NULL) enet_socket_destroy(sock);
+            return false;
+        }
+        tcp = sock;
+        tcp_age.start();
+    }
+    return true;
+}
+
+int httpget::get(const char *url1, uint timeout, int range, bool head)
+{
+    int redirects = 0, res = 0, transferred = 0;
+    vector<char> *content = NULL;
+    ASSERT(timeout); // zero is not allowed
+    stopwatch to;
+    to.start();
+    reset(1); // clear old response values and url
+    url = newstring(url1);
+
+    bool reconnected = false, retry = false;
+    for(;;)
+    { // (in a loop because of possible redirects)
+        if(!connect()) return -1;
+
+        // assemble GET request
+        cvecprintf(rawsnd, "%s %s HTTP/1.1%s", head ? "HEAD" : "GET", url, crlf);
+        cvecprintf(rawsnd, "Host: %s%s", hostname, crlf);
+        cvecprintf(rawsnd, "Accept: */*%s", crlf);
+        if(referrer) cvecprintf(rawsnd, "Referer: %s%s", referrer, crlf);
+        if(useragent) cvecprintf(rawsnd, "User-Agent: %s%s", useragent, crlf);
+        if(range > 0) cvecprintf(rawsnd, "Range: bytes=%d-%sTE: gzip%s", range, crlf, crlf);  // content-encoding gzip for range requests is nonsense, so we'll ask for transfer-encoding instead (not that any server will deliver that...)
+        else if(!head) cvecprintf(rawsnd, "Accept-Encoding: gzip%s", crlf); // ask for gzip only for full requests, because we intend to unzip instantly
+        cvecprintf(rawsnd, "%s", crlf);
+
+        // send request
+        execcallback(0);
+        ENetBuffer buf;
+        buf.data = rawsnd.getbuf();
+        buf.dataLength = rawsnd.length();
+        while(buf.dataLength > 0)
+        {
+            enet_uint32 events = ENET_SOCKET_WAIT_SEND;
+            if(enet_socket_wait(tcp, &events, 250) >= 0 && events)
+            {
+                int sent = enet_socket_send(tcp, NULL, &buf, 1);
+                if(sent < 0)
+                {
+                    if(!reconnected)
+                    { // line went dead, maybe the server closed it - try reconnecting once
+                        retry = true;
+                        break;
+                    }
+                    else ERROR("connection failed");
+                }
+                buf.data = (char *)buf.data + sent;
+                buf.dataLength -= sent;
+                tcp_age.start();
+                traffic += sent;
+            }
+            float progress = ((float) to.elapsed()) / timeout;
+            if(progress > 1.0f) ERROR("timeout");
+            if(execcallback(progress)) return -1; // show progress bar, check for user interrupt
+        }
+
+        // get response
+        bool again = false, haveheader = false;
+        if(!retry) for(;;)
+        {
+            float progress = ((float) (elapsedtime = to.elapsed())) / timeout;
+            if(progress > 1.0f) ERROR("timeout");
+            enet_uint32 events = ENET_SOCKET_WAIT_RECEIVE;
+            if(rawrec.length() > maxsize || datarec.length() > maxsize) break;
+            if(enet_socket_wait(tcp, &events, 250) >= 0 && events)
+            {
+                if(rawrec.length() >= rawrec.capacity()) rawrec.reserve(4096);
+                buf.data = rawrec.getbuf() + rawrec.length();
+                buf.dataLength = rawrec.capacity() - rawrec.length();
+                int recv = enet_socket_receive(tcp, NULL, &buf, 1);
+                if(recv <= 0)
+                {
+                    if(!haveheader && !reconnected) retry = true; // line may be half dead - retry once
+                    break;
+                }
+                rawrec.advance(recv);
+                tcp_age.start();
+                traffic += recv;
+                if((transferred += recv) > maxtransfer) ERROR("transfer size exceeds limit");
+
+                // parsing received data
+                rawrec.add('\0'); // append '\0' to use string functions safely - will be removed again
+                if(!haveheader)
+                { // still getting header
+                    size_t headendlen = strlen(lfcrlf);
+                    const char *endhead = strstr((const char *)rawrec.getbuf(), lfcrlf);
+                    if(!endhead && --headendlen) endhead = strstr((const char *)rawrec.getbuf(), "\n\n"); // we're tolerant to LF only line endings
+                    if(endhead)
+                    {
+                        size_t headlen = endhead - rawrec.getbuf() + headendlen;
+                        header = newstring(rawrec.getbuf(), headlen);
+                        rawrec.remove(0, headlen);
+                        haveheader = true;
+
+                        // parse header
+                        char *header_uc = filtertext(newstring(headlen), header, FTXT_TOUPPER, headlen);
+                        char *p = strstr(header, "HTTP/");
+                        if(p) p = strchr(p, ' ');
+                        response = p ? atoi(p + 1) : -1;
+                        if(p) p = strchr(p, '\n');
+                        if(!p || response < 200 || response > 307) ERROR("server error");
+                        if(response >= 300) // handle redirects
+                        {
+                            p = strstr(header_uc + (p - header), "LOCATION: ");
+                            if(p) p = header + (p - header_uc);
+                            DELSTRING(header_uc);
+                            if(++redirects > maxredirects || !p) ERROR(p ? "too many redirects" : "redirect error");
+                            // read redirect url
+                            disconnect();
+                            p = strchr(p, ' ') + 1;
+                            char *r = strstr(p, "://");
+                            if(r) p = r + 3;
+                            if((r = strchr(p, '/')))
+                            {
+                                *r = '\0';
+                                again = set_host(p);
+                                *r = '/';
+                                p = r + strcspn(r, " \r\n");
+                                *p = '\0';
+                                const char *nurl = newstring(r);
+                                reset(1);
+                                url = nurl;
+                            }
+                            if(!again) ERROR("bad redirect url");
+                            break; // done receiving
+                        }
+                        // response was 2xx, check for some extras
+                        char *u = header_uc + (p - header), *f, *n;
+                        if((f = strstr(u, "TRANSFER-ENCODING: ")) && (n = strchr(f, LF)))
+                        {
+                            if((f = strstr(f, "CHUNKED")) && f < n) chunked = 1;
+                            if((f = strstr(f, "GZIP")) && f < n) gzipped = 1; // untested: I could not find a server who delivers that
+                        }
+                        if((f = strstr(u, "CONTENT-ENCODING: ")) && (n = strchr(f, LF)) && (f = strstr(f, "GZIP")) && f < n) gzipped = 1;
+                        if((f = strstr(u, "CONTENT-LENGTH: ")) && (f = strchr(f, ' '))) contentlength = atoi(f + 1);
+                        if((f = strstr(u, "CONTENT-RANGE: ")) && (n = strchr(f, LF)) && (f = strstr(f, "BYTES ")) && (f = strchr(f, ' ')) && f < n) offset = atoi(f + 1);
+                        DELSTRING(header_uc);
+                    }
+                }
+                if(haveheader)
+                { // check if data is complete
+                    const char *d = rawrec.getbuf(), *e = d;
+                    if(chunked)
+                    { // n * "chunksize\r\nchunkdata\r\n" + "0\r\n\r\n"
+                        while(strchr(d, LF))
+                        {
+                            int chunklen = (int) strtol(d, (char**) &e, 16);
+                            if(*e == CR) e++;
+                            if(*e == LF && rawrec.length() >= chunklen + (e - d) + 2 && (e[chunklen + 1] == LF || (e[chunklen + 1] == CR && e[chunklen + 2] == LF)))
+                            { // chunk is fully in buffer
+                                e++;
+                                loopi(chunklen) datarec.add(*e++);
+                                if(*e == CR) e++;
+                                if(*e++ != LF) ERROR("chunk frame error");
+                                rawrec.remove(0, int(e - d));
+                                if(chunklen == 0)
+                                {
+                                    contentlength = datarec.length();
+                                    content = &datarec;
+                                }
+                            }
+                            else break;
+                        }
+                    }
+                    else
+                    {
+                        if(contentlength > 0)
+                        {
+                            progress = float(rawrec.length() - 1) / contentlength; // get a real progress bar
+                            if(rawrec.length() - 1 >= contentlength) content = &rawrec;
+                        }
+                    }
+                }
+                rawrec.drop();
+            }
+            if(execcallback(progress)) return -1; // show progress bar, check for user interrupt
+            if(content || (haveheader && head)) break;
+        }
+        if(retry && !reconnected)
+        { // TCP connection showed some error before the response header was fully received: retry the connection once, it may have just been a timeout
+            again = reconnected = true;
+            reset(2); // only clear response values, not url
+            disconnect(); // but close the socket
+        }
+        if(again) continue; // redirect
+        else break;
+    }
+    if(!content) content = &rawrec; // end by "server close connection" - this should maybe be flagged as error instead
+    if(gzipped)
+    {
+        z_stream zs;
+        memset(&zs, 0, sizeof(zs));
+        if(inflateInit2(&zs, 15 + 32) == Z_OK)
+        {
+            zs.next_in = (Bytef*)content->getbuf();
+            zs.avail_in = content->length();
+            uchar outbuf[1<<15];
+            int zstat;
+            do
+            {
+                zs.next_out = (Bytef*)outbuf;
+                zs.avail_out = sizeof(outbuf);
+                zstat = inflate(&zs, Z_NO_FLUSH);
+                if(zstat == Z_OK || zstat == Z_STREAM_END)
+                {
+                    int got = sizeof(outbuf) - zs.avail_out;
+                    if(got > 0)
+                    {
+                        res += got;
+                        if(outvec) loopi(got) outvec->add(outbuf[i]);
+                        if(outstream) outstream->write(outbuf, got);
+                    }
+                }
+                else ERROR("gunzip error");
+                if(res > maxsize) break;
+            }
+            while(zstat == Z_OK);
+            inflateEnd(&zs);
+        }
+    }
+    else
+    {
+        res = content->length();
+        if(outvec) outvec->insert(outvec->length(), (const uchar *)content->getbuf(), res);
+        if(outstream) outstream->write(content->getbuf(), res);
+    }
+    if(res > maxsize) err = "output truncated";
+    elapsedtime = to.elapsed();
+    return res;
+}
+
+char *urlencode(const char *s, bool strict)
+{
+    vector<char> d;
+    while(*s)
+    {
+        if(isalnum(*s) || strchr("-_.~", *s) || (!strict && strchr(":/?#[]@!$&'()*+,;=", *s))) d.add(*s++); // see RFC 3986 2.2
+        else cvecprintf(d, "%%%02X", *s++);
+    }
+    d.add('\0');
+    return newstring(d.getbuf());
+}
+
diff --git a/source/src/protos.h b/source/src/protos.h
index 32505d29..50b22dba 100644
--- a/source/src/protos.h
+++ b/source/src/protos.h
@@ -238,6 +238,41 @@ extern serverinfo *getconnectedserverinfo();
 extern void pingservers();
 extern void updatefrommaster(int force);
 
+// http
+struct httpget
+{
+    // parameters
+    const char *hostname, *url;                        // set by set_host() and get()
+    const char *useragent, *referrer;                  // can be set manually with "newstrings", deleted automatically
+    const char *err;                                   // read-only
+    ENetAddress ip;                                    // set by set_host(), port may be changed manually
+    int maxredirects, maxtransfer, maxsize;            // set manually
+    int (__cdecl *callbackfunc)(void *data, float progress);
+    void *callbackdata;
+
+    // internal
+    ENetSocket tcp;
+    stopwatch tcp_age;
+    vector<char> rawsnd, rawrec, datarec;
+
+    // result
+    char *header;
+    int response, chunked, gzipped, contentlength, offset, elapsedtime, traffic;
+    vector<uchar> *outvec;                             // receives data, if not NULL (must be set up manually)
+    stream *outstream;                                 // receives data, if not NULL (must be set up manually)
+
+    httpget() { memset(&hostname, 0, sizeof(struct httpget)); tcp = ENET_SOCKET_NULL; reset(); }
+    ~httpget() { reset(); DELSTRING(useragent); DELSTRING(referrer); DELETEP(outvec); DELETEP(outstream); }
+    void reset(int keep = 0);                          // cleanup
+    bool set_host(const char *newhostname);            // set and resolve host
+    bool execcallback(float progress);
+    void disconnect();
+    bool connect(bool force = false);
+    int get(const char *url1, uint timeout, int range = 0, bool head = false); // get web ressource
+};
+
+extern char *urlencode(const char *s, bool strict = false);
+
 struct packetqueue
 {
     ringbuf<ENetPacket *, 8> packets;