From 7988443b1d139d5dc6c2a7bc93b21058647254ed Mon Sep 17 00:00:00 2001
From: ac-stef <ac_stef@users.sf.net>
Date: Mon, 31 Aug 2015 15:03:22 +0200
Subject: [PATCH] partial autodownload rewrite

* own source file for autodownload
* use cleaned up zip functions
* use new http get function (no more curl)
* enforce strict file naming rules and file size limits
* ping all package servers on every game start to determine ping times
* support media file update lists (also kept on package servers)
* allow users to manually prioritise package servers
* automatically select server with newest media file revision

new or changed commands:

addpckserver hostname [priority]
  Adds a host name to the list of package servers. If a priority is given,
  it influences the sorting of servers. Servers with higher priority are
  queried first. If servers have the same priority, they are sorted by
  ping. Default priority is zero.

getpackageserver
  Returns a table with four columns of all configured package servers. The
  columns are: host name, priority, ping and resolved.
  "resolved" is a flag, indicating, if the server answered the ping during
  game start.

removed cubescript idents: sortpckservers gamestarts
DOCREF addpckserver getpckserver
---
 source/codeblocks/AssaultCube.cbp       |   4 +
 source/codeblocks/AssaultCube_Linux.cbp |   3 +
 source/src/Makefile                     |   4 +
 source/src/autodownload.cpp             | 495 ++++++++++++++++++++++++
 source/src/client.cpp                   | 302 +--------------
 source/src/entity.h                     |  23 --
 source/src/main.cpp                     |  10 +-
 source/src/openal.cpp                   | 138 +++----
 source/src/protocol.h                   |   2 +
 source/src/protos.h                     |   8 +-
 source/src/rendermodel.cpp              |   6 +-
 source/src/serverbrowser.cpp            |   6 +-
 source/src/texture.cpp                  |   6 +-
 source/src/tools.cpp                    |  30 +-
 source/src/tools.h                      |   1 +
 source/src/zip.cpp                      | 163 --------
 16 files changed, 616 insertions(+), 585 deletions(-)
 create mode 100644 source/src/autodownload.cpp

diff --git a/source/codeblocks/AssaultCube.cbp b/source/codeblocks/AssaultCube.cbp
index d1eed2ad..6eae9e9e 100755
--- a/source/codeblocks/AssaultCube.cbp
+++ b/source/codeblocks/AssaultCube.cbp
@@ -431,6 +431,10 @@
 			<Option target="default" />
 			<Option target="debug" />
 		</Unit>
+		<Unit filename="../src/autodownload.cpp">
+			<Option target="default" />
+			<Option target="debug" />
+		</Unit>
 		<Unit filename="../src/serverchecks.h">
 			<Option target="default" />
 			<Option target="debug" />
diff --git a/source/codeblocks/AssaultCube_Linux.cbp b/source/codeblocks/AssaultCube_Linux.cbp
index 85a2a496..e5514205 100644
--- a/source/codeblocks/AssaultCube_Linux.cbp
+++ b/source/codeblocks/AssaultCube_Linux.cbp
@@ -287,6 +287,9 @@
 		<Unit filename="../src/http.cpp">
 			<Option target="&lt;{~None~}&gt;" />
 		</Unit>
+		<Unit filename="../src/autodownload.cpp">
+			<Option target="&lt;{~None~}&gt;" />
+		</Unit>
 		<Unit filename="../src/serverchecks.h">
 			<Option target="&lt;{~None~}&gt;" />
 		</Unit>
diff --git a/source/src/Makefile b/source/src/Makefile
index 406ee449..811a188e 100644
--- a/source/src/Makefile
+++ b/source/src/Makefile
@@ -56,6 +56,7 @@ endif
 CLIENT_OBJS= \
 	crypto.o \
 	audiomanager.o \
+	autodownload.o \
 	client.o \
 	clientgame.o \
 	clients2c.o \
@@ -224,6 +225,9 @@ crypto.o: console.h protos.h
 audiomanager.o: cube.h platform.h tools.h geom.h model.h protocol.h sound.h
 audiomanager.o: weapon.h entity.h world.h command.h varray.h vote.h
 audiomanager.o: console.h protos.h
+autodownload.o: cube.h platform.h tools.h geom.h model.h protocol.h sound.h
+autodownload.o: weapon.h entity.h world.h command.h varray.h vote.h console.h
+autodownload.o: protos.h
 client.o: cube.h platform.h tools.h geom.h model.h protocol.h sound.h
 client.o: weapon.h entity.h world.h command.h varray.h vote.h
 client.o: console.h protos.h bot/bot.h bot/bot_util.h bot/bot_waypoint.h
diff --git a/source/src/autodownload.cpp b/source/src/autodownload.cpp
new file mode 100644
index 00000000..15d5ed3a
--- /dev/null
+++ b/source/src/autodownload.cpp
@@ -0,0 +1,495 @@
+// autodownload.cpp
+// fetching missing media files from http servers (~ akimbo)
+
+#include "cube.h"
+
+struct pckserver
+{
+    string host;
+    int priority, ping, resolved;
+    hashtable<const char *, int> updates;
+    pckserver() { memset(&host, 0, (char *)&updates - (char *)&host); }
+};
+
+struct package
+{
+    string name, fullpath, requestname, host;
+    int type, iszip, strictnames, server;
+    const char **exts;
+    package() { memset(&name, 0, sizeof(struct package)); }
+};
+
+vector<pckserver *> pckservers;
+vector<package *> pendingpackages;
+
+VARP(autodownload, 0, 1, 1);
+
+sl_semaphore sem_pckservers(1, NULL); // control access to the pckservers vector
+
+void resetpckservers()
+{
+    delfile(findfile("config" PATHDIVS "pcksources.cfg", "w"));
+    conoutf("\f3Restart AssaultCube to take effect after resetting the list of packages source servers");
+}
+COMMAND(resetpckservers, "");
+
+void addpckserver(char *host, char *priority) // add/adjust a package server
+{
+    sem_pckservers.wait();
+    if(*host)
+    {
+        pckserver *s = NULL;
+        loopv(pckservers) if(!strcmp(host, pckservers[i]->host)) s = pckservers[i];
+        if(!s) s = pckservers.add(new pckserver);
+        copystring(s->host, host);
+        if(*priority) s->priority = atoi(priority);
+    }
+    sem_pckservers.post();
+}
+COMMAND(addpckserver, "ss");
+
+void getpckserver() // return a table of all package servers with four columns
+{
+    sem_pckservers.wait();
+    vector<char> res;
+    loopv(pckservers) cvecprintf(res, "\"%s\" %d %d %d\n", pckservers[i]->host, pckservers[i]->priority, pckservers[i]->ping, pckservers[i]->resolved);
+    if(res.length()) res.last() = '\0';
+    else res.add('\0');
+    result(res.getbuf());
+    sem_pckservers.post();
+}
+COMMAND(getpckserver, "");
+
+SDL_mutex *pckpinglog_lock; // auxilliary locking for pckping_log
+vector<char> pckping_log; // access controlled by sem_pckservers (and pckpinglog_lock)
+const char *updatestxt = "/updates.txt";
+
+int pingpckserver(void *data) // fetch updates.txt from a media server, measure required time
+{
+    httpget h;
+    pckserver *s = (pckserver *)data;
+
+    s->resolved = h.set_host(s->host) ? 1 : 0;
+    SDL_mutexP(pckpinglog_lock);
+    cvecprintf(pckping_log, "lresolving hostname %s %s\n", s->host, s->resolved ? "succeeded" : "failed");
+    SDL_mutexV(pckpinglog_lock);
+    if(s->resolved)
+    {
+        int res = h.get(updatestxt, 5000, 0, true); // http HEAD request only - to get a nice ping value
+        if(h.response > 0)
+        {
+            s->ping = h.elapsedtime;
+            SDL_mutexP(pckpinglog_lock);
+            cvecprintf(pckping_log, "lhttp head %s%s response time: %d, status: %d, contentlength: %d\n", s->host, updatestxt, s->ping, h.response, h.contentlength);
+            SDL_mutexV(pckpinglog_lock);
+            if(h.response == 200)
+            {
+                h.outvec = new vector<uchar>;
+                res = h.get(updatestxt, 5000);     // http GET
+                if(res >= 0 && h.response == 200)
+                { // parse updates.txt
+                    h.outvec->add('\0');
+                    s->updates.clear();
+                    parseupdatelist(s->updates, (char *)h.outvec->getbuf()); // don't filter entries
+                    SDL_mutexP(pckpinglog_lock);
+                    cvecprintf(pckping_log, "lgot %s%s: %d bytes, %d valid update lines\n", s->host, updatestxt, res, s->updates.numelems);
+                    SDL_mutexV(pckpinglog_lock);
+                }
+                else
+                {
+                    SDL_mutexP(pckpinglog_lock);
+                    cvecprintf(pckping_log, "lretrieving %s%s failed: response %d%s%s\n", s->host, updatestxt, h.response, h.err ? ", err: " : "", h.err ? h.err : "");
+                    SDL_mutexV(pckpinglog_lock);
+                }
+                DELETEP(h.outvec);
+            }
+        }
+        else
+        { // connection to server failed
+            SDL_mutexP(pckpinglog_lock);
+            cvecprintf(pckping_log, "lpinging %s failed%s%s\n", s->host, h.err ? ", err: " : "", h.err ? h.err : "");
+            SDL_mutexV(pckpinglog_lock);
+        }
+    }
+    h.disconnect();
+    return 0;
+}
+
+int pingallpckservers(void *data)
+{
+    sem_pckservers.wait();
+    if(autodownload)
+    {
+        pckpinglog_lock = SDL_CreateMutex();
+        vector<SDL_Thread *> pckthreads;
+        int nop, good = 0;
+        loopv(pckservers) pckthreads.add(SDL_CreateThread(pingpckserver, pckservers[i])); // start pinging all servers at once
+        loopv(pckthreads) if(pckthreads[i]) SDL_WaitThread(pckthreads[i], &nop); // wait for all ping threads to finish
+        SDL_DestroyMutex(pckpinglog_lock);
+        loopv(pckservers)
+        {
+            pckserver *s = pckservers[i];
+            if(s->resolved && s->ping) good++;
+        }
+        int bad = pckservers.length() - good;
+        cvecprintf(pckping_log, "csuccessfully pinged %d media server%s, %d failure%s\n", good, good == 1 ? "" : "s", bad, bad == 1 ? "" : "s");
+    }
+    else cvecprintf(pckping_log, "c\f4(automatic media file download deactivated)\n");
+    pckping_log.add('e'); // signal end of thread
+    sem_pckservers.post();
+    return 0;
+}
+
+SDL_Thread *pingallpckthread = NULL;
+
+void setupautodownload()
+{
+    // fetch updates from all configured servers
+    // in a background thread
+    // during startup
+    pingallpckthread = SDL_CreateThread(pingallpckservers, NULL);
+}
+
+void pollautodownloadresponse() // thread-safe feedback from the autodownload-updater thread
+{
+    static int donepolling = 0;
+    if(donepolling) return;
+    if(!sem_pckservers.trywait())
+    {
+        if(pckping_log.length())
+        {
+            pckping_log.add('\0');
+            char *p, *l = pckping_log.getbuf();
+            do
+            { // break into single lines
+                if((p = strchr(l, '\n'))) *p = '\0';
+                if(*l == 'c') conoutf("%s", l + 1);  // line starts with "c" -> console
+                else if(*l == 'e') donepolling = 1;  // line starts with "e" -> end polling
+                else clientlogf("%s", l + 1);        // line starts with "l" -> log
+                l = p + 1;
+            }
+            while(p);
+            pckping_log.setsize(0);
+        }
+        sem_pckservers.post();
+    }
+    if(donepolling)
+    {
+        int nop;
+        SDL_WaitThread(pingallpckthread, &nop);    // detaching the thread would be preferrable, but SDL 1.2 doesn't provide that
+        pingallpckthread = NULL;
+    }
+}
+
+const char *mmexts[] = { ".md2", ".md3", ".cfg", ".txt", ".jpg", ".png", "" },
+           *mapexts[] = { ".cgz", ".cfg", "" },
+           *sbexts[] = { "_lf.jpg", "_rt.jpg", "_ft.jpg", "_bk.jpg", "_dn.jpg", "_up.jpg", "_license.txt", "" };
+
+bool requirepackage(int type, const char *name, const char *host)
+{
+    const char *prefix = NULL, *checkprefix = NULL, *forceext = NULL;
+    package *pck = new package;
+    switch(type)
+    {
+        case PCK_AUDIO:
+            prefix = "packages/audio/";
+            checkprefix = "ambience/";
+            forceext = ".ogg";
+            break;
+
+        case PCK_MAPMODEL:
+            prefix = "packages/models/";
+            checkprefix = "mapmodels/";
+            pck->iszip = 1;
+            pck->exts = mmexts;
+            break;
+
+        case PCK_TEXTURE:
+            checkprefix = "packages/";
+            forceext = ".jpg";
+            break;
+
+        case PCK_SKYBOX:
+            prefix = "packages/textures/skymaps/";
+            pck->iszip = pck->strictnames = 1;
+            pck->exts = sbexts;
+            break;
+
+        case PCK_MAP:
+            prefix = "packages/maps/";
+            pck->iszip = pck->strictnames = 1;
+            pck->exts = mapexts;
+            break;
+
+        default:
+            conoutf("\f3requirepackage: illegal package type %d (\"%s\")", type, name); // should probably be fatal()
+            delete pck;
+            return false;
+    }
+    string upath;
+    copystring(upath, name);
+    unixpath(upath);
+    filtertext(pck->name, name, FTXT__MEDIAFILEPATH); // also changes backslashes to slashes
+    if(strlen(upath) > 100 || strcmp(upath, pck->name) || (checkprefix && strncmp(pck->name, checkprefix, strlen(checkprefix))))
+    {
+        conoutf("\f3requirepackage: illegal media path \"%s\" (type: %d)", name, type);
+        delete pck;
+        return false;
+    }
+    if(forceext && strlen(pck->name) > strlen(forceext) && !strcmp(pck->name + strlen(pck->name) - strlen(forceext), forceext)) forceext = NULL; // filename already has required extension
+    formatstring(pck->fullpath)("%s%s%s", prefix ? prefix : "", pck->name, forceext ? forceext : "");
+    formatstring(pck->requestname)("/%s%s%s", pck->fullpath, type == PCK_MAP ? ".cgz" : "", pck->iszip ? ".zip" : ""); // special case map zips: file ending .cgz.zip
+
+    loopv(pendingpackages) if(!strcmp(pendingpackages[i]->requestname, pck->requestname))
+    { // request already queued
+        delete pck;
+        return false;
+    }
+    pck->type = type;
+    if(host) copystring(pck->host, host); // assign fixed server
+    pendingpackages.add(pck);
+    return true;
+}
+
+VAR(rereadtexturelists, 0, 1, 1); // flag to indicate additional texture files to trigger a menu rebuild (defaults to "1" to build the menus after startup)
+VAR(rereadsoundlists, 0, 1, 1); // same for map sounds
+
+void processdownload(package *pck, stream *f) // write downloaded data to file(s)
+{
+    f->seek(0);
+    if(pck->iszip)
+    { // unpack zip file
+        vector<const char *> files;
+        void *mz = zipmanualopen(f, files);
+        if(mz)
+        {
+            string filename, cleanzname;
+            loopv(files)
+            {
+                if(i >= MAXFILESINADZIP) break;
+                const char *zname = behindpath(files[i]); // ignore _all_ paths inside the zip
+                filtertext(cleanzname, zname, FTXT__MEDIAFILENAME);
+                bool isok = false;
+                for(int n = 0; *pck->exts[n]; n++)
+                {
+                    if(pck->strictnames)
+                    { // only base names and listed extensions allowed
+                        formatstring(filename)("%s%s", pck->fullpath, pck->exts[n]);
+                        isok = !strcmp(behindpath(filename), zname);
+                    }
+                    else
+                    { // any name with listed extension, written to subdirectory
+                        int zlen = (int)strlen(zname), elen = (int)strlen(pck->exts[n]);
+                        if((isok = zlen > elen && !strcmp(zname + zlen - elen, pck->exts[n]))) formatstring(filename)("%s/%s", pck->fullpath, zname);
+                    }
+                    if(isok) break;
+                }
+                if(isok && !strcmp(cleanzname, zname))
+                {
+                    stream *zf = openfile(path(filename), "wb");
+                    if(zf)
+                    {
+                        int got = zipmanualread(mz, i, zf, MAXMEDIADOWNLOADFILESIZE); // extract file from zip and write to @filename
+                        clientlogf("extracted %d bytes from %s %s, written to %s", got, pck->name, files[i], filename);
+                        delete zf;
+                    }
+                }
+                else conoutf("\f3illegal filename \"%s\" in %s", files[i], pck->requestname);
+            }
+            zipmanualclose(mz);
+        }
+        else clientlogf("failed to open zip file %s", pck->requestname);
+    }
+    else
+    { // write single file
+        stream *d = openfile(path(pck->fullpath, true), "wb");
+        if(d)
+        {
+            streamcopy(d, f, MAXMEDIADOWNLOADFILESIZE);
+            delete d;
+        }
+        delete f;
+    }
+
+    switch(pck->type) // some housekeeping
+    {
+        case PCK_SKYBOX:
+        case PCK_TEXTURE:
+            rereadtexturelists = 1;
+            break;
+
+        case PCK_AUDIO:
+            rereadsoundlists = 1;
+            break;
+
+        case PCK_MAP:
+        case PCK_MAPMODEL:
+            break;
+    }
+}
+
+bool canceldownloads = false;
+int progress_n, progress_of;
+
+int progress_callback_dlpackage(void *data, float progress)
+{
+    loadingscreen("downloading package %d of %d...\n%s  %d%%\n(ESC to cancel)", progress_n, progress_of, (const char *)data, int(100.0 * progress));
+    if(interceptkey(SDLK_ESCAPE))
+    {
+        canceldownloads = true;
+        loadingscreen();
+        return 1;
+    }
+    return 0;
+}
+
+bool dlpackage(httpget &h, package *pck, pckserver *s) // download one package from one server
+{
+    const char *host = s ? s->host : pck->host;
+    h.callbackfunc = progress_callback_dlpackage;
+    h.callbackdata = pck->name;
+    progress_n = progress_of - pendingpackages.length();
+    if(*host && h.set_host(host))
+    {
+        h.outstream = openvecfile(NULL);
+        int got = h.get(pck->requestname, 60000);
+        if(!h.response)
+        {
+            if(!canceldownloads && s) s->ping = 0; // received a hard error from the server connection, better not try this one again
+            clientlogf("download %s%s failed%s%s", host, pck->requestname, h.err ? ", err: " : "", h.err ? h.err : "");
+            h.disconnect();
+        }
+        else
+        {
+            if(h.response == 200)
+            {
+                clientlogf("downloaded %s%s, %d bytes (%d raw), %d msec", host, pck->requestname, got, h.contentlength, h.elapsedtime);
+                processdownload(pck, h.outstream);
+                h.outstream = NULL; // deleted by processdownload()
+                return true;
+            }
+            else clientlogf("download %s%s failed, server response %d%s%s", host, pck->requestname, h.response, h.err ? ", err: " : "", h.err ? h.err : "");
+        }
+        DELETEP(h.outstream);
+    }
+    else
+    {
+        if(s) s->resolved = 0;
+        clientlogf("resolving host \"%s\" failed", host);
+        h.disconnect();
+    }
+    return false;
+}
+
+int packagesort_server(package **a, package **b) { return (*b)->server - (*a)->server; }
+int packagesort_name(package **a, package **b) { return strcmp((*b)->name, (*a)->name); }
+int packagesort_host(package **a, package **b) { return strcmp((*b)->host, (*a)->host); }
+
+int pckserversort(pckserver **a, pckserver **b)
+{
+    if((*a)->resolved != (*b)->resolved) return (*b)->resolved - (*a)->resolved; // group resolvable hosts at the top
+    if((*a)->priority != (*b)->priority) return (*b)->priority - (*a)->priority; // manual priority wins over ping (high priority to the top)
+    if((*a)->ping && (*b)->ping) return (*a)->ping - (*b)->ping; // smaller ping wins
+    return (*b)->ping - (*a)->ping; // or whichever ping is not zero
+}
+
+int downloadpackages() // get all pending packages
+{
+    bool failed = false;
+    httpget h;
+    canceldownloads = false;
+    progress_of = pendingpackages.length();
+    sem_pckservers.wait();
+    pckservers.sort(pckserversort);
+    loopv(pendingpackages)
+    {
+        package *pck = pendingpackages[i];
+        int maxrev = 0, *rev = NULL;
+        pck->server = -1;                  // server with highest revision for that content
+        loopvj(pckservers)
+        {
+            pckserver *s = pckservers[j];
+            if(s->resolved && s->ping && (rev = s->updates.access(pck->requestname)) && *rev > maxrev)
+            {
+                maxrev = *rev;
+                pck->server = j;
+            }
+        }
+    }
+
+    // get all packages with auto-assigned servers
+    pendingpackages.sort(packagesort_server);
+    loopvrev(pendingpackages)
+    {
+        if(canceldownloads) break;
+        package *pck = pendingpackages[i];
+        if(!*pck->host && pckservers.inrange(pck->server) && pckservers[pck->server]->resolved)
+        {
+            if(dlpackage(h, pck, pckservers[pck->server])) delete pendingpackages.remove(i);
+            else pck->server = -1; // auto-assigned server didn't work - try any other now
+        }
+    }
+
+    // try all packages from all servers - high priority servers first
+    pendingpackages.sort(packagesort_name);
+    loopvj(pckservers)
+    {
+        pckserver *s = pckservers[j];
+        if(!s->resolved || !s->ping) continue;
+        loopv(pendingpackages)
+        {
+            if(canceldownloads) break;
+            package *pck = pendingpackages[i];
+            if(!*pck->host)
+            {
+                if(dlpackage(h, pck, s)) delete pendingpackages.remove(i--);
+            }
+            if(!s->resolved || !s->ping) break;
+        }
+    }
+    if(canceldownloads) pendingpackages.deletecontents();
+
+    // print error messages for all failed packages - and remove them
+    loopv(pendingpackages)
+    {
+        package *pck = pendingpackages[i];
+        if(!*pck->host)
+        {
+            conoutf("\f3failed to load %s from any host, giving up", pck->name);
+            delete pendingpackages.remove(i--);
+            failed = true;
+        }
+    }
+
+    // get all packages with manually assigned hosts - those will be deleted from the list regardless of success!
+    pendingpackages.sort(packagesort_host);
+    loopvrev(pendingpackages)
+    {
+        package *pck = pendingpackages[i];
+        if(!canceldownloads && !dlpackage(h, pck, NULL))
+        {
+            conoutf("\f3failed to load %s from %s, giving up", pck->name, pck->host);
+            failed = true;
+        }
+        delete pendingpackages.remove(i);
+    }
+    sem_pckservers.post();
+    h.disconnect();
+    return failed ? 0 : h.traffic;
+}
+
+void writepcksourcecfg()
+{
+    if(pckservers.length())
+    {
+        stream *f = openfile("config" PATHDIVS "pcksources.cfg", "w");
+        if(!f) return;
+        f->printf("// list of package source servers (only add servers you trust!)\n\n");
+        loopv(pckservers)
+        {
+            pckserver *s = pckservers[i];
+            f->printf("addpckserver %s %d // ping: %d, resolved: %d, updates: %d\n", s->host, s->priority, s->ping, s->resolved, s->updates.numelems);
+        }
+        delete f;
+    }
+}
diff --git a/source/src/client.cpp b/source/src/client.cpp
index 6f6c00c6..ebc26899 100644
--- a/source/src/client.cpp
+++ b/source/src/client.cpp
@@ -796,8 +796,7 @@ void getmap(char *name, char *callback)
     }
     else
     {
-        defformatstring(package)("packages/maps/%s.cgz", name);
-        requirepackage(PCK_MAP, package);
+        requirepackage(PCK_MAP, name);
         if(downloadpackages())
         {
             if(callback && *callback) execute(callback);
@@ -883,302 +882,3 @@ COMMAND(listdemos, "");
 COMMANDN(setmr, setminutesremaining, "i");
 COMMANDN(rewind, rewinddemo, "i");
 
-// packages auto - downloader
-
-// arrays
-vector<pckserver *> pckservers;
-hashtable<const char *, package *> pendingpackages;
-
-// cubescript
-VARP(autodownload, 0, 1, 1);
-
-void resetpckservers()
-{
-    pckservers.deletecontents();
-    delfile(findfile("config" PATHDIVS "pcksources.cfg", "w"));
-    conoutf("\f3Restart AssaultCube to take effect after resetting the list of packages source servers");
-}
-
-void addpckserver(char *addr)
-{
-    pckserver *srcserver = new pckserver();
-    srcserver->addr = newstring(addr);
-    srcserver->pending = true;
-    pckservers.add(srcserver);
-}
-
-COMMAND(resetpckservers, "");
-COMMAND(addpckserver, "s");
-
-// cURL / Network
-
-bool havecurl = false, canceldownloads = false;
-
-void setupcurl()
-{
-    if(curl_global_init(CURL_GLOBAL_NOTHING)) conoutf("\f3could not init cURL, content downloads not available");
-    else
-    {
-        havecurl = true;
-        execfile("config/pcksources.cfg");
-    }
-}
-
-// detect the "nearest" server
-int pckserversort(pckserver **a, pckserver **b)
-{
-    if((*a)->ping < 0) return ((*b)->ping < 0) ? 0 : 1;
-    if((*b)->ping < 0) return -1;
-
-    return (*a)->ping == (*b)->ping ? 0 : ((*a)->ping < (*b)->ping ? -1 : 1);
-}
-
-SDL_Thread* pingpcksrvthread = NULL;
-SDL_mutex *pingpcksrvlock = NULL;
-int pingpckservers(void *data)
-{
-    SDL_mutexP(pingpcksrvlock);
-    vector<pckserver> serverstoping;
-    loopv(pckservers) serverstoping.add(*pckservers[i]);
-    SDL_mutexV(pingpcksrvlock);
-    // measure the time it took to receive each's server response
-    // not very accurate but it should do the trick
-    loopv(serverstoping)
-    {
-        double ping = 0, namelookup = 0;
-        pckserver *serv = &serverstoping[i];
-        CURL *cu = curl_easy_init();
-        curl_easy_setopt(cu, CURLOPT_URL, serv->addr);
-        curl_easy_setopt(cu, CURLOPT_NOSIGNAL, 1);          // Fixes crashbug for some buggy libcurl versions (Linux)
-        curl_easy_setopt(cu, CURLOPT_NOPROGRESS, 1);
-        curl_easy_setopt(cu, CURLOPT_NOBODY, 1);            // don't download response body (as its size may vary a lot from one server to another)
-        curl_easy_setopt(cu, CURLOPT_CONNECTTIMEOUT, 10);   // the timeout should be large here
-        int result = curl_easy_perform(cu);
-        curl_easy_getinfo(cu, CURLINFO_TOTAL_TIME, &ping);
-        curl_easy_getinfo(cu, CURLINFO_NAMELOOKUP_TIME, &namelookup);
-        ping -= namelookup; // ignore DNS lookup time as it should be performed only once
-        curl_easy_cleanup(cu);
-        cu = NULL;
-        if(result == CURLE_OPERATION_TIMEDOUT || result == CURLE_COULDNT_RESOLVE_HOST)
-            serv->responsive = false;
-        else
-            serv->ping = (int)(ping*1000);
-    }
-
-    SDL_mutexP(pingpcksrvlock);
-    loopv(serverstoping) loopvj(pckservers) if(!strcmp(serverstoping[i].addr, pckservers[j]->addr)) *pckservers[j] = serverstoping[i];
-    pckservers.sort(pckserversort);
-    // print the results. we should make it silent once tested enough
-    loopv(pckservers)
-    {
-        pckserver *serv = pckservers[i];
-        if(serv->ping > 0) clientlogf("%d. %s (%d ms)", i+1, serv->addr, serv->ping);
-        else clientlogf("%d. %s (did not reply)", i+1, serv->addr);
-    }
-    SDL_mutexV(pingpcksrvlock);
-
-    SDL_DestroyMutex(pingpcksrvlock);
-    pingpcksrvthread = NULL;
-    pingpcksrvlock = NULL;
-    return 0;
-}
-
-void sortpckservers()
-{
-    if(pingpcksrvthread || !havecurl) return;
-    pingpcksrvlock = SDL_CreateMutex();
-    pingpcksrvthread = SDL_CreateThread(pingpckservers, NULL);
-}
-COMMAND(sortpckservers, "");
-
-size_t write_callback(void *ptr, size_t size, size_t nmemb, FILE *stream)
-{
-    return fwrite(ptr, size, nmemb, stream);
-}
-
-int progress_callback_dlpackage(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
-{
-    package *pck = (package *)clientp;
-    loadingscreen("downloading package %d out of %d...\n%s %.0f/%.0f KB (%.1f%%)\n(ESC to cancel)", pck->number + 1, pck->number + pendingpackages.numelems,
-        pck->name, dlnow/double(1024.0), dltotal/double(1024.0), dltotal == 0 ? 0 : (dlnow/dltotal * double(100.0)));
-    if(interceptkey(SDLK_ESCAPE))
-    {
-        canceldownloads = true;
-        loadingscreen();
-        return 1;
-    }
-    return 0;
-}
-
-VAR(rereadtexturelists, 0, 1, 1); // flag to indicate additional texture files to trigger a menu rebuild (defaults to "1" to build the menus after startup)
-VAR(rereadsoundlists, 0, 1, 1); // same for map sounds
-
-int processdownload(package *pck)
-{
-    string tmpname = "";
-    copystring(tmpname, findfile("tmp", "rb"));
-    if(!pck->pending)
-    {
-        switch(pck->type)
-        {
-            case PCK_TEXTURE: case PCK_AUDIO:
-            {
-                const char *pckname = findfile(path(pck->name, true), "w+");
-                // with textures/sounds, the image/audio file itself is sent. Just need to copy it from the temporary file
-                if(rename(tmpname, pckname)) conoutf("\f3failed to install %s", pckname);
-                if(pck->type == PCK_TEXTURE) rereadtexturelists = 1;
-                else rereadsoundlists = 1;
-                break;
-            }
-
-            case PCK_MAP: case PCK_MAPMODEL:
-            {
-//                addzip("tmp", pck->name, NULL, true, pck->type);
-                break;
-            }
-
-            case PCK_SKYBOX:
-            {
-                char *fname = newstring(pck->name), *ls = strrchr(fname, '/');
-                if(ls) *ls = '\0';
-//                addzip("tmp", fname, NULL, true, pck->type);
-                rereadtexturelists = 1;
-                break;
-            }
-
-            default:
-                conoutf("could not install package %s", pck->name);
-                break;
-        }
-        delfile(tmpname);
-    }
-    return 0;
-}
-
-// download a package
-double dlpackage(package *pck)
-{
-    if(!pck || !pck->source) return false;
-    const char *tmpname = findfile("tmp", "wb");
-    FILE *outfile = fopen(tmpname, "wb");
-    string req, pckname = "";
-    formatstring(req)("%s/%s%s", pck->source->addr, strreplace(pckname, pck->name, " ", "%20"), (pck->type==PCK_MAP || pck->type==PCK_MAPMODEL || pck->type==PCK_SKYBOX) ? ".zip" : "");
-    conoutf("downloading %s from %s ...", pck->name, pck->source->addr);
-    if(!outfile)
-    {
-        pck->pending = false;
-        conoutf("\f3failed to write temporary file \"%s\"", tmpname);
-        return 0;
-    }
-
-    int result, httpresult = 0;
-    double dlsize;
-    pck->curl = curl_easy_init();
-    curl_easy_setopt(pck->curl, CURLOPT_URL, req);
-    curl_easy_setopt(pck->curl, CURLOPT_NOSIGNAL, 1);  // Fixes crashbug for some buggy libcurl versions (Linux)
-    curl_easy_setopt(pck->curl, CURLOPT_WRITEFUNCTION, write_callback);
-    curl_easy_setopt(pck->curl, CURLOPT_WRITEDATA, outfile);
-    curl_easy_setopt(pck->curl, CURLOPT_NOPROGRESS, 0);
-    curl_easy_setopt(pck->curl, CURLOPT_PROGRESSFUNCTION, progress_callback_dlpackage);
-    curl_easy_setopt(pck->curl, CURLOPT_PROGRESSDATA, pck);
-    curl_easy_setopt(pck->curl, CURLOPT_CONNECTTIMEOUT, 10);     // generous timeout for Bukz ;)
-    result = curl_easy_perform(pck->curl);
-    curl_easy_getinfo(pck->curl, CURLINFO_RESPONSE_CODE, &httpresult);
-    curl_easy_getinfo(pck->curl, CURLINFO_SIZE_DOWNLOAD, &dlsize);
-    curl_easy_cleanup(pck->curl);
-    pck->curl = NULL;
-    fclose(outfile);
-
-    pck->pending = false;
-    if(result == CURLE_OPERATION_TIMEDOUT || result == CURLE_COULDNT_RESOLVE_HOST)
-    {
-        // mark source unresponsive
-        pck->source->responsive = false;
-        conoutf("\f3could not connect to %s", pck->source->addr);
-
-        // try to find another source
-        pckserver *source = NULL;
-        loopv(pckservers) if(pckservers[i]->responsive) { source = pckservers[i]; break; }
-        if(!source)
-        {
-            conoutf("\f3no more servers to connect to");
-            canceldownloads = true;
-            return 0;
-        }
-
-        // update all packages
-        enumerate(pendingpackages, package *, pack,
-        {
-            pack->source = source;
-        });
-
-        pck->pending = true;
-
-        return dlpackage(pck); // retry current
-    }
-    if(!result && httpresult == 200) processdownload(pck);
-    else if(result == CURLE_ABORTED_BY_CALLBACK) conoutf("\f3download cancelled");
-    else conoutf("\f2request for %s failed (cURL %d, HTTP %d)", req, result, httpresult);
-    return (!result && httpresult == 200) ? dlsize : 0;
-}
-
-int lastpackagesdownload = 0;
-int downloadpackages()
-{
-    if(!pendingpackages.numelems) return 0;
-    else if(totalmillis - lastpackagesdownload < 2000)
-    {
-        conoutf("\f3quick download attempt aborted");
-        return 0;
-    }
-
-    double total = 0;
-    int downloaded = 0;
-    enumerate(pendingpackages, package *, pck,
-    {
-        if(!canceldownloads)
-        {
-            if(connected) c2skeepalive(); // try to avoid time out
-            pck->number = downloaded++;
-            total += dlpackage(pck);
-        }
-        pendingpackages.remove(pck->name);
-        delete pck;
-    });
-    canceldownloads = false;
-    lastpackagesdownload = lastmillis;
-    return (int)total;
-}
-
-bool requirepackage(int type, const char *path)
-{
-    pckserver *source = NULL;
-    loopv(pckservers) if(pckservers[i]->responsive) { source = pckservers[i]; break; }
-    if(!source) { conoutf("\f3no responsive source server found, can't download"); return false; }
-
-    char *upath = unixpath(newstring(path)), *cleanpath = newstring(path);
-    filtertext(cleanpath, cleanpath, FTXT__MEDIAFILEPATH);
-    if(strcmp(upath, cleanpath)) { conoutf("refuse to download \"%s\": illegal characters", path); delstring(upath); delstring(cleanpath); return false; }
-    if(!havecurl || canceldownloads || type < 0 || type >= PCK_NUM || pendingpackages.access(cleanpath)) { delstring(cleanpath); return false; }
-
-    package *pck = new package;
-    pck->name = cleanpath;
-    pck->type = type;
-    pck->source = source;
-    pck->pending = true;
-    pendingpackages.access(pck->name, pck);
-    delstring(upath);
-    return true;
-}
-
-void writepcksourcecfg()
-{
-    if(pckservers.length())
-    {
-        stream *f = openfile("config" PATHDIVS "pcksources.cfg", "w");
-        if(!f) return;
-        f->printf("// list of package source servers (only add servers you trust!)\n\n");
-        loopv(pckservers) f->printf("addpckserver %s\n", pckservers[i]->addr);
-        delete f;
-    }
-}
diff --git a/source/src/entity.h b/source/src/entity.h
index c3aa260e..888b4274 100644
--- a/source/src/entity.h
+++ b/source/src/entity.h
@@ -651,26 +651,3 @@ inline const char *killmessage(int gun, bool gib = false)
     return killmessages[gib?1:0][gun];
 }
 
-#ifndef STANDALONE
-struct pckserver
-{
-    char *addr;
-    bool pending, responsive;
-    int ping;
-
-    pckserver() : addr(NULL), pending(false), responsive(true), ping(-1) {}
-};
-
-enum { PCK_TEXTURE, PCK_SKYBOX, PCK_MAPMODEL, PCK_AUDIO, PCK_MAP, PCK_NUM };
-
-struct package
-{
-    char *name;
-    int type, number;
-    bool pending;
-    pckserver *source;
-    CURL *curl;
-
-    package() : name(NULL), type(-1), number(0), pending(false), source(NULL), curl(NULL) {}
-};
-#endif
diff --git a/source/src/main.cpp b/source/src/main.cpp
index b1715e5a..884ae8b8 100644
--- a/source/src/main.cpp
+++ b/source/src/main.cpp
@@ -902,7 +902,6 @@ static int clockrealbase = 0, clockvirtbase = 0;
 static void clockreset() { clockrealbase = SDL_GetTicks(); clockvirtbase = totalmillis; }
 VARFP(clockerror, 990000, 1000000, 1010000, clockreset());
 VARFP(clockfix, 0, 0, 1, clockreset());
-VARP(gamestarts, 0, 0, INT_MAX);
 
 const char *rndmapname()
 {
@@ -1269,7 +1268,9 @@ int main(int argc, char **argv)
     exec("config/securemaps.cfg");
     exec("config/admin.cfg");
     execfile("config/servers.cfg");
+    execfile("config/pcksources.cfg");
     loadhistory();
+    setupautodownload();
     int xmn = loadallxmaps();
     if(xmn) conoutf("loaded %d xmaps", xmn);
     per_idents = true;
@@ -1314,9 +1315,6 @@ int main(int argc, char **argv)
     initlog("models");
     preload_playermodels();
     preload_hudguns();
-    initlog("curl");
-    setupcurl();
-
     preload_entmodels();
 
     initlog("docs");
@@ -1340,9 +1338,6 @@ int main(int argc, char **argv)
 
     if(initscript) execute(initscript);
 
-    gamestarts = max(1, gamestarts+1);
-    if(autodownload && (gamestarts % 100 == 1)) sortpckservers();
-
     initlog("mainloop");
 
     inputgrab(grabinput = true);
@@ -1423,6 +1418,7 @@ int main(int argc, char **argv)
             direct_connect = false;
             connectserv(servername, &serverport, password);
         }
+        pollautodownloadresponse();
     }
 
     quit();
diff --git a/source/src/openal.cpp b/source/src/openal.cpp
index ffcda03e..a466d06d 100644
--- a/source/src/openal.cpp
+++ b/source/src/openal.cpp
@@ -266,95 +266,85 @@ bool sbuffer::load(bool trydl)
     {
         const char *exts[] = { "", ".wav", ".ogg" };
         string filepath;
-        loopk(2)
+        loopi(sizeof(exts)/sizeof(exts[0]))
         {
-            loopi(sizeof(exts)/sizeof(exts[0]))
+            formatstring(filepath)("packages/audio/%s%s", name, exts[i]);
+            stream *f = openfile(path(filepath), "rb");
+            if(!f) continue;
+
+            size_t len = strlen(filepath);
+            if(len >= 4 && !strcasecmp(filepath + len - 4, ".ogg"))
             {
-                formatstring(filepath)("packages/audio/%s%s", name, exts[i]);
-                stream *f = openfile(path(filepath), "rb");
-                if(!f && k>0 && trydl) // only try donwloading after trying all extensions
+                OggVorbis_File oggfile;
+                if(!ov_open_callbacks(f, &oggfile, NULL, 0, oggcallbacks))
                 {
-                    requirepackage(PCK_AUDIO, filepath);
-                    bool skip = false;
-                    loopj(sizeof(exts)/sizeof(exts[0])) if(strstr(name, exts[j])) skip = true;  // don't try extensions if name already has a known extension
-                    if(skip) break;
-                    continue;
-                }
-                if(!f) continue;
+                    vorbis_info *info = ov_info(&oggfile, -1);
 
-                size_t len = strlen(filepath);
-                if(len >= 4 && !strcasecmp(filepath + len - 4, ".ogg"))
-                {
-                    OggVorbis_File oggfile;
-                    if(!ov_open_callbacks(f, &oggfile, NULL, 0, oggcallbacks))
+                    const size_t BUFSIZE = 32*1024;
+                    vector<char> buf;
+                    int bitstream;
+                    long bytes;
+
+                    do
                     {
-                        vorbis_info *info = ov_info(&oggfile, -1);
+                        char buffer[BUFSIZE];
+                        bytes = ov_read(&oggfile, buffer, BUFSIZE, isbigendian(), 2, 1, &bitstream);
+                        loopi(bytes) buf.add(buffer[i]);
+                    } while(bytes > 0);
 
-                        const size_t BUFSIZE = 32*1024;
-                        vector<char> buf;
-                        int bitstream;
-                        long bytes;
-
-                        do
-                        {
-                            char buffer[BUFSIZE];
-                            bytes = ov_read(&oggfile, buffer, BUFSIZE, isbigendian(), 2, 1, &bitstream);
-                            loopi(bytes) buf.add(buffer[i]);
-                        } while(bytes > 0);
-
-                        alBufferData(id, info->channels == 2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16, buf.getbuf(), buf.length(), info->rate);
-                        ov_clear(&oggfile);
-                    }
-                    else
-                    {
-                        delete f;
-                        continue;
-                    }
+                    alBufferData(id, info->channels == 2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16, buf.getbuf(), buf.length(), info->rate);
+                    ov_clear(&oggfile);
                 }
                 else
                 {
-                    SDL_AudioSpec wavspec;
-                    uint32_t wavlen;
-                    uint8_t *wavbuf;
-
-                    if(!SDL_LoadWAV_RW(f->rwops(), 1, &wavspec, &wavbuf, &wavlen))
-                    {
-                        SDL_ClearError();
-                        continue;
-                    }
-
-                    ALenum format;
-                    switch(wavspec.format) // map wav header to openal format
-                    {
-                        case AUDIO_U8:
-                        case AUDIO_S8:
-                            format = wavspec.channels==2 ? AL_FORMAT_STEREO8 : AL_FORMAT_MONO8;
-                            break;
-                        case AUDIO_U16:
-                        case AUDIO_S16:
-                            format = wavspec.channels==2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16;
-                            break;
-                        default:
-                            SDL_FreeWAV(wavbuf);
-                            delete f;
-                            unload();
-                            return false;
-                    }
-
-                    alBufferData(id, format, wavbuf, wavlen, wavspec.freq);
-                    SDL_FreeWAV(wavbuf);
                     delete f;
+                    continue;
+                }
+            }
+            else
+            {
+                SDL_AudioSpec wavspec;
+                uint32_t wavlen;
+                uint8_t *wavbuf;
 
-                    if(ALERR)
-                    {
-                        unload();
-                        return false;
-                    };
+                if(!SDL_LoadWAV_RW(f->rwops(), 1, &wavspec, &wavbuf, &wavlen))
+                {
+                    SDL_ClearError();
+                    continue;
                 }
 
-                return true;
+                ALenum format;
+                switch(wavspec.format) // map wav header to openal format
+                {
+                    case AUDIO_U8:
+                    case AUDIO_S8:
+                        format = wavspec.channels==2 ? AL_FORMAT_STEREO8 : AL_FORMAT_MONO8;
+                        break;
+                    case AUDIO_U16:
+                    case AUDIO_S16:
+                        format = wavspec.channels==2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16;
+                        break;
+                    default:
+                        SDL_FreeWAV(wavbuf);
+                        delete f;
+                        unload();
+                        return false;
+                }
+
+                alBufferData(id, format, wavbuf, wavlen, wavspec.freq);
+                SDL_FreeWAV(wavbuf);
+                delete f;
+
+                if(ALERR)
+                {
+                    unload();
+                    return false;
+                };
             }
+
+            return true;
         }
+        if(trydl) requirepackage(PCK_AUDIO, name); // only try donwloading after trying all extensions
     }
     unload(); // loading failed
     return false;
diff --git a/source/src/protocol.h b/source/src/protocol.h
index b00bcc8e..3da9cb80 100644
--- a/source/src/protocol.h
+++ b/source/src/protocol.h
@@ -11,6 +11,8 @@
 #define DEMO_MINTIME 10000              // don't keep demo recordings with less than 10 seconds
 #define MAXMAPSENDSIZE 65536
 #define MAXCFGFILESIZE 65536
+#define MAXMEDIADOWNLOADFILESIZE 1024000 // hard cap on filesizes (raw and unzipped) - to limit the effect of zip bombs - no nice error messages: just cap
+#define MAXFILESINADZIP 21              // max number of files extracted from a zip by autodownload
 
 extern bool modprotocol;
 #define CUR_PROTOCOL_VERSION (modprotocol ? -PROTOCOL_VERSION : PROTOCOL_VERSION)
diff --git a/source/src/protos.h b/source/src/protos.h
index 50b22dba..d43df3a6 100644
--- a/source/src/protos.h
+++ b/source/src/protos.h
@@ -370,10 +370,12 @@ extern const char *crosshairnames[];
 extern Texture *crosshairs[];
 extern void drawcrosshair(playerent *p, int n, struct color *c = NULL, float size = -1.0f);
 
-// client
+// autodownload
+enum { PCK_TEXTURE = 0, PCK_SKYBOX, PCK_MAPMODEL, PCK_AUDIO, PCK_MAP, PCK_NUM };
 extern int autodownload;
-extern void setupcurl();
-extern bool requirepackage(int type, const char *path);
+extern void setupautodownload();
+extern void pollautodownloadresponse();
+extern bool requirepackage(int type, const char *name, const char *host = NULL);
 extern int downloadpackages();
 extern void sortpckservers();
 extern void writepcksourcecfg();
diff --git a/source/src/rendermodel.cpp b/source/src/rendermodel.cpp
index 21350eac..959001ca 100644
--- a/source/src/rendermodel.cpp
+++ b/source/src/rendermodel.cpp
@@ -279,11 +279,7 @@ model *loadmodel(const char *name, int i, bool trydl)     // load model by name
             {
                 delete m;
                 m = loadingmodel = NULL;
-                if(trydl)
-                {
-                    defformatstring(dl)("packages/models/%s", name);
-                    requirepackage(PCK_MAPMODEL, dl);
-                }
+                if(trydl && !strncmp(name, mmpath, strlen(mmpath))) requirepackage(PCK_MAPMODEL, name);
                 else
                 {
                     conoutf("\f3failed to load model %s", name);
diff --git a/source/src/serverbrowser.cpp b/source/src/serverbrowser.cpp
index ea559f3d..bc2c2271 100644
--- a/source/src/serverbrowser.cpp
+++ b/source/src/serverbrowser.cpp
@@ -1304,7 +1304,11 @@ void clearservers()
 }
 
 #define RETRIEVELIMIT 5000
-extern size_t write_callback(void *ptr, size_t size, size_t nmemb, FILE *stream);
+size_t write_callback(void *ptr, size_t size, size_t nmemb, FILE *stream)
+{
+    return fwrite(ptr, size, nmemb, stream);
+}
+
 extern char *global_name;
 bool cllock = false, clfail = false;
 
diff --git a/source/src/texture.cpp b/source/src/texture.cpp
index dc80a363..91474f69 100644
--- a/source/src/texture.cpp
+++ b/source/src/texture.cpp
@@ -384,8 +384,7 @@ GLuint loadsurface(const char *texname, int &xs, int &ys, int &bpp, int clamp =
     else if(!s) s = IMG_Load(ffile);
     if(!s)
     {
-        if(trydl)
-            requirepackage(PCK_TEXTURE, file);
+        if(trydl) requirepackage(PCK_TEXTURE, file);
         else if(!silent_texture_load) conoutf("couldn't load texture %s", texname);
         return 0;
     }
@@ -591,8 +590,7 @@ void loadskymap(bool reload)
         sky[i] = textureload(name, 3);
         if(sky[i] == notexture && !reload && autodownload && execcontext == IEXC_MAPCFG)
         {
-            defformatstring(dl)("packages/%s%s", legacyprefix, loadsky);
-            requirepackage(PCK_SKYBOX, dl);
+            requirepackage(PCK_SKYBOX, loadsky);
             break;
         }
     }
diff --git a/source/src/tools.cpp b/source/src/tools.cpp
index 4e31d5f0..940acce4 100644
--- a/source/src/tools.cpp
+++ b/source/src/tools.cpp
@@ -643,7 +643,29 @@ int sl_waitthread(void *ti)
 }
 #endif
 
-
-
-
-
+void parseupdatelist(hashtable<const char *, int> &ht, char *buf, const char *prefix, const char *suffix)
+{
+    for(char *d = buf; *d; d++) if(!isalnum(*d) && !strchr("._-() /\n", *d)) *d = ' '; // allowed chars in media path strings (except ' ')
+    char *p, *l = buf, *m;
+    int rev, *revp, plen = prefix ? (int)strlen(prefix) : 0, slen = suffix ? (int)strlen(suffix) : 0;
+    do
+    {
+        if((p = strchr(l, '\n'))) *p = '\0'; // break into single lines
+        l += strspn(l, " "); // skip leading spaces
+        if((m = strchr(l, ' ')) && (rev = atoi(m + 1))) // string has a space and a number != 0 after it
+        {
+            *m = '\0';
+            if((!plen || !strncmp(l, prefix, plen)) && // prefix is either not required or present
+               (!slen || (m - l > slen && !strcmp(m - slen, suffix)))) // suffix is either not required or present
+            {
+                l += plen; // skip prefix
+                m[-slen] = '\0'; // cut suffix
+                revp = ht.access(l);
+                if(revp) *revp = rev;
+                else ht.access(newstring(l), rev);
+            }
+        }
+        l = p + 1;
+    }
+    while(p);
+}
diff --git a/source/src/tools.h b/source/src/tools.h
index ba8e8bff..d0b21e6b 100644
--- a/source/src/tools.h
+++ b/source/src/tools.h
@@ -988,6 +988,7 @@ extern int cmpipmatch(const struct iprange *a, const struct iprange *b);
 extern int cvecprintf(vector<char> &v, const char *s, ...) PRINTFARGS(2, 3);
 extern const char *hiddenpwd(const char *pwd, int showchars = 0);
 extern int getlistindex(const char *key, const char *list[], bool acceptnumeric = true, int deflt = -1);
+extern void parseupdatelist(hashtable<const char *, int> &ht, char *buf, const char *prefix = NULL, const char *suffix = NULL);
 
 #if defined(WIN32) && !defined(_DEBUG) && !defined(__GNUC__)
 extern void stackdumper(unsigned int type, EXCEPTION_POINTERS *ep);
diff --git a/source/src/zip.cpp b/source/src/zip.cpp
index ee1f6522..5959ac02 100644
--- a/source/src/zip.cpp
+++ b/source/src/zip.cpp
@@ -191,169 +191,6 @@ static bool readlocalfileheader(stream *f, ziplocalfileheader &h, uint offset)
     return true;
 }
 
-#if 0
-ziparchive *findzip(const char *name)
-{
-    loopv(archives) if(!strcmp(name, archives[i]->name)) return archives[i];
-    return NULL;
-}
-
-static bool checkprefix(vector<zipfile> &files, const char *prefix, int prefixlen)
-{
-    loopv(files)
-    {
-        if(!strncmp(files[i].name, prefix, prefixlen)) return false;
-    }
-    return true;
-}
-
-bool extractzipfile(ziparchive *a, zipfile *f, const char *name);
-
-bool fitspackage(char *filename, int type)
-{
-    if(strchr(filename, ' ')) return false; // refuse all filenames with spaces
-    char *extension = strrchr(filename, '.');
-    if(!extension) return false;
-    ++extension;
-    switch(type)
-    {
-        case PCK_MAPMODEL:  return !strcmp(extension, "md2") || !strcmp(extension, "md3")
-                               || !strcmp(extension, "cfg") || !strcmp(extension, "txt")
-                               || !strcmp(extension, "jpg") || !strcmp(extension, "png");
-        case PCK_MAP:       return !strcmp(extension, "cgz") || !strcmp(extension, "cfg");
-        case PCK_SKYBOX:    return !strcmp(extension, "jpg") || !strcmp(extension, "png");
-        default:            return true;
-    }
-}
-
-static void mountzip(ziparchive &arch, vector<zipfile> &files, const char *mountdir, const char *stripdir, bool extract)
-{
-    string packagesdir = "packages/";
-    path(packagesdir);
-    int striplen = stripdir ? (int)strlen(stripdir) : 0;
-    if(arch.type == PCK_MAP) mountdir = "packages/maps/";
-    if(!mountdir && !stripdir) loopv(files)
-    {
-        zipfile &f = files[i];
-        const char *foundpackages = strstr(f.name, packagesdir);
-        if(foundpackages)
-        {
-            if(foundpackages > f.name)
-            {
-                stripdir = f.name;
-                striplen = foundpackages - f.name;
-            }
-            break;
-        }
-        const char *foundcgz = strstr(f.name, ".cgz");
-        if(foundcgz)
-        {
-            const char *cgzdir = foundcgz;
-            while(--cgzdir >= f.name && *cgzdir != PATHDIV);
-            if(cgzdir < f.name || checkprefix(files, f.name, cgzdir + 1 - f.name))
-            {
-                if(cgzdir >= f.name)
-                {
-                    stripdir = f.name;
-                    striplen = cgzdir + 1 - f.name;
-                }
-                if(!mountdir) mountdir = "packages/maps/";
-                break;
-            }
-            mountdir = "packages/maps/";
-        }
-    }
-    string mdir = "", fname;
-    if(mountdir)
-    {
-        copystring(mdir, mountdir);
-        if(fixpackagedir(mdir) <= 1) mdir[0] = '\0';
-    }
-    loopv(files)
-    {
-        zipfile &f = files[i];
-        formatstring(fname)("%s%s", mdir, striplen && !strncmp(f.name, stripdir, striplen) ? &f.name[striplen] : f.name);
-        if(arch.files.access(fname)) continue;
-        char *mname = newstring(fname);
-        zipfile &mf = arch.files[mname];
-        mf = f;
-        mf.name = mname;
-
-        if(extract)
-        {
-            if(fitspackage(fname, arch.type)) extractzipfile(&arch, &f, fname);
-            else conoutf("\f3could not extract \"%s\" : file extension not supported", fname);
-        }
-    }
-}
-
-bool addzip(const char *name, const char *mount, const char *strip, bool extract, int type)
-{
-    string pname;
-    copystring(pname, name);
-    path(pname);
-    /*int plen = (int)strlen(pname);
-    if(plen < 4 || !strchr(&pname[plen-4], '.')) concatstring(pname, ".zip");*/
-
-    /*ziparchive *exists = findzip(pname);
-    if(exists)
-    {
-        conoutf("already added zip %s", pname);
-        return true;
-    }*/
-
-    stream *f = openfile(pname, "rb");
-    if(!f)
-    {
-        conoutf("could not open file %s", pname);
-        return false;
-    }
-    zipdirectoryheader h;
-    vector<zipfile> files;
-    if(!findzipdirectory(f, h) || !readzipdirectory(f, h.entries, h.offset, h.size, files))
-    {
-        conoutf("could not read directory in zip %s", pname);
-        delete f;
-        return false;
-    }
-
-    ziparchive *arch = new ziparchive;
-    arch->name = newstring(pname);
-    arch->data = f;
-    arch->type = type;
-    mountzip(*arch, files, mount, strip, extract);
-    if(!extract) archives.add(arch);
-    else delete arch;
-    return true;
-}
-
-bool removezip(const char *name)
-{
-    string pname;
-    copystring(pname, name);
-    path(pname);
-    /*int plen = (int)strlen(pname);
-    if(plen < 4 || !strchr(&pname[plen-4], '.')) concatstring(pname, ".zip");*/
-    ziparchive *exists = findzip(pname);
-    if(!exists)
-    {
-        conoutf("zip %s is not loaded", pname);
-        return false;
-    }
-    if(exists->openfiles)
-    {
-        conoutf("zip %s has open files", pname);
-        return false;
-    }
-    //conoutf("removed zip %s", exists->name);
-    archives.removeobj(exists);
-    delete exists;
-    return true;
-}
-#endif
-
-
-
 // stream interface
 
 struct zipstream : stream