Files
assaultcube-/source/src/autodownload.cpp
T

542 lines
20 KiB
C++

// autodownload.cpp
// fetching missing media files from http servers (~ akimbo)
#include "cube.h"
#define DEBUGCOND (autodownloaddebug == 1)
struct pckserver
{
string host;
int priority, ping, resolved;
hashtable<const char *, int> updates;
pckserver() { memset(&host, 0, (char *)&updates - (char *)&host); }
};
struct package
{
string name, fullpath, requestname, host;
int type, iszip, strictnames, server;
const char **exts;
package() { memset(&name, 0, sizeof(struct package)); }
};
vector<pckserver *> pckservers;
vector<package *> pendingpackages;
VARP(autodownload, 0, 1, 1);
VAR(autodownloaddebug, 0, 0, 1);
sl_semaphore sem_pckservers(1, NULL); // control access to the pckservers vector
void resetpckservers()
{
delfile(findfile("config" PATHDIVS "pcksources.cfg", "w"));
conoutf("\f3Restart AssaultCube to take effect after resetting the list of packages source servers");
}
COMMAND(resetpckservers, "");
void addpckserver(char *host, char *priority) // add/adjust a package server
{
sem_pckservers.wait();
if(*host)
{
pckserver *s = NULL;
loopv(pckservers) if(!strcmp(host, pckservers[i]->host)) s = pckservers[i];
DEBUG((s ? "updated" : "new") << " host " << host << ", priority " << priority);
if(!s) s = pckservers.add(new pckserver);
copystring(s->host, host);
if(*priority) s->priority = atoi(priority);
}
sem_pckservers.post();
}
COMMAND(addpckserver, "ss");
void getpckserver() // return a table of all package servers with four columns
{
vector<char> res;
if(sem_pckservers.timedwait(1000)) cvecprintf(res, "\"serverlist is busy, try again later\" "); // single-word return value signals an error
else
{
loopv(pckservers) cvecprintf(res, "\"%s\" %d %d %d\n", pckservers[i]->host, pckservers[i]->priority, pckservers[i]->ping, pckservers[i]->resolved);
sem_pckservers.post();
}
resultcharvector(res, -1);
}
COMMAND(getpckserver, "");
SDL_mutex *pckpinglog_lock; // auxilliary locking for pckping_log
vector<char> pckping_log; // access controlled by sem_pckservers (and pckpinglog_lock)
const char *updatestxt = "/updates.txt";
int pingpckserver(void *data) // fetch updates.txt from a media server, measure required time
{
httpget h;
urlparse u;
pckserver *s = (pckserver *)data;
u.set(s->host);
s->resolved = h.set_host(u.domain) ? 1 : 0;
SDL_mutexP(pckpinglog_lock);
string b;
cvecprintf(pckping_log, "lresolving hostname %s %s (%s)\n", u.domain, s->resolved ? "succeeded" : "failed", iptoa(ENET_NET_TO_HOST_32(h.ip.host), b));
SDL_mutexV(pckpinglog_lock);
if(s->resolved)
{
if(*u.port) h.set_port(atoi(u.port));
defformatstring(url)("%s%s", u.path, updatestxt);
int res = h.get(url, 5000, 10000, 0, true); // http HEAD request only - to get a nice ping value
if(res >= 0 && h.response > 0)
{
s->ping = h.elapsedtime;
SDL_mutexP(pckpinglog_lock);
cvecprintf(pckping_log, "lhttp head %s:%d%s response time: %d, status: %d, contentlength: %d\n", u.domain, h.ip.port, url, s->ping, h.response, h.contentlength);
#ifdef _DEBUG
if(DEBUGCOND) { cvecprintf(pckping_log, "l%s", escapestring(h.header, true, true)); pckping_log.add('\n'); }
#endif
SDL_mutexV(pckpinglog_lock);
if(h.response == 200)
{
h.outvec = new vector<uchar>;
res = h.get(url, 5000, 20000); // http GET
if(res >= 0 && h.response == 200)
{ // parse updates.txt
h.outvec->add('\0');
s->updates.clear();
parseupdatelist(s->updates, (char *)h.outvec->getbuf()); // don't filter entries
SDL_mutexP(pckpinglog_lock);
cvecprintf(pckping_log, "lgot %s:%d%s: %d bytes, %d valid update lines\n", u.domain, h.ip.port, url, res, s->updates.numelems);
SDL_mutexV(pckpinglog_lock);
}
else
{
SDL_mutexP(pckpinglog_lock);
cvecprintf(pckping_log, "lretrieving %s:%d%s failed: response %d, err: %s\n", u.domain, h.ip.port, url, h.response, h.err ? h.err : "(null)");
#ifdef _DEBUG
if(DEBUGCOND)
{
if(!h.header) { h.rawrec.setsize(min(h.rawrec.length(), MAXSTRLEN)); h.rawrec.add('\0'); }
cvecprintf(pckping_log, "lheader: %s", escapestring(h.header ? h.header : h.rawrec.getbuf(), true, true)); pckping_log.add('\n');
}
#endif
SDL_mutexV(pckpinglog_lock);
}
DELETEP(h.outvec);
}
}
else
{ // connection to server failed
SDL_mutexP(pckpinglog_lock);
cvecprintf(pckping_log, "lpinging %s failed, err: %s\n", s->host, h.err ? h.err : "(null)");
SDL_mutexV(pckpinglog_lock);
}
}
h.disconnect();
return 0;
}
int pingallpckservers(void *data)
{
sem_pckservers.wait();
if(autodownload)
{
pckpinglog_lock = SDL_CreateMutex();
vector<SDL_Thread *> pckthreads;
int nop, good = 0, disabled = 0;
loopv(pckservers) if(pckservers[i]->priority > -1000) pckthreads.add(SDL_CreateThread(pingpckserver, "PingPackageServer", pckservers[i])); // start pinging all servers at once
loopv(pckthreads) if(pckthreads[i]) SDL_WaitThread(pckthreads[i], &nop); // wait for all ping threads to finish
SDL_DestroyMutex(pckpinglog_lock);
loopv(pckservers)
{
pckserver *s = pckservers[i];
if(s->priority <= -1000) disabled++;
else if(s->resolved && s->ping) good++;
}
int bad = pckservers.length() - good - disabled;
cvecprintf(pckping_log, "csuccessfully pinged %d media server%s, %d failure%s, %d disabled\n", good, good == 1 ? "" : "s", bad, bad == 1 ? "" : "s", disabled);
}
else cvecprintf(pckping_log, "c\f4(automatic media file download deactivated)\n");
pckping_log.add('e'); // signal end of thread
sem_pckservers.post();
return 0;
}
SDL_Thread *pingallpckthread = NULL;
void setupautodownload()
{
// fetch updates from all configured servers
// in a background thread
// during startup
pingallpckthread = SDL_CreateThread(pingallpckservers, "PingAllPackageServers", NULL);
}
void pollautodownloadresponse() // thread-safe feedback from the autodownload-updater thread
{
static int donepolling = 0;
if(donepolling) return;
if(!sem_pckservers.trywait())
{
if(pckping_log.length())
{
pckping_log.add('\0');
char *p, *l = pckping_log.getbuf();
do
{ // break into single lines
if((p = strchr(l, '\n'))) *p = '\0';
if(*l == 'c') conoutf("%s", l + 1); // line starts with "c" -> console
else if(*l == 'e') donepolling = 1; // line starts with "e" -> end polling
else clientlogf("%s", l + 1); // line starts with "l" -> log
l = p + 1;
}
while(p);
pckping_log.setsize(0);
}
sem_pckservers.post();
}
if(donepolling)
{
int nop;
SDL_WaitThread(pingallpckthread, &nop); // detaching the thread would be preferrable, but SDL 1.2 doesn't provide that
pingallpckthread = NULL;
}
}
const char *mmexts[] = { ".md2", ".md3", ".cfg", ".txt", ".jpg", ".png", "" },
*mapexts[] = { ".cgz", ".cfg", "" },
*sbexts[] = { "_lf.jpg", "_rt.jpg", "_ft.jpg", "_bk.jpg", "_dn.jpg", "_up.jpg", "_license.txt", "" };
bool requirepackage(int type, const char *name, const char *host)
{
const char *prefix = NULL, *checkprefix = NULL, *forceext = NULL;
package *pck = new package;
switch(type)
{
case PCK_AUDIO:
prefix = "packages/audio/";
checkprefix = "ambience/";
forceext = ".ogg";
break;
case PCK_MAPMODEL:
prefix = "packages/models/";
checkprefix = "mapmodels/";
pck->iszip = 1;
pck->exts = mmexts;
break;
case PCK_TEXTURE:
checkprefix = "packages/";
forceext = ".jpg";
break;
case PCK_SKYBOX:
prefix = "packages/textures/skymaps/";
pck->iszip = pck->strictnames = 1;
pck->exts = sbexts;
break;
case PCK_MAP:
prefix = "packages/maps/";
pck->iszip = pck->strictnames = 1;
pck->exts = mapexts;
break;
case PCK_MOD:
prefix = "mods/";
forceext = ".zip";
break;
default:
conoutf("\f3requirepackage: illegal package type %d (\"%s\")", type, name); // should probably be fatal()
delete pck;
return false;
}
string upath;
copystring(upath, name);
unixpath(upath);
filtertext(pck->name, name, FTXT__MEDIAFILEPATH); // also changes backslashes to slashes
if(strlen(upath) > 100 || strcmp(upath, pck->name) || (checkprefix && strncmp(pck->name, checkprefix, strlen(checkprefix))) || (type == PCK_MOD && !validzipmodname(upath)))
{
conoutf("\f3requirepackage: illegal media path \"%s\" (type: %d)", name, type);
delete pck;
return false;
}
if(forceext && strlen(pck->name) > strlen(forceext) && !strcmp(pck->name + strlen(pck->name) - strlen(forceext), forceext)) forceext = NULL; // filename already has required extension
formatstring(pck->fullpath)("%s%s%s", prefix ? prefix : "", pck->name, forceext ? forceext : "");
formatstring(pck->requestname)("/%s%s%s", pck->fullpath, type == PCK_MAP ? ".cgz" : "", pck->iszip ? ".zip" : ""); // special case map zips: file ending .cgz.zip
loopv(pendingpackages) if(!strcmp(pendingpackages[i]->requestname, pck->requestname))
{ // request already queued
delete pck;
return false;
}
pck->type = type;
if(host) copystring(pck->host, host); // assign fixed server
DEBUG("name: " << pck->name << ", fullpath: " << pck->fullpath << ", requestname: " << pck->requestname << ", host: " << pck->host << ", type: " << pck->type << ", iszip: " << pck->iszip
<< ", strictnames: " << pck->strictnames << ", server: " << pck->server << ", exts: " << (pck->exts ? *pck->exts : ""));
pendingpackages.add(pck);
return true;
}
VAR(rereadtexturelists, 0, 1, 1); // flag to indicate additional texture files to trigger a menu rebuild (defaults to "1" to build the menus after startup)
VAR(rereadsoundlists, 0, 1, 1); // same for map sounds
void processdownload(package *pck, stream *f) // write downloaded data to file(s)
{
f->seek(0);
if(pck->iszip)
{ // unpack zip file
vector<const char *> files;
void *mz = zipmanualopen(f, files);
if(mz)
{
string filename, cleanzname;
loopv(files)
{
if(i >= MAXFILESINADZIP) break;
const char *zname = behindpath(files[i]); // ignore _all_ paths inside the zip
filtertext(cleanzname, zname, FTXT__MEDIAFILENAME);
bool isok = false;
for(int n = 0; *pck->exts[n]; n++)
{
if(pck->strictnames)
{ // only base names and listed extensions allowed
formatstring(filename)("%s%s", pck->fullpath, pck->exts[n]);
isok = !strcmp(behindpath(filename), zname);
}
else
{ // any name with listed extension, written to subdirectory
int zlen = (int)strlen(zname), elen = (int)strlen(pck->exts[n]);
if((isok = (zlen > elen && !strcmp(zname + zlen - elen, pck->exts[n])))) formatstring(filename)("%s/%s", pck->fullpath, zname);
}
if(isok) break;
}
if(isok && !strcmp(cleanzname, zname))
{
stream *zf = openfile(path(filename), "wb");
if(zf)
{
DEBUGCODE(int got =) zipmanualread(mz, i, zf, MAXMEDIADOWNLOADFILESIZE); // extract file from zip and write to @filename
DEBUGCODE(clientlogf("extracted %d bytes from %s %s, written to %s", got, pck->name, files[i], filename));
delete zf;
}
}
else conoutf("\f3illegal filename \"%s\" in %s", files[i], pck->requestname);
}
zipmanualclose(mz);
}
else clientlogf("failed to open zip file %s", pck->requestname);
}
else
{ // write single file
stream *d = openfile(path(pck->fullpath, true), "wb");
if(d)
{
streamcopy(d, f, pck->type == PCK_MOD ? MAXMODDOWNLOADSIZE : MAXMEDIADOWNLOADFILESIZE);
delete d;
}
delete f;
}
switch(pck->type) // some housekeeping
{
case PCK_SKYBOX:
case PCK_TEXTURE:
rereadtexturelists = 1;
break;
case PCK_AUDIO:
rereadsoundlists = 1;
break;
case PCK_MAP:
case PCK_MAPMODEL:
break;
}
}
bool canceldownloads = false;
int progress_n, progress_of;
int progress_callback_dlpackage(void *data, float progress)
{
if(progress_of < 0)
{
defformatstring(txt)("downloading %s... (esc to abort)", (const char *)data);
if(progress < 0) show_out_of_renderloop_progress(progress + 1.0f, "waiting for response (esc to abort)");
else show_out_of_renderloop_progress(min(progress, 1.0f), txt);
}
else loadingscreen("downloading package %d of %d...\n%s \fs%s%d%%\fr\n(ESC to cancel)", progress_n, progress_of, (const char *)data, progress < 0 ? "\f3waiting " : "", int(100.0 * fabs(progress)));
if(interceptkey(SDLK_ESCAPE))
{
canceldownloads = true;
return 1;
}
return 0;
}
bool dlpackage(httpget &h, package *pck, pckserver *s) // download one package from one server
{
urlparse u;
u.set(s ? s->host : pck->host);
h.callbackfunc = progress_callback_dlpackage;
h.callbackdata = pck->name;
progress_n = progress_of - pendingpackages.length() + 1;
if(*u.domain && h.set_host(u.domain))
{
if(*u.port) h.set_port(atoi(u.port));
h.outstream = openvecfile(NULL);
defformatstring(url)("%s%s", u.path, pck->requestname);
int got = h.get(url, 6000, 90000);
if(got < 0 || !h.response)
{
if(!canceldownloads && s) s->ping = 0; // received a hard error from the server connection, better not try this one again
clientlogf("download %s:%d%s failed, err: %s", u.domain, h.ip.port, url, h.err ? h.err : "(null)");
h.disconnect();
}
else
{
if(h.response == 200)
{
DEBUGCODE(clientlogf("downloaded %s:%d%s, %d bytes (%d raw), %d msec", u.domain, h.ip.port, url, got, h.contentlength, h.elapsedtime));
processdownload(pck, h.outstream);
h.outstream = NULL; // deleted by processdownload()
return true;
}
else clientlogf("download %s:%d%s failed, server response %d%s%s", u.domain, h.ip.port, url, h.response, h.err ? ", err: " : "", h.err ? h.err : "");
}
DELETEP(h.outstream);
}
else
{
if(s) s->resolved = 0;
clientlogf("resolving host \"%s\" failed", u.domain);
h.disconnect();
}
return false;
}
int packagesort_server(package **a, package **b) { return (*b)->server - (*a)->server; }
int packagesort_name(package **a, package **b) { return strcmp((*b)->name, (*a)->name); }
int packagesort_host(package **a, package **b) { return strcmp((*b)->host, (*a)->host); }
int pckserversort(pckserver **a, pckserver **b)
{
if((*a)->resolved != (*b)->resolved) return (*b)->resolved - (*a)->resolved; // group resolvable hosts at the top
if((*a)->priority != (*b)->priority) return (*b)->priority - (*a)->priority; // manual priority wins over ping (high priority to the top)
if((*a)->ping && (*b)->ping) return (*a)->ping - (*b)->ping; // smaller ping wins
return (*b)->ping - (*a)->ping; // or whichever ping is not zero
}
int downloadpackages(bool loadscr) // get all pending packages
{
bool failed = false;
httpget h;
canceldownloads = false;
progress_of = loadscr ? pendingpackages.length() : -1;
if(sem_pckservers.timedwait(1000))
{ // can't lock the server list -> this means, the server ping threads have not finished yet
if(pendingpackages.length()) conoutf("\f3failed to download packages: still pinging the servers...");
return 0;
}
pckservers.sort(pckserversort);
loopv(pendingpackages)
{
package *pck = pendingpackages[i];
int maxrev = 0, *rev = NULL;
pck->server = -1; // server with highest revision for that content
loopvj(pckservers)
{
pckserver *s = pckservers[j];
if(s->priority > -1000 && s->resolved && s->ping && (rev = s->updates.access(pck->requestname)) && *rev > maxrev)
{
maxrev = *rev;
pck->server = j;
}
}
}
// get all packages with auto-assigned servers
pendingpackages.sort(packagesort_server);
loopvrev(pendingpackages)
{
if(canceldownloads) break;
package *pck = pendingpackages[i];
if(!*pck->host && pckservers.inrange(pck->server) && pckservers[pck->server]->resolved)
{
if(dlpackage(h, pck, pckservers[pck->server])) delete pendingpackages.remove(i);
else pck->server = -1; // auto-assigned server didn't work - try any other now
}
}
// try all packages from all servers - high priority servers first
pendingpackages.sort(packagesort_name);
loopvj(pckservers)
{
pckserver *s = pckservers[j];
if(s->priority <= -1000 || !s->resolved || !s->ping) continue;
loopv(pendingpackages)
{
if(canceldownloads) break;
package *pck = pendingpackages[i];
if(!*pck->host)
{
if(dlpackage(h, pck, s)) delete pendingpackages.remove(i--);
}
if(!s->resolved || !s->ping) break;
}
}
if(canceldownloads) pendingpackages.deletecontents();
// print error messages for all failed packages - and remove them
loopv(pendingpackages)
{
package *pck = pendingpackages[i];
if(!*pck->host)
{
conoutf("\f3failed to load %s from any host, giving up", pck->name);
delete pendingpackages.remove(i--);
failed = true;
}
}
// get all packages with manually assigned hosts - those will be deleted from the list regardless of success!
pendingpackages.sort(packagesort_host);
loopvrev(pendingpackages)
{
package *pck = pendingpackages[i];
if(!canceldownloads && !dlpackage(h, pck, NULL))
{
conoutf("\f3failed to load %s from %s, giving up", pck->name, pck->host);
failed = true;
}
delete pendingpackages.remove(i);
}
sem_pckservers.post();
h.disconnect();
return failed ? 0 : h.traffic;
}
void writepcksourcecfg()
{
if(pckservers.length())
{
stream *f = openfile("config" PATHDIVS "pcksources.cfg", "w");
if(!f) return;
f->printf("// list of package source servers (only add servers you trust!)\n\n");
f->printf("autodownloaddebug %d\n", autodownloaddebug); // saved.cfg is loaded too late for this to be in it and be efficient...
loopv(pckservers)
{
pckserver *s = pckservers[i];
if(s->priority > -10000) f->printf("addpckserver %s %d // ping: %d, resolved: %d, updates: %d\n", s->host, s->priority, s->ping, s->resolved, s->updates.numelems);
}
delete f;
}
}
void getmod(char *name)
{
if(requirepackage(PCK_MOD, name) && downloadpackages(false)) conoutf("mod package %s successfully downloaded", name);
else conoutf("failed to download mod package %s", name);
}
COMMAND(getmod, "s");