mirror of
https://github.com/burrow-dev/burrow-server.git
synced 2026-11-18 17:27:50 -05:00
perf: save only changed accounts, cache missing assets, lighter logging and backup
Ping on prod jumped to seconds because node's main thread was busy most of every second. - accounts.save(account, ...) names the changed accounts, which are written within a second; the rest are looked at a few at a time (10 ms a save for those in play, 5 ms for the others). One save under prod-like load: 289 ms before, 16 ms after. - an asset the CDN does not have gets a cacheable 404 and is logged once - the proxy's own connections no longer count against the web cap, which had node reset requests nginx had already let in - a repeated client error is written once in ten minutes; the panel's log buffer trims in batches - the daily backup uses sqlite's online backup off the main thread
This commit is contained in:
1 parent
edca7bf790
commit
aa455f52d0
26 files changed
+334
-220
No files matched your search
+50
-12
@@ -151,6 +151,7 @@ const send = (res, status, body, type = "application/json") => {
|
||||
const TEN_MINUTES_MS = 600000;
|
||||
const failuresByAddress = limits.limiter(config.failedLoginsPerTenMinutes, TEN_MINUTES_MS);
|
||||
const searches = limits.limiter(120, 60000); // item finder searches, per address
|
||||
const MISSING_ASSET_SECONDS = 600; // how long a client may keep the CDN's "no such file"
|
||||
const cacheMisses = limits.limiter(config.cacheMissesPerTenMinutes, TEN_MINUTES_MS); // files fetched from the official CDN for one address
|
||||
const errorReports = limits.limiter(config.errorReportsPerMinute, 60000);
|
||||
const paintingPosts = limits.limiter(60, TEN_MINUTES_MS); // per address; each also needs a token from the game
|
||||
@@ -169,6 +170,27 @@ const keepClientError = text => {
|
||||
.then(size => (size < MAX_CLIENT_ERROR_FILE ? fs.promises.appendFile(CLIENT_ERROR_FILE, text) : null))
|
||||
.catch(() => {});
|
||||
};
|
||||
// the same error from many clients (one broken item, met by everyone who
|
||||
// sees it) is written once in REPEAT_WINDOW_MS, and the next time after that
|
||||
// with how many there were between: a log, and a thread that waits for it,
|
||||
// had a stack of twenty lines several times a second. null for one to leave
|
||||
// out; else what to add to its line.
|
||||
const REPEAT_WINDOW_MS = 600000;
|
||||
const MAX_REMEMBERED_ERRORS = 1000;
|
||||
const recentErrors = new Map(); // section and message -> {at, more}
|
||||
const repeatedError = text => {
|
||||
const seen = recentErrors.get(text);
|
||||
if (seen && Date.now() - seen.at < REPEAT_WINDOW_MS) {
|
||||
seen.more++;
|
||||
return null;
|
||||
}
|
||||
recentErrors.delete(text);
|
||||
if (recentErrors.size >= MAX_REMEMBERED_ERRORS) {
|
||||
recentErrors.delete(recentErrors.keys().next().value);
|
||||
}
|
||||
recentErrors.set(text, {at: Date.now(), more: 0});
|
||||
return seen && seen.more > 0 ? `\n\t(and ${seen.more} more like it in the ${Math.round(REPEAT_WINDOW_MS / 60000)} minutes before)` : "";
|
||||
};
|
||||
const THEMED_ROOM = /^(\d+)\/roomDefs\/theme_(\d+)\/(.+)$/;
|
||||
|
||||
// the web pages, as against what the game and the client load
|
||||
@@ -348,7 +370,7 @@ const renameAndSignIn = (res, body, from) => {
|
||||
delete account.nameRejected;
|
||||
manage.forget(account);
|
||||
accounts.recordLogin(account, "launcher", from.address);
|
||||
accounts.save();
|
||||
accounts.save(account);
|
||||
console.log(`[accounts] a rejected username was replaced by "${account.username}"`);
|
||||
accounts.setLoginChoices(account, body.login_choices);
|
||||
const token = accounts.issueToken(account);
|
||||
@@ -384,7 +406,7 @@ const authenticate = async (req, res, from) => {
|
||||
const tried = accounts.byName(accountKey);
|
||||
if (tried) {
|
||||
accounts.recordLogin(tried, "launcher", from.address, false);
|
||||
accounts.save();
|
||||
accounts.save(tried);
|
||||
}
|
||||
}
|
||||
// error_code 100 = expired refresh token, 101 = wrong credentials
|
||||
@@ -429,7 +451,7 @@ const authenticate = async (req, res, from) => {
|
||||
if (!totp.verify(account, String(body.totp))) {
|
||||
failuresByAddress.record(from.key);
|
||||
accounts.recordLogin(account, "launcher", from.address, false);
|
||||
accounts.save();
|
||||
accounts.save(account);
|
||||
send(res, 401, {error_code: 101, error: "totp_wrong", captcha_pass: captcha.issuePass(account, from)});
|
||||
return;
|
||||
}
|
||||
@@ -440,13 +462,13 @@ const authenticate = async (req, res, from) => {
|
||||
// its username was rejected: a new one first (above). the official
|
||||
// launcher shows this as a wrong password; the account page asks too.
|
||||
if (account.nameRejected) {
|
||||
accounts.save(); // a two-step code just used stays used
|
||||
accounts.save(account); // a two-step code just used stays used
|
||||
send(res, 401, {error_code: 101, error: "name_rejected", rename_ticket: renameTicket(account, deviceToken)});
|
||||
return;
|
||||
}
|
||||
if (!byRefresh) {
|
||||
accounts.recordLogin(account, "launcher", from.address);
|
||||
accounts.save();
|
||||
accounts.save(account);
|
||||
}
|
||||
// the custom client's advanced settings; the official launcher sends none.
|
||||
accounts.setLoginChoices(account, body.login_choices);
|
||||
@@ -687,9 +709,12 @@ const handle = async (req, res) => {
|
||||
const section = String(report.section).slice(0, 20).replace(/[^\w-]/g, "?");
|
||||
const trace = message.startsWith("media ");
|
||||
if (trace ? config.logPackets : from.local || errorReports.allow(from.key)) {
|
||||
console.log(`[${trace ? "client-trace" : "client-error"}] (${section}) ${message}`);
|
||||
if (!trace) {
|
||||
keepClientError(`${new Date().toISOString()} (${section}) ${message}\n`);
|
||||
const again = trace ? "" : repeatedError(`${section} ${message}`);
|
||||
if (again !== null) {
|
||||
console.log(`[${trace ? "client-trace" : "client-error"}] (${section}) ${message}${again}`);
|
||||
if (!trace) {
|
||||
keepClientError(`${new Date().toISOString()} (${section}) ${message}${again}\n`);
|
||||
}
|
||||
}
|
||||
}
|
||||
send(res, 200, {});
|
||||
@@ -790,7 +815,15 @@ const handle = async (req, res) => {
|
||||
const body = (themed && Object.values(config.themes).some(theme => theme.build === Number(themed[2])) && await cdn.get(`${themed[2]}/roomDefs/${themed[3]}`, mayFetch))
|
||||
|| await cdn.get(themed ? `${themed[1]}/roomDefs/${themed[3]}` : relPath, mayFetch);
|
||||
if (!body) {
|
||||
console.log(`[http] 404 ${url.pathname}`);
|
||||
// one the CDN does not have is asked for again and again (the same few
|
||||
// hundred, by every client that meets them): its client may keep the
|
||||
// answer a while, and it was logged when the CDN said so ([cdn] 404).
|
||||
if (cdn.knownMissing(themed ? `${themed[1]}/roomDefs/${themed[3]}` : relPath)) {
|
||||
res.setHeader("Cache-Control", `public, max-age=${MISSING_ASSET_SECONDS}`);
|
||||
}
|
||||
else {
|
||||
console.log(`[http] 404 ${url.pathname}`);
|
||||
}
|
||||
send(res, 404, "Not found", "text/plain");
|
||||
return;
|
||||
}
|
||||
@@ -815,12 +848,15 @@ const start = () => new Promise((resolve, reject) => {
|
||||
: http.createServer(onRequest);
|
||||
// nobody gets to hold a request open by trickling it in, nor a reply by
|
||||
// never reading it; and there is an end to how many connections there
|
||||
// may be, in all and from one address.
|
||||
// may be, in all and from one address. a proxy in front (nginx) keeps
|
||||
// its own count of those; its connections here are not counted, since
|
||||
// turning them away only had it fail the requests of players it had
|
||||
// already let in, who then asked again.
|
||||
server.requestTimeout = 30000;
|
||||
server.headersTimeout = 15000;
|
||||
server.timeout = config.httpSocketTimeoutMs;
|
||||
server.maxConnections = config.maxHttpConnections;
|
||||
const open = new Map(); // address, as limits count it -> connections
|
||||
let openInAll = 0;
|
||||
server.on("connection", socket => {
|
||||
const address = limits.key(socket.remoteAddress);
|
||||
// behind a proxy every connection is the proxy's own.
|
||||
@@ -828,12 +864,14 @@ const start = () => new Promise((resolve, reject) => {
|
||||
return;
|
||||
}
|
||||
const count = (open.get(address) || 0) + 1;
|
||||
if (count > config.maxHttpConnectionsPerAddress) {
|
||||
if (count > config.maxHttpConnectionsPerAddress || openInAll >= config.maxHttpConnections) {
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
open.set(address, count);
|
||||
openInAll++;
|
||||
socket.once("close", () => {
|
||||
openInAll--;
|
||||
const left = open.get(address) - 1;
|
||||
if (left > 0) {
|
||||
open.set(address, left);
|
||||
|
||||
Reference in new issue
Block a user