mirror of
https://github.com/burrow-dev/burrow-server.git
synced 2026-11-19 01:27:51 -05:00
perf: save only changed accounts, cache missing assets, lighter logging and backup
Ping on prod jumped to seconds because node's main thread was busy most of every second. - accounts.save(account, ...) names the changed accounts, which are written within a second; the rest are looked at a few at a time (10 ms a save for those in play, 5 ms for the others). One save under prod-like load: 289 ms before, 16 ms after. - an asset the CDN does not have gets a cacheable 404 and is logged once - the proxy's own connections no longer count against the web cap, which had node reset requests nginx had already let in - a repeated client error is written once in ten minutes; the panel's log buffer trims in batches - the daily backup uses sqlite's online backup off the main thread
This commit is contained in:
1 parent
edca7bf790
commit
aa455f52d0
26 files changed
+334
-220
No files matched your search
+148
-79
@@ -3,7 +3,7 @@
|
||||
const crypto = require("crypto");
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const {DatabaseSync} = require("node:sqlite");
|
||||
const {DatabaseSync, backup: copyDatabase} = require("node:sqlite");
|
||||
const config = require("../config.js");
|
||||
const defpacks = require("./defpacks.js");
|
||||
const names = require("./names.js");
|
||||
@@ -13,9 +13,9 @@ const names = require("./names.js");
|
||||
// JSON document per account in `state`.
|
||||
//
|
||||
// every account is held in memory while the server runs. game code changes
|
||||
// those objects directly and then calls save(), which has every account
|
||||
// whose state changed written back, a moment later: however many times it
|
||||
// is called in that moment, the accounts are gone through once (flush).
|
||||
// those objects directly and then calls save(account, ...), which has the
|
||||
// accounts named written back a moment later, and the others looked at a
|
||||
// few at a time (flush, below).
|
||||
|
||||
const SCHEMA = `
|
||||
CREATE TABLE IF NOT EXISTS accounts (
|
||||
@@ -415,21 +415,30 @@ const load = () => {
|
||||
console.log(`[accounts] ${byKey.size} accounts in ${path.join(config.dataDir, "game.sqlite")}`);
|
||||
};
|
||||
|
||||
// Which accounts the save after a message looks at. turning an account into
|
||||
// JSON to see whether it changed costs, and doing it for every account there
|
||||
// is, a second after nearly every message, held the whole server up for
|
||||
// longer the more accounts there were: a second and more, for everyone. so
|
||||
// it looks at the accounts in play: online ones (hold, from lib/world.js),
|
||||
// and ones looked up lately (by name, id, token, password: how the admin
|
||||
// panel, buddies and the like get at an offline account), until a save
|
||||
// finds them written and WARM_MS has gone by. the rest are looked at a few
|
||||
// at a time behind those (SWEEP_BUDGET_MS a save), round and round, for
|
||||
// what reached an account some other way (accounts.all()). a stop of the
|
||||
// server, and every flush() called by name, looks at all of them.
|
||||
// Which accounts a save looks at. turning an account into JSON to see
|
||||
// whether it changed costs (up to a millisecond for a big one), and doing it
|
||||
// a second after nearly every message for every account in play (online
|
||||
// ones, and the buddies, dens and cards they had looked up lately) held the
|
||||
// whole server up for most of every second once there were enough players.
|
||||
// so game code says which accounts it changed, save(account, ...), and those
|
||||
// are written within SAVE_DELAY_MS. behind them, for whatever changed an
|
||||
// account without saying so, the accounts in play are looked at a few at a
|
||||
// time (PLAY_BUDGET_MS a save), round and round: online ones (hold, from
|
||||
// lib/world.js) and ones looked up lately (by name, id, token, password: how
|
||||
// the admin panel, buddies and the like get at an offline account), for
|
||||
// WARM_MS. behind those every other account is looked at the same way
|
||||
// (SWEEP_BUDGET_MS), for what reached an account some other way
|
||||
// (accounts.all()). a stop of the server, and every flush() called by name,
|
||||
// looks at all of them.
|
||||
const WARM_MS = 10 * 60 * 1000;
|
||||
const PLAY_BUDGET_MS = 10;
|
||||
const SWEEP_BUDGET_MS = 5;
|
||||
const held = new Set(); // accounts online
|
||||
const warm = new Map(); // account -> when last looked up
|
||||
const dirty = new Set(); // accounts said to have changed since the last save
|
||||
let playOrder = [];
|
||||
let playAt = 0;
|
||||
let playLeft = 0; // accounts in play still to be looked at since the last save()
|
||||
let sweepOrder = [];
|
||||
let sweepAt = 0;
|
||||
let sweepLeft = 0; // accounts the sweep still has to look at since the last save()
|
||||
@@ -457,9 +466,60 @@ const changedState = (account, changed) => {
|
||||
return false;
|
||||
};
|
||||
|
||||
const elapsedMs = started => Number(process.hrtime.bigint() - started) / 1e6;
|
||||
|
||||
// the next accounts in play, while there is time: the ones online and the
|
||||
// ones looked up lately. one looked up longer ago than WARM_MS is looked at
|
||||
// once more, then left to the sweep.
|
||||
const lookAtPlay = (changed, looked) => {
|
||||
const started = process.hrtime.bigint();
|
||||
while (playLeft > 0 && elapsedMs(started) < PLAY_BUDGET_MS) {
|
||||
if (playAt >= playOrder.length) {
|
||||
const now = Date.now();
|
||||
playOrder = [...new Set([...held, ...warm.keys()])];
|
||||
playAt = 0;
|
||||
for (const [account, at] of warm) {
|
||||
if (now - at > WARM_MS && !held.has(account)) {
|
||||
warm.delete(account);
|
||||
}
|
||||
}
|
||||
if (playOrder.length === 0) {
|
||||
playLeft = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
const account = playOrder[playAt++];
|
||||
playLeft--;
|
||||
if (!looked.has(account) && byId.get(account.id) === account) {
|
||||
looked.add(account);
|
||||
changedState(account, changed);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const lookAtRest = (changed, looked) => {
|
||||
const started = process.hrtime.bigint();
|
||||
while (sweepLeft > 0 && elapsedMs(started) < SWEEP_BUDGET_MS) {
|
||||
if (sweepAt >= sweepOrder.length) {
|
||||
sweepOrder = [...byId.values()];
|
||||
sweepAt = 0;
|
||||
if (sweepOrder.length === 0) {
|
||||
sweepLeft = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
const account = sweepOrder[sweepAt++];
|
||||
sweepLeft--;
|
||||
if (!looked.has(account) && byId.get(account.id) === account) {
|
||||
looked.add(account);
|
||||
changedState(account, changed);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// writes every account whose state changed since it was last written, in
|
||||
// one transaction; with {some: true}, of the accounts in play and the
|
||||
// sweep's next few (above).
|
||||
// one transaction; with {some: true}, of the accounts said to have changed
|
||||
// and the next few of the others (above).
|
||||
const flush = ({some = false} = {}) => {
|
||||
clearTimeout(saveTimer);
|
||||
saveTimer = null;
|
||||
@@ -467,49 +527,28 @@ const flush = ({some = false} = {}) => {
|
||||
return;
|
||||
}
|
||||
const changed = [];
|
||||
const named = [...dirty];
|
||||
dirty.clear();
|
||||
if (some) {
|
||||
const now = Date.now();
|
||||
const looked = new Set();
|
||||
for (const account of held) {
|
||||
looked.add(account);
|
||||
changedState(account, changed);
|
||||
}
|
||||
for (const [account, at] of warm) {
|
||||
if (looked.has(account)) {
|
||||
continue;
|
||||
}
|
||||
looked.add(account);
|
||||
if (byId.get(account.id) !== account) {
|
||||
warm.delete(account); // removed
|
||||
}
|
||||
else if (!changedState(account, changed) && now - at > WARM_MS && !held.has(account)) {
|
||||
warm.delete(account);
|
||||
}
|
||||
}
|
||||
const started = process.hrtime.bigint();
|
||||
while (sweepLeft > 0 && Number(process.hrtime.bigint() - started) / 1e6 < SWEEP_BUDGET_MS) {
|
||||
if (sweepAt >= sweepOrder.length) {
|
||||
sweepOrder = [...byId.values()];
|
||||
sweepAt = 0;
|
||||
if (sweepOrder.length === 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
const account = sweepOrder[sweepAt++];
|
||||
sweepLeft--;
|
||||
if (!looked.has(account) && byId.get(account.id) === account) {
|
||||
for (const account of named) {
|
||||
if (byId.get(account.id) === account) {
|
||||
looked.add(account);
|
||||
changedState(account, changed);
|
||||
}
|
||||
}
|
||||
lookAtPlay(changed, looked);
|
||||
lookAtRest(changed, looked);
|
||||
}
|
||||
else {
|
||||
for (const account of byId.values()) {
|
||||
changedState(account, changed);
|
||||
}
|
||||
playLeft = 0;
|
||||
sweepLeft = 0;
|
||||
}
|
||||
if (sweepLeft > 0) {
|
||||
save(); // the sweep goes on in a moment
|
||||
if (playLeft > 0 || sweepLeft > 0) {
|
||||
later(CONTINUE_MS); // the looking goes on in a moment
|
||||
}
|
||||
if (changed.length === 0) {
|
||||
return;
|
||||
@@ -524,6 +563,7 @@ const flush = ({some = false} = {}) => {
|
||||
}
|
||||
catch (err) {
|
||||
db.exec("ROLLBACK");
|
||||
named.forEach(account => dirty.add(account));
|
||||
throw err;
|
||||
}
|
||||
for (const [id, json] of changed) {
|
||||
@@ -531,16 +571,17 @@ const flush = ({some = false} = {}) => {
|
||||
}
|
||||
};
|
||||
|
||||
// "Something changed": the accounts are written within SAVE_DELAY_MS. going
|
||||
// through every account costs more the more there are, and game code calls
|
||||
// this after nearly every message (and some of it a hundred times in a
|
||||
// row), so the going-through is done once for all the calls of a moment.
|
||||
// what a crash can lose is that moment; a stop of the server writes first
|
||||
// "Something changed": the accounts named are written within SAVE_DELAY_MS,
|
||||
// and the looking-through above starts again. game code calls this after
|
||||
// nearly every message (and some of it a hundred times in a row), so it is
|
||||
// done once for all the calls of a moment. what a crash can lose is that
|
||||
// moment, for an account named, and the time a round of the looking takes,
|
||||
// for one changed without saying so; a stop of the server writes first
|
||||
// (server.js).
|
||||
const SAVE_DELAY_MS = 1000;
|
||||
const CONTINUE_MS = 250;
|
||||
let saveTimer = null;
|
||||
const save = () => {
|
||||
sweepLeft = byId.size;
|
||||
const later = delay => {
|
||||
if (saveTimer === null) {
|
||||
saveTimer = setTimeout(() => {
|
||||
try {
|
||||
@@ -550,25 +591,47 @@ const save = () => {
|
||||
console.error(`[accounts] could not save: ${err.message}`);
|
||||
save(); // tried again in a moment: nothing is lost while the server runs
|
||||
}
|
||||
}, SAVE_DELAY_MS);
|
||||
}, delay);
|
||||
}
|
||||
};
|
||||
const save = (...changedAccounts) => {
|
||||
for (const account of changedAccounts) {
|
||||
if (account) {
|
||||
dirty.add(account);
|
||||
}
|
||||
}
|
||||
playLeft = held.size + warm.size;
|
||||
sweepLeft = byId.size;
|
||||
later(SAVE_DELAY_MS);
|
||||
};
|
||||
|
||||
// A copy of the whole database, made once a day into data/backups/ and the
|
||||
// latest BACKUPS_KEPT kept. it is a whole file of its own (VACUUM INTO), to
|
||||
// be copied somewhere else by whoever runs the server: a copy on the same
|
||||
// disk is no help when the disk goes.
|
||||
// latest BACKUPS_KEPT kept. it is a whole file of its own, to be copied
|
||||
// somewhere else by whoever runs the server: a copy on the same disk is no
|
||||
// help when the disk goes. it is copied a batch of pages at a time off the
|
||||
// main thread (sqlite's online backup): made in one go (VACUUM INTO, after
|
||||
// writing every account) it held the game up for seconds.
|
||||
const BACKUPS_KEPT = 14;
|
||||
const backup = () => {
|
||||
let backingUp = false;
|
||||
const backup = async () => {
|
||||
const folder = path.join(config.dataDir, "backups");
|
||||
const file = path.join(folder, `game-${new Date().toISOString().slice(0, 10)}.sqlite`);
|
||||
if (fs.existsSync(file)) {
|
||||
if (backingUp || fs.existsSync(file)) {
|
||||
return;
|
||||
}
|
||||
fs.mkdirSync(folder, {recursive: true, mode: 0o700});
|
||||
flush();
|
||||
db.prepare("VACUUM INTO ?").run(file);
|
||||
fs.chmodSync(file, 0o600);
|
||||
backingUp = true;
|
||||
const part = `${file}.part`;
|
||||
try {
|
||||
fs.mkdirSync(folder, {recursive: true, mode: 0o700});
|
||||
fs.rmSync(part, {force: true});
|
||||
await copyDatabase(db, part);
|
||||
fs.chmodSync(part, 0o600);
|
||||
fs.renameSync(part, file); // only a whole copy has the day's name
|
||||
}
|
||||
finally {
|
||||
backingUp = false;
|
||||
fs.rmSync(part, {force: true});
|
||||
}
|
||||
const old = fs.readdirSync(folder).filter(name => /^game-\d{4}-\d{2}-\d{2}\.sqlite$/.test(name)).sort().slice(0, -BACKUPS_KEPT);
|
||||
for (const name of old) {
|
||||
fs.unlinkSync(path.join(folder, name));
|
||||
@@ -577,12 +640,9 @@ const backup = () => {
|
||||
};
|
||||
const startBackups = () => {
|
||||
const attempt = () => {
|
||||
try {
|
||||
backup();
|
||||
}
|
||||
catch (err) {
|
||||
backup().catch(err => {
|
||||
console.error(`[accounts] backup failed: ${err.message}`);
|
||||
}
|
||||
});
|
||||
};
|
||||
attempt();
|
||||
setInterval(attempt, 3600000).unref();
|
||||
@@ -733,12 +793,16 @@ const rename = (account, username, anyName = false) => {
|
||||
byKey.delete(oldKey);
|
||||
account.username = username;
|
||||
byKey.set(newKey, account);
|
||||
const named = [account];
|
||||
for (const other of byId.values()) {
|
||||
other.buddies = other.buddies.map(name => (name === oldKey ? newKey : name));
|
||||
other.blocked = other.blocked.map(name => (name === oldKey ? newKey : name));
|
||||
other.buddyRequests = other.buddyRequests.map(name => (name === oldKey ? newKey : name));
|
||||
if ([other.buddies, other.blocked, other.buddyRequests].some(list => list.includes(oldKey))) {
|
||||
other.buddies = other.buddies.map(name => (name === oldKey ? newKey : name));
|
||||
other.blocked = other.blocked.map(name => (name === oldKey ? newKey : name));
|
||||
other.buddyRequests = other.buddyRequests.map(name => (name === oldKey ? newKey : name));
|
||||
named.push(other);
|
||||
}
|
||||
}
|
||||
save();
|
||||
save(...named);
|
||||
return null;
|
||||
};
|
||||
|
||||
@@ -772,13 +836,18 @@ const remove = account => {
|
||||
byId.delete(account.id);
|
||||
savedState.delete(account.id);
|
||||
retire(key);
|
||||
const named = [];
|
||||
for (const other of byId.values()) {
|
||||
other.tradeOffers = other.tradeOffers.filter(offer => offer.fromId !== account.id);
|
||||
other.buddies = other.buddies.filter(name => name !== key);
|
||||
other.blocked = other.blocked.filter(name => name !== key);
|
||||
other.buddyRequests = other.buddyRequests.filter(name => name !== key);
|
||||
if (other.tradeOffers.some(offer => offer.fromId === account.id)
|
||||
|| [other.buddies, other.blocked, other.buddyRequests].some(list => list.includes(key))) {
|
||||
other.tradeOffers = other.tradeOffers.filter(offer => offer.fromId !== account.id);
|
||||
other.buddies = other.buddies.filter(name => name !== key);
|
||||
other.blocked = other.blocked.filter(name => name !== key);
|
||||
other.buddyRequests = other.buddyRequests.filter(name => name !== key);
|
||||
named.push(other);
|
||||
}
|
||||
}
|
||||
save();
|
||||
save(...named);
|
||||
console.log(`[accounts] deleted "${account.username}" (id ${account.id})`);
|
||||
};
|
||||
|
||||
|
||||
Reference in new issue
Block a user