perf: save only changed accounts, cache missing assets, lighter logging and backup

Ping on prod jumped to seconds because node's main thread was busy most
of every second.

- accounts.save(account, ...) names the changed accounts, which are
  written within a second; the rest are looked at a few at a time
  (10 ms a save for those in play, 5 ms for the others). One save under
  prod-like load: 289 ms before, 16 ms after.
- an asset the CDN does not have gets a cacheable 404 and is logged once
- the proxy's own connections no longer count against the web cap, which
  had node reset requests nginx had already let in
- a repeated client error is written once in ten minutes; the panel's
  log buffer trims in batches
- the daily backup uses sqlite's online backup off the main thread
This commit is contained in:
Burrowdev committed 2026-10-17 11:57:40 +03:00
1 parent edca7bf790
commit aa455f52d0
26 files changed
+334 -220

No files matched your search

+148 -79
View File
@@ -3,7 +3,7 @@
const crypto = require("crypto");
const fs = require("fs");
const path = require("path");
const {DatabaseSync} = require("node:sqlite");
const {DatabaseSync, backup: copyDatabase} = require("node:sqlite");
const config = require("../config.js");
const defpacks = require("./defpacks.js");
const names = require("./names.js");
@@ -13,9 +13,9 @@ const names = require("./names.js");
// JSON document per account in `state`.
//
// every account is held in memory while the server runs. game code changes
// those objects directly and then calls save(), which has every account
// whose state changed written back, a moment later: however many times it
// is called in that moment, the accounts are gone through once (flush).
// those objects directly and then calls save(account, ...), which has the
// accounts named written back a moment later, and the others looked at a
// few at a time (flush, below).
const SCHEMA = `
CREATE TABLE IF NOT EXISTS accounts (
@@ -415,21 +415,30 @@ const load = () => {
console.log(`[accounts] ${byKey.size} accounts in ${path.join(config.dataDir, "game.sqlite")}`);
};
// Which accounts the save after a message looks at. turning an account into
// JSON to see whether it changed costs, and doing it for every account there
// is, a second after nearly every message, held the whole server up for
// longer the more accounts there were: a second and more, for everyone. so
// it looks at the accounts in play: online ones (hold, from lib/world.js),
// and ones looked up lately (by name, id, token, password: how the admin
// panel, buddies and the like get at an offline account), until a save
// finds them written and WARM_MS has gone by. the rest are looked at a few
// at a time behind those (SWEEP_BUDGET_MS a save), round and round, for
// what reached an account some other way (accounts.all()). a stop of the
// server, and every flush() called by name, looks at all of them.
// Which accounts a save looks at. turning an account into JSON to see
// whether it changed costs (up to a millisecond for a big one), and doing it
// a second after nearly every message for every account in play (online
// ones, and the buddies, dens and cards they had looked up lately) held the
// whole server up for most of every second once there were enough players.
// so game code says which accounts it changed, save(account, ...), and those
// are written within SAVE_DELAY_MS. behind them, for whatever changed an
// account without saying so, the accounts in play are looked at a few at a
// time (PLAY_BUDGET_MS a save), round and round: online ones (hold, from
// lib/world.js) and ones looked up lately (by name, id, token, password: how
// the admin panel, buddies and the like get at an offline account), for
// WARM_MS. behind those every other account is looked at the same way
// (SWEEP_BUDGET_MS), for what reached an account some other way
// (accounts.all()). a stop of the server, and every flush() called by name,
// looks at all of them.
const WARM_MS = 10 * 60 * 1000;
const PLAY_BUDGET_MS = 10;
const SWEEP_BUDGET_MS = 5;
const held = new Set(); // accounts online
const warm = new Map(); // account -> when last looked up
const dirty = new Set(); // accounts said to have changed since the last save
let playOrder = [];
let playAt = 0;
let playLeft = 0; // accounts in play still to be looked at since the last save()
let sweepOrder = [];
let sweepAt = 0;
let sweepLeft = 0; // accounts the sweep still has to look at since the last save()
@@ -457,9 +466,60 @@ const changedState = (account, changed) => {
return false;
};
const elapsedMs = started => Number(process.hrtime.bigint() - started) / 1e6;
// the next accounts in play, while there is time: the ones online and the
// ones looked up lately. one looked up longer ago than WARM_MS is looked at
// once more, then left to the sweep.
const lookAtPlay = (changed, looked) => {
const started = process.hrtime.bigint();
while (playLeft > 0 && elapsedMs(started) < PLAY_BUDGET_MS) {
if (playAt >= playOrder.length) {
const now = Date.now();
playOrder = [...new Set([...held, ...warm.keys()])];
playAt = 0;
for (const [account, at] of warm) {
if (now - at > WARM_MS && !held.has(account)) {
warm.delete(account);
}
}
if (playOrder.length === 0) {
playLeft = 0;
break;
}
}
const account = playOrder[playAt++];
playLeft--;
if (!looked.has(account) && byId.get(account.id) === account) {
looked.add(account);
changedState(account, changed);
}
}
};
const lookAtRest = (changed, looked) => {
const started = process.hrtime.bigint();
while (sweepLeft > 0 && elapsedMs(started) < SWEEP_BUDGET_MS) {
if (sweepAt >= sweepOrder.length) {
sweepOrder = [...byId.values()];
sweepAt = 0;
if (sweepOrder.length === 0) {
sweepLeft = 0;
break;
}
}
const account = sweepOrder[sweepAt++];
sweepLeft--;
if (!looked.has(account) && byId.get(account.id) === account) {
looked.add(account);
changedState(account, changed);
}
}
};
// writes every account whose state changed since it was last written, in
// one transaction; with {some: true}, of the accounts in play and the
// sweep's next few (above).
// one transaction; with {some: true}, of the accounts said to have changed
// and the next few of the others (above).
const flush = ({some = false} = {}) => {
clearTimeout(saveTimer);
saveTimer = null;
@@ -467,49 +527,28 @@ const flush = ({some = false} = {}) => {
return;
}
const changed = [];
const named = [...dirty];
dirty.clear();
if (some) {
const now = Date.now();
const looked = new Set();
for (const account of held) {
looked.add(account);
changedState(account, changed);
}
for (const [account, at] of warm) {
if (looked.has(account)) {
continue;
}
looked.add(account);
if (byId.get(account.id) !== account) {
warm.delete(account); // removed
}
else if (!changedState(account, changed) && now - at > WARM_MS && !held.has(account)) {
warm.delete(account);
}
}
const started = process.hrtime.bigint();
while (sweepLeft > 0 && Number(process.hrtime.bigint() - started) / 1e6 < SWEEP_BUDGET_MS) {
if (sweepAt >= sweepOrder.length) {
sweepOrder = [...byId.values()];
sweepAt = 0;
if (sweepOrder.length === 0) {
break;
}
}
const account = sweepOrder[sweepAt++];
sweepLeft--;
if (!looked.has(account) && byId.get(account.id) === account) {
for (const account of named) {
if (byId.get(account.id) === account) {
looked.add(account);
changedState(account, changed);
}
}
lookAtPlay(changed, looked);
lookAtRest(changed, looked);
}
else {
for (const account of byId.values()) {
changedState(account, changed);
}
playLeft = 0;
sweepLeft = 0;
}
if (sweepLeft > 0) {
save(); // the sweep goes on in a moment
if (playLeft > 0 || sweepLeft > 0) {
later(CONTINUE_MS); // the looking goes on in a moment
}
if (changed.length === 0) {
return;
@@ -524,6 +563,7 @@ const flush = ({some = false} = {}) => {
}
catch (err) {
db.exec("ROLLBACK");
named.forEach(account => dirty.add(account));
throw err;
}
for (const [id, json] of changed) {
@@ -531,16 +571,17 @@ const flush = ({some = false} = {}) => {
}
};
// "Something changed": the accounts are written within SAVE_DELAY_MS. going
// through every account costs more the more there are, and game code calls
// this after nearly every message (and some of it a hundred times in a
// row), so the going-through is done once for all the calls of a moment.
// what a crash can lose is that moment; a stop of the server writes first
// "Something changed": the accounts named are written within SAVE_DELAY_MS,
// and the looking-through above starts again. game code calls this after
// nearly every message (and some of it a hundred times in a row), so it is
// done once for all the calls of a moment. what a crash can lose is that
// moment, for an account named, and the time a round of the looking takes,
// for one changed without saying so; a stop of the server writes first
// (server.js).
const SAVE_DELAY_MS = 1000;
const CONTINUE_MS = 250;
let saveTimer = null;
const save = () => {
sweepLeft = byId.size;
const later = delay => {
if (saveTimer === null) {
saveTimer = setTimeout(() => {
try {
@@ -550,25 +591,47 @@ const save = () => {
console.error(`[accounts] could not save: ${err.message}`);
save(); // tried again in a moment: nothing is lost while the server runs
}
}, SAVE_DELAY_MS);
}, delay);
}
};
const save = (...changedAccounts) => {
for (const account of changedAccounts) {
if (account) {
dirty.add(account);
}
}
playLeft = held.size + warm.size;
sweepLeft = byId.size;
later(SAVE_DELAY_MS);
};
// A copy of the whole database, made once a day into data/backups/ and the
// latest BACKUPS_KEPT kept. it is a whole file of its own (VACUUM INTO), to
// be copied somewhere else by whoever runs the server: a copy on the same
// disk is no help when the disk goes.
// latest BACKUPS_KEPT kept. it is a whole file of its own, to be copied
// somewhere else by whoever runs the server: a copy on the same disk is no
// help when the disk goes. it is copied a batch of pages at a time off the
// main thread (sqlite's online backup): made in one go (VACUUM INTO, after
// writing every account) it held the game up for seconds.
const BACKUPS_KEPT = 14;
const backup = () => {
let backingUp = false;
const backup = async () => {
const folder = path.join(config.dataDir, "backups");
const file = path.join(folder, `game-${new Date().toISOString().slice(0, 10)}.sqlite`);
if (fs.existsSync(file)) {
if (backingUp || fs.existsSync(file)) {
return;
}
fs.mkdirSync(folder, {recursive: true, mode: 0o700});
flush();
db.prepare("VACUUM INTO ?").run(file);
fs.chmodSync(file, 0o600);
backingUp = true;
const part = `${file}.part`;
try {
fs.mkdirSync(folder, {recursive: true, mode: 0o700});
fs.rmSync(part, {force: true});
await copyDatabase(db, part);
fs.chmodSync(part, 0o600);
fs.renameSync(part, file); // only a whole copy has the day's name
}
finally {
backingUp = false;
fs.rmSync(part, {force: true});
}
const old = fs.readdirSync(folder).filter(name => /^game-\d{4}-\d{2}-\d{2}\.sqlite$/.test(name)).sort().slice(0, -BACKUPS_KEPT);
for (const name of old) {
fs.unlinkSync(path.join(folder, name));
@@ -577,12 +640,9 @@ const backup = () => {
};
const startBackups = () => {
const attempt = () => {
try {
backup();
}
catch (err) {
backup().catch(err => {
console.error(`[accounts] backup failed: ${err.message}`);
}
});
};
attempt();
setInterval(attempt, 3600000).unref();
@@ -733,12 +793,16 @@ const rename = (account, username, anyName = false) => {
byKey.delete(oldKey);
account.username = username;
byKey.set(newKey, account);
const named = [account];
for (const other of byId.values()) {
other.buddies = other.buddies.map(name => (name === oldKey ? newKey : name));
other.blocked = other.blocked.map(name => (name === oldKey ? newKey : name));
other.buddyRequests = other.buddyRequests.map(name => (name === oldKey ? newKey : name));
if ([other.buddies, other.blocked, other.buddyRequests].some(list => list.includes(oldKey))) {
other.buddies = other.buddies.map(name => (name === oldKey ? newKey : name));
other.blocked = other.blocked.map(name => (name === oldKey ? newKey : name));
other.buddyRequests = other.buddyRequests.map(name => (name === oldKey ? newKey : name));
named.push(other);
}
}
save();
save(...named);
return null;
};
@@ -772,13 +836,18 @@ const remove = account => {
byId.delete(account.id);
savedState.delete(account.id);
retire(key);
const named = [];
for (const other of byId.values()) {
other.tradeOffers = other.tradeOffers.filter(offer => offer.fromId !== account.id);
other.buddies = other.buddies.filter(name => name !== key);
other.blocked = other.blocked.filter(name => name !== key);
other.buddyRequests = other.buddyRequests.filter(name => name !== key);
if (other.tradeOffers.some(offer => offer.fromId === account.id)
|| [other.buddies, other.blocked, other.buddyRequests].some(list => list.includes(key))) {
other.tradeOffers = other.tradeOffers.filter(offer => offer.fromId !== account.id);
other.buddies = other.buddies.filter(name => name !== key);
other.blocked = other.blocked.filter(name => name !== key);
other.buddyRequests = other.buddyRequests.filter(name => name !== key);
named.push(other);
}
}
save();
save(...named);
console.log(`[accounts] deleted "${account.username}" (id ${account.id})`);
};