fix: send a game at login only the custom items its packs lack

Every custom item's full def row went to every game at login, though the
item and name packs it fetched minutes before already have them all: with
5000 clothing and 452 den items that was 3.3 MB, 4.2 MB with the colours,
over the send backlog limit (94b8de8 raised it).

- a fetch of the item, den item or name pack notes, per address, how far
  the custom items had changed; at login a game is sent the items changed
  since, and those with a picture (the packs have no pictures). an address
  that fetched nothing lately is sent all, as before.
- "xdi" rows carry the fields the client reads (addItemDef,
  addDenItemDef), not the whole base row, and clothing is for any animal
  as in the pack.
- accounts keep a digest of the state last written, not a copy of it
  (~13 KB an account, ~185 MB on prod).

Measured locally with prod-sized data (20000 colours, 5452 items, 14k
accounts): the login burst 4.2 MB -> 1.5 MB (the colours are most of what
is left), live heap after start 419 -> 297 MB.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
BurrowdevandClaude Opus 5.5 committed 2026-10-20 14:44:07 +03:00
1 parent 9f69b3511b
commit c5b1bc37d9
6 files changed
+146 -15

No files matched your search

+8 -5
View File
@@ -78,7 +78,10 @@ const LOGIN_DAYS_KEPT = 90; // and none older than this: they hold addresses
let db = null;
const byKey = new Map(); // lowercased username -> account
const byId = new Map();
const savedState = new Map(); // account id -> state JSON as last written
// account id -> digest of the state JSON as last written: enough to tell a
// change, without a second copy of every account's state (~13 KB each).
const savedState = new Map();
const stateDigest = json => crypto.createHash("sha256").update(json).digest("base64");
// A name somebody gave up (a rename, a deleted account) is nobody's to take
// for a while: whoever had it is still known by it to the players they met.
const NAME_HELD_DAYS = 30;
@@ -331,7 +334,7 @@ const insert = (username, salt, hash, createdAt, state) => {
const result = db.prepare("INSERT INTO accounts (username, username_key, password_salt, password_hash, created_at, state) VALUES (?, ?, ?, ?, ?, ?)")
.run(username, username.toLowerCase(), salt, hash, createdAt, json);
account.id = Number(result.lastInsertRowid);
savedState.set(account.id, json);
savedState.set(account.id, stateDigest(json));
remember(account);
touch(account);
return account;
@@ -381,7 +384,7 @@ const load = () => {
createdAt: row.created_at,
...JSON.parse(row.state),
});
savedState.set(account.id, row.state);
savedState.set(account.id, stateDigest(row.state));
remember(account);
}
importLegacyFile();
@@ -475,7 +478,7 @@ const release = account => {
const changedState = (account, changed) => {
const json = stateJson(account);
if (savedState.get(account.id) !== json) {
if (savedState.get(account.id) !== stateDigest(json)) {
changed.push([account.id, json]);
return true;
}
@@ -583,7 +586,7 @@ const flush = ({some = false} = {}) => {
throw err;
}
for (const [id, json] of changed) {
savedState.set(id, json);
savedState.set(id, stateDigest(json));
}
};
+74 -8
View File
@@ -3,14 +3,15 @@
const {logText} = require("./logtext.js");
const accounts = require("./accounts.js");
const defpacks = require("./defpacks.js");
const limits = require("./limits.js");
// items the staff make while the server runs, with no restart of the server
// or of anybody's game. each is an existing item's art (the clothing piece
// or den item it is made from) under an id, name, price and colours of its
// own. the server's tables and the packs new games fetch have it at once
// (defpacks.setCustomItem); games already playing are sent its def row
// ("xdi", which the patched client adds to its tables), and so is every game
// at login, in case it was started before the item was made.
// ("xdi", which the patched client adds to its tables), and so is a game at
// login if its packs may be older than the item (sendAll).
//
// an item can be taken out of the shop, renamed and repriced, or deleted:
// then it goes from every inventory too (lib/itemcontrol.js).
@@ -29,6 +30,21 @@ const CHUNK = 40; // def rows per "xdi" message
const TYPE_CLOTHING = "1";
const OFFERED_SPECIAL_TYPES = ["0", "4"]; // den items that can be copied: plain, mannequin
const CURRENCY_GEMS = "0";
// the fields of a def row the patched client reads from "xdi"
// (ItemXtCommManager.addItemDef, DenXtCommManager.addDenItemDef): the rest of
// the base's row is only weight, ~600 bytes a row and 5000 rows at login.
const CLIENT_FIELDS = {
clothing: ["id", "layerDefId", "layerId", "type", "value", "titleStrId", "isGiftable", "membersOnly", "itemStatus", "enviroType",
"avatarUseFlags", "recycleValue", "currencyType", "defense", "attack", "attackMediaRefId", "combatType", "modifierType",
"modifierValue", "criticalHit", "combinedCost", "colors", "availabilityStartTime", "availabilityEndTime"],
den: ["id", "abbrName", "combinedCost", "cost", "currencyType", "enviroType", "flag", "gameDefId", "itemStatus", "layer",
"mannequinAvatarDefId", "mannequinCatId", "membersOnly", "nameStrId", "parentArrayId", "promoType", "recycleValue",
"scalePercent", "sortCat", "specialType", "typeCat", "availabilityStartTime", "availabilityEndTime", "ecoPower"],
};
// how long a fetch of the packs is taken to describe a game of that address
// that logs in (sendAll). a game logs in minutes after it fetched them.
const PACKS_SEEN_MS = 2 * 3600000;
const MAX_PACKS_SEEN = 100000; // addresses
const SCHEMA = `CREATE TABLE IF NOT EXISTS custom_items (
kind TEXT NOT NULL, id INTEGER NOT NULL, base INTEGER NOT NULL, name TEXT NOT NULL,
@@ -37,7 +53,12 @@ const SCHEMA = `CREATE TABLE IF NOT EXISTS custom_items (
const items = new Map(KINDS.map(kind => [kind, new Map()])); // kind -> id -> entry
let made = 0; // entries in the order they were made (`seq`), for lists
let changes = 0; // goes up with every change a game must be told of; an entry's `changed` is the count at its last
let broadcast = () => {}; // (cmd, ...params) to every game playing; set by lib/world.js
// address key -> {at, clothing, den, names}: `changes` when that address
// last fetched the packs with custom items in, the oldest within
// PACKS_SEEN_MS (players behind one address fetch at different times).
const packsSeen = new Map();
const db = () => accounts.database();
const now = () => Math.floor(Date.now() / 1000);
@@ -64,7 +85,24 @@ const rowOf = entry => {
return {...base, ...own, titleStrId: String(STRING_BASE.clothing + entry.id - FIRST_ID), value: String(entry.cost), colors: entry.colors.join(","), isGiftable: "1"};
};
const apply = entry => defpacks.setCustomItem(entry.kind, rowOf(entry), entry.name, entry.inShop);
const apply = entry => {
defpacks.setCustomItem(entry.kind, rowOf(entry), entry.name, entry.inShop);
entry.changed = ++changes;
};
// what "xdi" carries of an entry: the fields the client reads, its name and
// picture. clothing goes through the clients' pack transform, so the wear
// rules reach custom items too.
const clientRow = entry => {
const row = entry.kind === "clothing" ? defpacks.clientItemRow(rowOf(entry)) : rowOf(entry);
const out = {};
for (const field of CLIENT_FIELDS[entry.kind]) {
if (row[field] !== undefined) {
out[field] = row[field];
}
}
return {...out, name: entry.name, art: entry.artPending ? "" : entry.art || "", artMode: entry.artMode || "sticker"};
};
// "xdi" [.., .., kind, json]: def rows with their names, as JSON. a field
// may not hold "%" (lib/sfs.js drops it), so a "%" in a name is written as
@@ -75,9 +113,7 @@ const messages = entries => {
const out = [];
for (const kind of KINDS) {
// a picture waiting for a moderator is not shown yet
const rows = entries.filter(entry => entry.kind === kind && !entry.pending)
.map(entry => ({...(kind === "clothing" ? defpacks.clientItemRow(rowOf(entry)) : rowOf(entry)), name: entry.name,
art: entry.artPending ? "" : entry.art || "", artMode: entry.artMode || "sticker"}));
const rows = entries.filter(entry => entry.kind === kind && !entry.pending).map(clientRow);
for (let at = 0; at < rows.length; at += CHUNK) {
out.push([kind, xtRows(rows.slice(at, at + CHUNK))]);
}
@@ -281,6 +317,7 @@ const setArtMode = (kind, id, mode) => {
}
entry.artMode = mode;
db().prepare("UPDATE custom_items SET art_mode = ? WHERE kind = ? AND id = ?").run(mode, kind, entry.id);
entry.changed = ++changes;
announce(entry);
return {item: entry};
};
@@ -295,6 +332,7 @@ const setArt = (kind, id, art, pending = false) => {
entry.art = art || null;
entry.artPending = Boolean(art && pending);
db().prepare("UPDATE custom_items SET art = ?, art_pending = ? WHERE kind = ? AND id = ?").run(entry.art, entry.artPending ? 1 : 0, kind, entry.id);
entry.changed = ++changes;
announce(entry);
};
@@ -362,9 +400,37 @@ module.exports = {
const def = baseDef(entry.kind, entry.base);
return def ? defpacks.string(entry.kind === "den" ? def.nameStrId : def.titleStrId) || String(def.abbrName || entry.base) : String(entry.base);
},
// every custom item, for a game logging in.
// an address fetched a pack with custom items in: "clothing", "den" or
// "names" (lib/defpacks.js customPackKind). called before the pack is made,
// so that a change while it is made counts as after.
sawPack(key, packKind) {
const time = Date.now();
const seen = packsSeen.get(key);
if (seen && time - seen.at < PACKS_SEEN_MS) {
if (seen[packKind] === undefined) {
seen[packKind] = changes;
}
return;
}
packsSeen.delete(key);
packsSeen.set(key, {at: time, [packKind]: changes});
// the oldest are first
for (const [oldKey, old] of packsSeen) {
if (packsSeen.size <= MAX_PACKS_SEEN && time - old.at < PACKS_SEEN_MS) {
break;
}
packsSeen.delete(oldKey);
}
},
// the custom items a game logging in may not have: those changed since its
// address fetched the packs, and those with a picture (which the packs do
// not carry); every one when the address fetched none lately.
sendAll(session) {
for (const params of messages(list().reverse().filter(entry => !entry.pending))) {
const seen = packsSeen.get(limits.key(session.address));
const fresh = seen && Date.now() - seen.at < PACKS_SEEN_MS && seen.names !== undefined ? seen : null;
const since = kind => (fresh && fresh[kind] !== undefined ? Math.min(fresh[kind], fresh.names) : -1);
const needed = entry => !entry.pending && (entry.changed > since(entry.kind) || (entry.art && !entry.artPending));
for (const params of messages(list().reverse().filter(needed))) {
session.sendXt("xdi", -1, ...params);
}
},
+6
View File
@@ -371,6 +371,8 @@ const CLIENT_PACKS = new Map([
]);
const clientPackPaths = new Map([...CLIENT_PACKS.keys()].map(id => [cdn.hashedPath(`defPacks/${id}`), id]));
const clientPackBodies = new Map(); // id -> {day, allOnSale, body}
// the packs custom items are in: their rows, and their names
const CUSTOM_PACK_KINDS = new Map([[DEF_ITEM, "clothing"], [DEF_DEN_ITEM, "den"], [DEF_LOCALIZATION_EN, "names"]]);
// the pack at `relPath` as clients should get it, or null when it is not
// one of the above (and comes straight from the CDN).
@@ -569,6 +571,10 @@ module.exports = {
// the custom items on sale, the newest first.
customShopIds: kind => [...customInShop[kind]].reverse(),
customVersion: () => customVersion,
// the avatarUseFlags every clothing row has in the clients' pack (any animal)
EVERY_AVATAR,
// "clothing", "den" or "names" for a pack path with custom items in (lib/customitems.js sawPack), else null.
customPackKind: relPath => CUSTOM_PACK_KINDS.get(clientPackPaths.get(relPath)) || null,
load,
itemDef: id => itemDefs.get(Number(id)),
clothingIds: () => clothingIds,
+5
View File
@@ -1049,6 +1049,11 @@ const handle = async (req, res) => {
send(res, extra.body ? 200 : 404, extra.body || "Not found", extra.body ? extra.type : "text/plain");
return;
}
// what custom items this game will have from its packs (customItems.sendAll)
const packKind = defpacks.customPackKind(relPath);
if (packKind) {
customItems.sawPack(from.key, packKind);
}
const clientPack = await defpacks.clientPack(relPath);
if (clientPack) {
send(res, 200, clientPack, "application/octet-stream");