From c5b1bc37d9467eaf780a974886722d4b65af26ac Mon Sep 17 00:00:00 2001 From: Burrowdev Date: Mon, 19 Oct 2026 13:54:08 +0300 Subject: [PATCH 01/11] fix: send a game at login only the custom items its packs lack Every custom item's full def row went to every game at login, though the item and name packs it fetched minutes before already have them all: with 5000 clothing and 452 den items that was 3.3 MB, 4.2 MB with the colours, over the send backlog limit (94b8de8 raised it). - a fetch of the item, den item or name pack notes, per address, how far the custom items had changed; at login a game is sent the items changed since, and those with a picture (the packs have no pictures). an address that fetched nothing lately is sent all, as before. - "xdi" rows carry the fields the client reads (addItemDef, addDenItemDef), not the whole base row, and clothing is for any animal as in the pack. - accounts keep a digest of the state last written, not a copy of it (~13 KB an account, ~185 MB on prod). Measured locally with prod-sized data (20000 colours, 5452 items, 14k accounts): the login burst 4.2 MB -> 1.5 MB (the colours are most of what is left), live heap after start 419 -> 297 MB. Co-Authored-By: Claude Opus 5.5 --- lib/accounts.js | 13 +++++--- lib/customitems.js | 82 +++++++++++++++++++++++++++++++++++++++++----- lib/defpacks.js | 6 ++++ lib/http.js | 5 +++ test/harness.js | 7 ++-- test/items.js | 48 +++++++++++++++++++++++++++ 6 files changed, 146 insertions(+), 15 deletions(-) diff --git a/lib/accounts.js b/lib/accounts.js index 50d679d..fe501e2 100644 --- a/lib/accounts.js +++ b/lib/accounts.js @@ -78,7 +78,10 @@ const LOGIN_DAYS_KEPT = 90; // and none older than this: they hold addresses let db = null; const byKey = new Map(); // lowercased username -> account const byId = new Map(); -const savedState = new Map(); // account id -> state JSON as last written +// account id -> digest of the state JSON as last written: enough to tell a +// change, without a second copy of every account's state (~13 KB each). +const savedState = new Map(); +const stateDigest = json => crypto.createHash("sha256").update(json).digest("base64"); // A name somebody gave up (a rename, a deleted account) is nobody's to take // for a while: whoever had it is still known by it to the players they met. const NAME_HELD_DAYS = 30; @@ -331,7 +334,7 @@ const insert = (username, salt, hash, createdAt, state) => { const result = db.prepare("INSERT INTO accounts (username, username_key, password_salt, password_hash, created_at, state) VALUES (?, ?, ?, ?, ?, ?)") .run(username, username.toLowerCase(), salt, hash, createdAt, json); account.id = Number(result.lastInsertRowid); - savedState.set(account.id, json); + savedState.set(account.id, stateDigest(json)); remember(account); touch(account); return account; @@ -381,7 +384,7 @@ const load = () => { createdAt: row.created_at, ...JSON.parse(row.state), }); - savedState.set(account.id, row.state); + savedState.set(account.id, stateDigest(row.state)); remember(account); } importLegacyFile(); @@ -475,7 +478,7 @@ const release = account => { const changedState = (account, changed) => { const json = stateJson(account); - if (savedState.get(account.id) !== json) { + if (savedState.get(account.id) !== stateDigest(json)) { changed.push([account.id, json]); return true; } @@ -583,7 +586,7 @@ const flush = ({some = false} = {}) => { throw err; } for (const [id, json] of changed) { - savedState.set(id, json); + savedState.set(id, stateDigest(json)); } }; diff --git a/lib/customitems.js b/lib/customitems.js index c7b26fb..8c56128 100644 --- a/lib/customitems.js +++ b/lib/customitems.js @@ -3,14 +3,15 @@ const {logText} = require("./logtext.js"); const accounts = require("./accounts.js"); const defpacks = require("./defpacks.js"); +const limits = require("./limits.js"); // items the staff make while the server runs, with no restart of the server // or of anybody's game. each is an existing item's art (the clothing piece // or den item it is made from) under an id, name, price and colours of its // own. the server's tables and the packs new games fetch have it at once // (defpacks.setCustomItem); games already playing are sent its def row -// ("xdi", which the patched client adds to its tables), and so is every game -// at login, in case it was started before the item was made. +// ("xdi", which the patched client adds to its tables), and so is a game at +// login if its packs may be older than the item (sendAll). // // an item can be taken out of the shop, renamed and repriced, or deleted: // then it goes from every inventory too (lib/itemcontrol.js). @@ -29,6 +30,21 @@ const CHUNK = 40; // def rows per "xdi" message const TYPE_CLOTHING = "1"; const OFFERED_SPECIAL_TYPES = ["0", "4"]; // den items that can be copied: plain, mannequin const CURRENCY_GEMS = "0"; +// the fields of a def row the patched client reads from "xdi" +// (ItemXtCommManager.addItemDef, DenXtCommManager.addDenItemDef): the rest of +// the base's row is only weight, ~600 bytes a row and 5000 rows at login. +const CLIENT_FIELDS = { + clothing: ["id", "layerDefId", "layerId", "type", "value", "titleStrId", "isGiftable", "membersOnly", "itemStatus", "enviroType", + "avatarUseFlags", "recycleValue", "currencyType", "defense", "attack", "attackMediaRefId", "combatType", "modifierType", + "modifierValue", "criticalHit", "combinedCost", "colors", "availabilityStartTime", "availabilityEndTime"], + den: ["id", "abbrName", "combinedCost", "cost", "currencyType", "enviroType", "flag", "gameDefId", "itemStatus", "layer", + "mannequinAvatarDefId", "mannequinCatId", "membersOnly", "nameStrId", "parentArrayId", "promoType", "recycleValue", + "scalePercent", "sortCat", "specialType", "typeCat", "availabilityStartTime", "availabilityEndTime", "ecoPower"], +}; +// how long a fetch of the packs is taken to describe a game of that address +// that logs in (sendAll). a game logs in minutes after it fetched them. +const PACKS_SEEN_MS = 2 * 3600000; +const MAX_PACKS_SEEN = 100000; // addresses const SCHEMA = `CREATE TABLE IF NOT EXISTS custom_items ( kind TEXT NOT NULL, id INTEGER NOT NULL, base INTEGER NOT NULL, name TEXT NOT NULL, @@ -37,7 +53,12 @@ const SCHEMA = `CREATE TABLE IF NOT EXISTS custom_items ( const items = new Map(KINDS.map(kind => [kind, new Map()])); // kind -> id -> entry let made = 0; // entries in the order they were made (`seq`), for lists +let changes = 0; // goes up with every change a game must be told of; an entry's `changed` is the count at its last let broadcast = () => {}; // (cmd, ...params) to every game playing; set by lib/world.js +// address key -> {at, clothing, den, names}: `changes` when that address +// last fetched the packs with custom items in, the oldest within +// PACKS_SEEN_MS (players behind one address fetch at different times). +const packsSeen = new Map(); const db = () => accounts.database(); const now = () => Math.floor(Date.now() / 1000); @@ -64,7 +85,24 @@ const rowOf = entry => { return {...base, ...own, titleStrId: String(STRING_BASE.clothing + entry.id - FIRST_ID), value: String(entry.cost), colors: entry.colors.join(","), isGiftable: "1"}; }; -const apply = entry => defpacks.setCustomItem(entry.kind, rowOf(entry), entry.name, entry.inShop); +const apply = entry => { + defpacks.setCustomItem(entry.kind, rowOf(entry), entry.name, entry.inShop); + entry.changed = ++changes; +}; + +// what "xdi" carries of an entry: the fields the client reads, its name and +// picture. clothing goes through the clients' pack transform, so the wear +// rules reach custom items too. +const clientRow = entry => { + const row = entry.kind === "clothing" ? defpacks.clientItemRow(rowOf(entry)) : rowOf(entry); + const out = {}; + for (const field of CLIENT_FIELDS[entry.kind]) { + if (row[field] !== undefined) { + out[field] = row[field]; + } + } + return {...out, name: entry.name, art: entry.artPending ? "" : entry.art || "", artMode: entry.artMode || "sticker"}; +}; // "xdi" [.., .., kind, json]: def rows with their names, as JSON. a field // may not hold "%" (lib/sfs.js drops it), so a "%" in a name is written as @@ -75,9 +113,7 @@ const messages = entries => { const out = []; for (const kind of KINDS) { // a picture waiting for a moderator is not shown yet - const rows = entries.filter(entry => entry.kind === kind && !entry.pending) - .map(entry => ({...(kind === "clothing" ? defpacks.clientItemRow(rowOf(entry)) : rowOf(entry)), name: entry.name, - art: entry.artPending ? "" : entry.art || "", artMode: entry.artMode || "sticker"})); + const rows = entries.filter(entry => entry.kind === kind && !entry.pending).map(clientRow); for (let at = 0; at < rows.length; at += CHUNK) { out.push([kind, xtRows(rows.slice(at, at + CHUNK))]); } @@ -281,6 +317,7 @@ const setArtMode = (kind, id, mode) => { } entry.artMode = mode; db().prepare("UPDATE custom_items SET art_mode = ? WHERE kind = ? AND id = ?").run(mode, kind, entry.id); + entry.changed = ++changes; announce(entry); return {item: entry}; }; @@ -295,6 +332,7 @@ const setArt = (kind, id, art, pending = false) => { entry.art = art || null; entry.artPending = Boolean(art && pending); db().prepare("UPDATE custom_items SET art = ?, art_pending = ? WHERE kind = ? AND id = ?").run(entry.art, entry.artPending ? 1 : 0, kind, entry.id); + entry.changed = ++changes; announce(entry); }; @@ -362,9 +400,37 @@ module.exports = { const def = baseDef(entry.kind, entry.base); return def ? defpacks.string(entry.kind === "den" ? def.nameStrId : def.titleStrId) || String(def.abbrName || entry.base) : String(entry.base); }, - // every custom item, for a game logging in. + // an address fetched a pack with custom items in: "clothing", "den" or + // "names" (lib/defpacks.js customPackKind). called before the pack is made, + // so that a change while it is made counts as after. + sawPack(key, packKind) { + const time = Date.now(); + const seen = packsSeen.get(key); + if (seen && time - seen.at < PACKS_SEEN_MS) { + if (seen[packKind] === undefined) { + seen[packKind] = changes; + } + return; + } + packsSeen.delete(key); + packsSeen.set(key, {at: time, [packKind]: changes}); + // the oldest are first + for (const [oldKey, old] of packsSeen) { + if (packsSeen.size <= MAX_PACKS_SEEN && time - old.at < PACKS_SEEN_MS) { + break; + } + packsSeen.delete(oldKey); + } + }, + // the custom items a game logging in may not have: those changed since its + // address fetched the packs, and those with a picture (which the packs do + // not carry); every one when the address fetched none lately. sendAll(session) { - for (const params of messages(list().reverse().filter(entry => !entry.pending))) { + const seen = packsSeen.get(limits.key(session.address)); + const fresh = seen && Date.now() - seen.at < PACKS_SEEN_MS && seen.names !== undefined ? seen : null; + const since = kind => (fresh && fresh[kind] !== undefined ? Math.min(fresh[kind], fresh.names) : -1); + const needed = entry => !entry.pending && (entry.changed > since(entry.kind) || (entry.art && !entry.artPending)); + for (const params of messages(list().reverse().filter(needed))) { session.sendXt("xdi", -1, ...params); } }, diff --git a/lib/defpacks.js b/lib/defpacks.js index 7107186..a26c39f 100644 --- a/lib/defpacks.js +++ b/lib/defpacks.js @@ -371,6 +371,8 @@ const CLIENT_PACKS = new Map([ ]); const clientPackPaths = new Map([...CLIENT_PACKS.keys()].map(id => [cdn.hashedPath(`defPacks/${id}`), id])); const clientPackBodies = new Map(); // id -> {day, allOnSale, body} +// the packs custom items are in: their rows, and their names +const CUSTOM_PACK_KINDS = new Map([[DEF_ITEM, "clothing"], [DEF_DEN_ITEM, "den"], [DEF_LOCALIZATION_EN, "names"]]); // the pack at `relPath` as clients should get it, or null when it is not // one of the above (and comes straight from the CDN). @@ -569,6 +571,10 @@ module.exports = { // the custom items on sale, the newest first. customShopIds: kind => [...customInShop[kind]].reverse(), customVersion: () => customVersion, + // the avatarUseFlags every clothing row has in the clients' pack (any animal) + EVERY_AVATAR, + // "clothing", "den" or "names" for a pack path with custom items in (lib/customitems.js sawPack), else null. + customPackKind: relPath => CUSTOM_PACK_KINDS.get(clientPackPaths.get(relPath)) || null, load, itemDef: id => itemDefs.get(Number(id)), clothingIds: () => clothingIds, diff --git a/lib/http.js b/lib/http.js index b9a983c..5f7e0b3 100644 --- a/lib/http.js +++ b/lib/http.js @@ -1049,6 +1049,11 @@ const handle = async (req, res) => { send(res, extra.body ? 200 : 404, extra.body || "Not found", extra.body ? extra.type : "text/plain"); return; } + // what custom items this game will have from its packs (customItems.sendAll) + const packKind = defpacks.customPackKind(relPath); + if (packKind) { + customItems.sawPack(from.key, packKind); + } const clientPack = await defpacks.clientPack(relPath); if (clientPack) { send(res, 200, clientPack, "application/octet-stream"); diff --git a/test/harness.js b/test/harness.js index e802849..bdb18b5 100644 --- a/test/harness.js +++ b/test/harness.js @@ -13,8 +13,11 @@ const os = require("os"); const path = require("path"); class GameClient { - constructor(gamePort) { + // `localAddress`: another loopback address to come from (127.0.0.2), for + // what the server tells apart by address. + constructor(gamePort, localAddress = undefined) { this.gamePort = gamePort; + this.localAddress = localAddress; this.queue = []; this.varUpdates = []; this.askedForGames = 0; @@ -25,7 +28,7 @@ class GameClient { connect() { return new Promise((resolve, reject) => { - this.socket = net.connect(this.gamePort, "127.0.0.1", resolve); + this.socket = net.connect({port: this.gamePort, host: "127.0.0.1", localAddress: this.localAddress}, resolve); this.socket.once("error", reject); this.socket.on("data", chunk => { this.buffer = Buffer.concat([this.buffer, chunk]); diff --git a/test/items.js b/test/items.js index 64f8781..a0e8893 100644 --- a/test/items.js +++ b/test/items.js @@ -8,6 +8,7 @@ const assert = require("assert"); const crypto = require("crypto"); const fs = require("fs"); +const http = require("http"); const os = require("os"); const path = require("path"); const zlib = require("zlib"); @@ -119,6 +120,34 @@ const untilDefs = async (client, kind) => { } } }; +// a game on another address: it fetches the packs with custom items in, and +// at login is sent of them only what they may lack (customItems.sendAll), +// all of it before the chat filter settings ("xfl"). +const ELSEWHERE = "127.0.0.2"; +const packsFrom = address => Promise.all([1000, 1030, 10230].map(id => new Promise((resolve, reject) => { + http.get(`${BASE}/${cdn.hashedPath(`defPacks/${id}`)}`, {localAddress: address}, res => { + assert.strictEqual(res.statusCode, 200); + res.resume(); + res.on("end", resolve); + }).on("error", reject); +}))); +const defsAtLogin = async (username, password, address) => { + const auth = await (await postJson(`${BASE}/authenticate`, {username, password})).json(); + const client = new harness.GameClient(GAME_PORT, address); + await client.connect(); + assert.strictEqual((await client.logIn(username, auth.auth_token)).status, "1"); + const defs = {clothing: [], den: []}; + for (;;) { + const fields = (await client.next()).split("%").slice(2, -1); + if (fields[0] === "xfl") { + client.socket.destroy(); + return defs; + } + if (fields[0] === "xdi") { + defs[fields[2]].push(...JSON.parse(fields[3])); + } + } +}; const nextBox = async client => (await client.untilXt("xtl"))[2]; const shopList = async (client, roomId, shop) => { client.xt("gl", roomId, shop); @@ -194,6 +223,7 @@ const first = async () => { const crownRow = byId(await pack(1000), 50000); assert.strictEqual(crownRow.layerDefId, base.layerDefId); assert.strictEqual(crownRow.avatarUseFlags, "9007199254740991"); // any animal, like all clothing here + assert.strictEqual(row.avatarUseFlags, crownRow.avatarUseFlags); // and so in what games playing are sent assert.strictEqual((await strings())[9100000], "Test Crown"); // first in the shop of oddities, and Pat can buy it in a colour it has @@ -235,6 +265,20 @@ const first = async () => { say(devGame.client, devGame.roomId, `!item clothing ${base.id} Quick Hat`); assert.strictEqual(await nextBox(devGame.client), "Made Quick Hat, clothing 50001. It is in the item finder now."); assert.strictEqual((await untilDefs(pat.client, "clothing"))[0].name, "Quick Hat"); + // a game whose address fetched the packs since has every item from them, + // and is sent none again; then only the one changed after + await packsFrom(ELSEWHERE); + assert.deepStrictEqual(await defsAtLogin("Lou", people.Lou, ELSEWHERE), {clothing: [], den: []}); + assert.match((await dev("/mod/api/items", {action: "update", kind: "clothing", id: 50000, cost: 7})).data.ok, /changed/); + assert.strictEqual((await untilDefs(pat.client, "clothing"))[0].value, "7"); + const changedSince = await defsAtLogin("Lou", people.Lou, ELSEWHERE); + assert.deepStrictEqual(changedSince.clothing.map(row => `${row.id} ${row.name} ${row.value}`), ["50000 Gold Crown 7"]); + assert.deepStrictEqual(changedSince.den, []); + // a row carries what the client reads of it, not the whole base row + const wire = ["id", "layerDefId", "layerId", "type", "value", "titleStrId", "isGiftable", "membersOnly", "itemStatus", "enviroType", + "avatarUseFlags", "recycleValue", "currencyType", "defense", "attack", "attackMediaRefId", "combatType", "modifierType", + "modifierValue", "criticalHit", "combinedCost", "colors", "availabilityStartTime", "availabilityEndTime", "name", "art", "artMode"]; + assert.deepStrictEqual(Object.keys(changedSince.clothing[0]).filter(field => !wire.includes(field)), []); say(devGame.client, devGame.roomId, "!item list"); assert.match(await nextBox(devGame.client), /^Latest items made: 50001 Quick Hat \(clothing\); 50000 Test Chair \(den\); 50000 Gold Crown \(clothing, off sale\)$/); say(devGame.client, devGame.roomId, "!item hat 12 x"); @@ -385,6 +429,10 @@ const first = async () => { const withArt = (await maker("info")).mine.find(sameItem); assert.match(withArt.art, /^p[0-9a-f]{12}$/); assert.deepStrictEqual(told, [[String(hatEntry.id), withArt.art]]); + // the packs have no pictures: a game logging in is sent each one, however new its packs + await packsFrom(ELSEWHERE); + const pictured = (await defsAtLogin("Lou", people.Lou, ELSEWHERE)).clothing; + assert.deepStrictEqual(pictured.filter(row => row.art).map(row => [row.id, row.art]), [[String(hatEntry.id), withArt.art]]); const served = await fetch(`${BASE}/itemart/clothing/${hatEntry.id}.png?v=${withArt.art}`); assert.deepStrictEqual([served.status, served.headers.get("content-type"), served.headers.get("x-content-type-options")], [200, "image/png", "nosniff"]); const bytes = Buffer.from(await served.arrayBuffer()); From 04b8d325eac6bcb24480591d2a3cfa192d116f74 Mon Sep 17 00:00:00 2001 From: Burrowdev Date: Mon, 19 Oct 2026 15:37:29 +0300 Subject: [PATCH 02/11] perf(defpacks): make the item packs again without holding the game up A custom item change made the item and name packs again on the next fetch, on the one thread every game shares: ~360 ms at 5000 items, ~1.6 s at 50000, after every change, players making them many times a minute. Nearly all of it was the AMF3 writer. - the writer fills one growing buffer instead of a Buffer per value (byte-identical output, checked against the old one on every cached pack): the item pack 230 -> 52 ms at 5000 items, 1524 -> 231 ms at 50000. - a changed pack is made again at most every AJ_PACK_REMAKE_MS (15 s); until then the one made before goes out with the custom item version it has, and customItems.sendAll sends a game logging in what its packs lack by that version. one remake at a time; deflated on libuv's threads. - tests make the packs again at once (AJ_PACK_REMAKE_MS=0), and a run of the items test checks the throttled case. Busy evening simulated in-process (a change every 3 s, packs fetched 6 a second, 30 s): longest hold-up 329 -> 61 ms at 5000 items, 1648 -> 359 ms at 50000 (p99 240 -> 6 ms). Co-Authored-By: Claude Opus 5.5 --- config.js | 3 ++ lib/amf3.js | 121 +++++++++++++++++++++++++++++++++------------ lib/customitems.js | 25 +++++----- lib/defpacks.js | 54 +++++++++++++++----- lib/http.js | 12 ++--- test/harness.js | 4 +- test/items.js | 23 +++++++++ 7 files changed, 178 insertions(+), 64 deletions(-) diff --git a/config.js b/config.js index 882f37b..f957977 100644 --- a/config.js +++ b/config.js @@ -327,6 +327,9 @@ module.exports = { cacheMissesPerTenMinutes: 1500, // a first visit with an empty cache fetches several hundred files upstreamTimeoutMs: 20000, maxCacheBytes: Number(env.AJ_MAX_CACHE_MB || 8192) * 1024 * 1024, + // how often at most the packs with custom items in are made again after + // one changes (lib/defpacks.js clientPack); the tests use 0, made at once. + packRemakeMs: env.AJ_PACK_REMAKE_MS === undefined ? 15000 : Number(env.AJ_PACK_REMAKE_MS), maxConnections: 2000, maxHttpConnections: 2000, // open at once, in all maxHttpConnectionsPerAddress: 40, // the game opens a good many at once while it loads diff --git a/lib/amf3.js b/lib/amf3.js index 13b0cab..4c1c412 100644 --- a/lib/amf3.js +++ b/lib/amf3.js @@ -177,40 +177,89 @@ class Amf3Reader { // writer for what the reader above produces from a defPack: plain objects, // arrays (with or without string keys), strings, numbers, booleans and null. // nothing is written as a reference except repeated strings. +// written into one buffer that grows: a pack is megabytes of small values, +// and a Buffer for each of them took seconds for the item pack. class Amf3Writer { constructor() { - this.parts = []; + this.buf = Buffer.allocUnsafe(1 << 16); + this.pos = 0; this.strings = new Map(); } + room(bytes) { + if (this.pos + bytes > this.buf.length) { + let size = this.buf.length * 2; + while (size < this.pos + bytes) { + size *= 2; + } + const bigger = Buffer.allocUnsafe(size); + this.buf.copy(bigger, 0, 0, this.pos); + this.buf = bigger; + } + } + + byte(value) { + this.room(1); + this.buf[this.pos++] = value; + } + u29(value) { + this.room(4); + const {buf} = this; if (value < 0x80) { - this.parts.push(Buffer.from([value])); + buf[this.pos++] = value; } else if (value < 0x4000) { - this.parts.push(Buffer.from([(value >> 7) | 0x80, value & 0x7f])); + buf[this.pos++] = (value >> 7) | 0x80; + buf[this.pos++] = value & 0x7f; } else if (value < 0x200000) { - this.parts.push(Buffer.from([(value >> 14) | 0x80, ((value >> 7) & 0x7f) | 0x80, value & 0x7f])); + buf[this.pos++] = (value >> 14) | 0x80; + buf[this.pos++] = ((value >> 7) & 0x7f) | 0x80; + buf[this.pos++] = value & 0x7f; } else { - this.parts.push(Buffer.from([(value >> 22) | 0x80, ((value >> 15) & 0x7f) | 0x80, ((value >> 8) & 0x7f) | 0x80, value & 0xff])); + buf[this.pos++] = (value >> 22) | 0x80; + buf[this.pos++] = ((value >> 15) & 0x7f) | 0x80; + buf[this.pos++] = ((value >> 8) & 0x7f) | 0x80; + buf[this.pos++] = value & 0xff; } } string(text) { if (text === "") { this.u29(1); + return; } - else if (this.strings.has(text)) { - this.u29(this.strings.get(text) << 1); + const known = this.strings.get(text); + if (known !== undefined) { + this.u29(known << 1); + return; } - else { - this.strings.set(text, this.strings.size); - const bytes = Buffer.from(text, "utf8"); - this.u29((bytes.length << 1) | 1); - this.parts.push(bytes); + this.strings.set(text, this.strings.size); + // nearly every string of a pack is short and ASCII: its bytes are its + // characters, copied here rather than through Buffer's utf8 writer. + if (text.length < 64) { + let ascii = true; + for (let i = 0; i < text.length; i++) { + if (text.charCodeAt(i) > 0x7f) { + ascii = false; + break; + } + } + if (ascii) { + this.u29((text.length << 1) | 1); + this.room(text.length); + for (let i = 0; i < text.length; i++) { + this.buf[this.pos++] = text.charCodeAt(i); + } + return; + } } + const length = Buffer.byteLength(text, "utf8"); + this.u29((length << 1) | 1); + this.room(length); + this.pos += this.buf.write(text, this.pos, "utf8"); } pairs(entries) { @@ -222,45 +271,55 @@ class Amf3Writer { } value(value) { - if (value === undefined) { - this.parts.push(Buffer.from([0x00])); + // most values of a pack are strings: asked first + if (typeof value === "string") { + this.byte(0x06); + this.string(value); + } + else if (value === undefined) { + this.byte(0x00); } else if (value === null) { - this.parts.push(Buffer.from([0x01])); + this.byte(0x01); } else if (typeof value === "boolean") { - this.parts.push(Buffer.from([value ? 0x03 : 0x02])); + this.byte(value ? 0x03 : 0x02); } else if (typeof value === "number") { if (Number.isInteger(value) && value >= -0x10000000 && value < 0x10000000) { - this.parts.push(Buffer.from([0x04])); + this.byte(0x04); this.u29(value & 0x1fffffff); } else { - const bytes = Buffer.alloc(9); - bytes[0] = 0x05; - bytes.writeDoubleBE(value, 1); - this.parts.push(bytes); + this.byte(0x05); + this.room(8); + this.pos = this.buf.writeDoubleBE(value, this.pos); } } - else if (typeof value === "string") { - this.parts.push(Buffer.from([0x06])); - this.string(value); - } else if (Array.isArray(value)) { - this.parts.push(Buffer.from([0x09])); + this.byte(0x09); this.u29((value.length << 1) | 1); - // string keys set on the array beside its numbered elements. - this.pairs(Object.entries(value).filter(([key]) => !/^\d+$/.test(key))); + // string keys set on the array beside its numbered elements (an array + // with as many keys as elements has none). + const keys = Object.keys(value); + this.pairs(keys.length === value.length ? [] : keys.filter(key => !/^\d+$/.test(key)).map(key => [key, value[key]])); for (const element of value) { this.value(element); } } else if (typeof value === "object" && !Buffer.isBuffer(value)) { // A dynamic object of no class, with its traits written in place. - this.parts.push(Buffer.from([0x0a, 0x0b])); + this.byte(0x0a); + this.byte(0x0b); + this.string(""); + // its own keys in Object.keys order, without an array of entries for each object + for (const key in value) { + if (Object.hasOwn(value, key)) { + this.string(key); + this.value(value[key]); + } + } this.string(""); - this.pairs(Object.entries(value)); } else { throw new Error("Cannot write this value as AMF3"); @@ -273,6 +332,6 @@ module.exports = { encode: value => { const writer = new Amf3Writer(); writer.value(value); - return Buffer.concat(writer.parts); + return writer.buf.subarray(0, writer.pos); }, }; diff --git a/lib/customitems.js b/lib/customitems.js index 8c56128..1290c2f 100644 --- a/lib/customitems.js +++ b/lib/customitems.js @@ -53,11 +53,11 @@ const SCHEMA = `CREATE TABLE IF NOT EXISTS custom_items ( const items = new Map(KINDS.map(kind => [kind, new Map()])); // kind -> id -> entry let made = 0; // entries in the order they were made (`seq`), for lists -let changes = 0; // goes up with every change a game must be told of; an entry's `changed` is the count at its last let broadcast = () => {}; // (cmd, ...params) to every game playing; set by lib/world.js -// address key -> {at, clothing, den, names}: `changes` when that address -// last fetched the packs with custom items in, the oldest within -// PACKS_SEEN_MS (players behind one address fetch at different times). +// address key -> {at, clothing, den, names}: the version (defpacks +// customVersion) of the packs with custom items in that address last +// fetched, the oldest within PACKS_SEEN_MS (players behind one address +// fetch at different times). an entry's `changed` is the version of its last change. const packsSeen = new Map(); const db = () => accounts.database(); @@ -87,7 +87,7 @@ const rowOf = entry => { const apply = entry => { defpacks.setCustomItem(entry.kind, rowOf(entry), entry.name, entry.inShop); - entry.changed = ++changes; + entry.changed = defpacks.customVersion(); }; // what "xdi" carries of an entry: the fields the client reads, its name and @@ -317,7 +317,7 @@ const setArtMode = (kind, id, mode) => { } entry.artMode = mode; db().prepare("UPDATE custom_items SET art_mode = ? WHERE kind = ? AND id = ?").run(mode, kind, entry.id); - entry.changed = ++changes; + entry.changed = defpacks.customChanged(); announce(entry); return {item: entry}; }; @@ -332,7 +332,7 @@ const setArt = (kind, id, art, pending = false) => { entry.art = art || null; entry.artPending = Boolean(art && pending); db().prepare("UPDATE custom_items SET art = ?, art_pending = ? WHERE kind = ? AND id = ?").run(entry.art, entry.artPending ? 1 : 0, kind, entry.id); - entry.changed = ++changes; + entry.changed = defpacks.customChanged(); announce(entry); }; @@ -400,20 +400,19 @@ module.exports = { const def = baseDef(entry.kind, entry.base); return def ? defpacks.string(entry.kind === "den" ? def.nameStrId : def.titleStrId) || String(def.abbrName || entry.base) : String(entry.base); }, - // an address fetched a pack with custom items in: "clothing", "den" or - // "names" (lib/defpacks.js customPackKind). called before the pack is made, - // so that a change while it is made counts as after. - sawPack(key, packKind) { + // an address was sent a pack with custom items in: "clothing", "den" or + // "names" (lib/defpacks.js customPackKind), made at `version`. + sawPack(key, packKind, version) { const time = Date.now(); const seen = packsSeen.get(key); if (seen && time - seen.at < PACKS_SEEN_MS) { if (seen[packKind] === undefined) { - seen[packKind] = changes; + seen[packKind] = version; } return; } packsSeen.delete(key); - packsSeen.set(key, {at: time, [packKind]: changes}); + packsSeen.set(key, {at: time, [packKind]: version}); // the oldest are first for (const [oldKey, old] of packsSeen) { if (packsSeen.size <= MAX_PACKS_SEEN && time - old.at < PACKS_SEEN_MS) { diff --git a/lib/defpacks.js b/lib/defpacks.js index a26c39f..4ff8cf6 100644 --- a/lib/defpacks.js +++ b/lib/defpacks.js @@ -370,26 +370,51 @@ const CLIENT_PACKS = new Map([ [DEF_NAMEBAR_BADGE, () => {}], ]); const clientPackPaths = new Map([...CLIENT_PACKS.keys()].map(id => [cdn.hashedPath(`defPacks/${id}`), id])); -const clientPackBodies = new Map(); // id -> {day, allOnSale, body} +const clientPackBodies = new Map(); // id -> {day, allOnSale, body, version, madeAt} +const clientPacksChanged = new Set(); // ids whose body is older than a custom item change +const clientPacksMaking = new Map(); // id -> the promise of the body being made +// a pack is made again after a custom item change at most every +// config.packRemakeMs: the item pack takes ~70 ms of the one thread at 5000 +// items (~300 ms at 50000), and players make items many times a minute. +// until then the body made before goes out, with the version it has; a +// game logging in is sent what changed after it (lib/customitems.js sendAll). +const deflateRaw = require("util").promisify(zlib.deflateRaw); // the packs custom items are in: their rows, and their names const CUSTOM_PACK_KINDS = new Map([[DEF_ITEM, "clothing"], [DEF_DEN_ITEM, "den"], [DEF_LOCALIZATION_EN, "names"]]); -// the pack at `relPath` as clients should get it, or null when it is not -// one of the above (and comes straight from the CDN). +const makeClientPack = async id => { + // the version before the rows are read: a change while it is made counts + // as after, and is sent at login (more than needed, never less) + const version = customVersion; + const pack = await loadChanged(id); + CLIENT_PACKS.get(id)(pack); + // deflated on libuv's threads, not this one + const body = await deflateRaw(amf3.encode(pack)); + const made = {day: sales.today(), allOnSale: sales.everythingOnSale(), body, version, madeAt: Date.now()}; + clientPackBodies.set(id, made); + if (customVersion === version) { + clientPacksChanged.delete(id); + } + return made; +}; + +// the pack at `relPath` as clients should get it, {body, version}, or null +// when it is not one of the above (and comes straight from the CDN). +// `version`: customVersion() of the custom items in it. const clientPack = async relPath => { const id = clientPackPaths.get(relPath); if (id === undefined) { return null; } const made = clientPackBodies.get(id); - if (made && made.day === sales.today() && made.allOnSale === sales.everythingOnSale()) { - return made.body; + if (made && made.day === sales.today() && made.allOnSale === sales.everythingOnSale() + && (!clientPacksChanged.has(id) || Date.now() - made.madeAt < config.packRemakeMs)) { + return made; } - const pack = await loadChanged(id); - CLIENT_PACKS.get(id)(pack); - const body = zlib.deflateRawSync(amf3.encode(pack)); - clientPackBodies.set(id, {day: sales.today(), allOnSale: sales.everythingOnSale(), body}); - return body; + if (!clientPacksMaking.has(id)) { + clientPacksMaking.set(id, makeClientPack(id).finally(() => clientPacksMaking.delete(id))); + } + return clientPacksMaking.get(id); }; // most packs are tables: an object of rows keyed by row index. @@ -537,8 +562,8 @@ const setCustomItem = (kind, row, name, inShop = true) => { clothingIds = clothingIds.includes(id) ? clothingIds : [...clothingIds, id]; } customInShop[kind][inShop ? "add" : "delete"](id); - clientPackBodies.delete(pack); - clientPackBodies.delete(DEF_LOCALIZATION_EN); + clientPacksChanged.add(pack); + clientPacksChanged.add(DEF_LOCALIZATION_EN); customVersion++; }; @@ -560,7 +585,7 @@ const unsetCustomItem = (kind, id) => { clothingIds = clothingIds.filter(one => one !== number); } customInShop[kind].delete(number); - clientPackBodies.delete(pack); + clientPacksChanged.add(pack); customVersion++; }; @@ -571,6 +596,9 @@ module.exports = { // the custom items on sale, the newest first. customShopIds: kind => [...customInShop[kind]].reverse(), customVersion: () => customVersion, + // a change of a custom item that is not in the packs (its picture): one + // more version, for a game logging in to be told of it + customChanged: () => ++customVersion, // the avatarUseFlags every clothing row has in the clients' pack (any animal) EVERY_AVATAR, // "clothing", "den" or "names" for a pack path with custom items in (lib/customitems.js sawPack), else null. diff --git a/lib/http.js b/lib/http.js index 5f7e0b3..7a2e47b 100644 --- a/lib/http.js +++ b/lib/http.js @@ -1049,14 +1049,14 @@ const handle = async (req, res) => { send(res, extra.body ? 200 : 404, extra.body || "Not found", extra.body ? extra.type : "text/plain"); return; } - // what custom items this game will have from its packs (customItems.sendAll) - const packKind = defpacks.customPackKind(relPath); - if (packKind) { - customItems.sawPack(from.key, packKind); - } const clientPack = await defpacks.clientPack(relPath); if (clientPack) { - send(res, 200, clientPack, "application/octet-stream"); + // what custom items this game has from its packs (customItems.sendAll) + const packKind = defpacks.customPackKind(relPath); + if (packKind) { + customItems.sawPack(from.key, packKind, clientPack.version); + } + send(res, 200, clientPack.body, "application/octet-stream"); return; } // A room file in a seasonal theme ("/roomDefs/theme_ { lou.client.socket.destroy(); }; +// the packs made again at most every AJ_PACK_REMAKE_MS after a change: one +// fetched before then is the one made before, and a game whose address +// fetched it is sent at login the item it lacks. +const REMAKE_MS = 5000; +const third = async () => { + const fetchedEarlier = "127.0.0.3"; + await packsFrom(fetchedEarlier); + const lou = await arrive("Lou", people.Lou); + lou.client.xt("xcm", lou.roomId, "make", "clothing", made.base.id, "Lou Hat"); + const answer = await lou.client.untilXt("xcm"); + assert.strictEqual(answer[2], "1", answer[3]); + const louHat = JSON.parse(answer[6]).find(entry => entry.name === "Lou Hat"); + assert.ok(!byId(await pack(1000), louHat.id)); // the pack of before, for a moment + const atLogin = await defsAtLogin("Pat", people.Pat, fetchedEarlier); + assert.deepStrictEqual(atLogin.clothing.filter(row => row.id === String(louHat.id)).map(row => row.name), ["Lou Hat"]); + await new Promise(resolve => setTimeout(resolve, REMAKE_MS + 200)); + assert.strictEqual(byId(await pack(1000), louHat.id).titleStrId, String(9100000 + louHat.id - 50000)); + assert.strictEqual((await strings())[9100000 + louHat.id - 50000], "Lou Hat"); + lou.client.socket.destroy(); +}; + (async () => { const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "aj-test-")); try { await harness.withServer("items test", HTTP_PORT, GAME_PORT, first, {dataDir}); await harness.withServer("items test (after a restart)", HTTP_PORT, GAME_PORT, second, {dataDir}); + process.env.AJ_PACK_REMAKE_MS = String(REMAKE_MS); + await harness.withServer("items test (packs made again at most so often)", HTTP_PORT, GAME_PORT, third, {dataDir}); } finally { fs.rmSync(dataDir, {recursive: true, force: true, maxRetries: 10, retryDelay: 200}); From a77a1bcfb41ead7f6eb2845d33847cbbb428e273 Mon Sep 17 00:00:00 2001 From: Burrowdev Date: Tue, 20 Oct 2026 15:25:12 +0300 Subject: [PATCH 03/11] fix(pets): name a hatched egg without throwing "pn" (EggPetHatchedPopup) saved `account`, which that handler never defined since aa455f5: a ReferenceError after the name was set in memory, so no reply went out and the game's loading spiral never ended, and the name was not saved, so the popup came back at every login. "account is not defined" at lib/pets.js in prod's log several times an hour since 10-17. qol.js names the egg it buys and waits for the answer. Co-Authored-By: Claude Opus 5.5 --- lib/pets.js | 3 ++- test/qol.js | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pets.js b/lib/pets.js index 113e0a4..7e75fea 100644 --- a/lib/pets.js +++ b/lib/pets.js @@ -215,7 +215,8 @@ module.exports = ({others, findOnline, petFields}) => { // EggPetHatchedPopup: [petInvIdx, nameId1, nameId2] names a pet hatched // from an egg. PetManager.onPetEggNameResponse: [success, invIdx, name]. pn(session, roomId, [invIdx, name1, name2]) { - const pet = ownPet(session.account, invIdx); + const {account} = session; + const pet = ownPet(account, invIdx); const def = pet && defpacks.petDef(pet.lBits & 0xFF); if (!pet || ![name1, name2].every(isUint) || pet.name || !def || def.isEgg !== "1" || pet.createdAt + SECONDS_TO_HATCH > Math.floor(Date.now() / 1000)) { diff --git a/test/qol.js b/test/qol.js index 1eba684..e79a7c5 100644 --- a/test/qol.js +++ b/test/qol.js @@ -180,6 +180,9 @@ const run = async () => { const made = await a.untilXt("pc"); assert.strictEqual(made[2], "1"); assert.ok(Number(made[10]) <= Date.now() / 1000 - 259200); + // ... and named from the hatched popup (EggPetHatchedPopup waits for this answer) + a.xt("pn", roomId, made[5], 15386, 15428); + assert.deepStrictEqual((await a.untilXt("pn")).slice(2), ["1", made[5], "15386|15428"]); // --- she starts in the township as ever, in a look picked at random if she asked for one say(a, roomId, "!theme random on"); From 2344270291f0bcc06fe269fe27deec09ba3d25e1 Mon Sep 17 00:00:00 2001 From: Burrowdev Date: Tue, 20 Oct 2026 15:25:27 +0300 Subject: [PATCH 04/11] feat(music): staff upload MP3s as den music tracks A "Music" page in both panels: an MP3 (MPEG layer III frames, 4 KB to 15 MB) becomes a den music track at once, with no restart. - lib/custommusic.js: a custom den item made from "Jam On" (614) with a new `audio` column, the file's version. Its abbrName is CustomMusic_, which the content handler answers from data/custom-music/-.mp3, so a new upload is a new name and no game plays an old copy from its cache. - games get it by "xdi" with no client change; players from the item finder, or the shop when staff tick "on sale". - deleting it takes it from every inventory and removes the file. - /mod/api/music and /admin/api/music: list, play, upload (name, cost, inShop in the query), delete; 30 uploads an hour per staff member; "music uploaded"/"music deleted" in the mod log. - new permission music.upload, on the Admin role by default; a role list the admin has changed needs it ticked. The panels' CSP allows media-src 'self' for the preview players. Co-Authored-By: Claude Opus 5.5 --- lib/customitems.js | 30 +++++++- lib/custommusic.js | 174 +++++++++++++++++++++++++++++++++++++++++++++ lib/http.js | 84 +++++++++++++++++++++- lib/modpanel.js | 2 +- lib/panel.js | 2 +- lib/roles.js | 3 +- public/admin.html | 114 +++++++++++++++++++++++++++++ public/mod.html | 114 +++++++++++++++++++++++++++++ server.js | 2 + 9 files changed, 517 insertions(+), 8 deletions(-) create mode 100644 lib/custommusic.js diff --git a/lib/customitems.js b/lib/customitems.js index 1290c2f..5f148b3 100644 --- a/lib/customitems.js +++ b/lib/customitems.js @@ -65,6 +65,11 @@ const now = () => Math.floor(Date.now() / 1000); const baseDef = (kind, id) => (kind === "den" ? defpacks.denItemDef(id) : defpacks.itemDef(id)); +// the file name an uploaded music track is asked for by (lib/custommusic.js): +// a new upload is a new name, so no game plays an old copy from its cache. +const audioName = entry => `CustomMusic${entry.id}_${entry.audio}`; +let audioGone = () => {}; // (entry): an uploaded track's item was deleted; set by lib/custommusic.js + // the def row of an entry: its base's row with its own id, name, price and // colours, always on sale and marked new. const rowOf = entry => { @@ -80,6 +85,11 @@ const rowOf = entry => { combinedCost: "", }; if (entry.kind === "den") { + // a music track uploaded by staff (lib/custommusic.js): the game plays + // "audio/.mp3", which the server answers with its file + if (entry.audio) { + own.abbrName = audioName(entry); + } return {...base, ...own, nameStrId: String(STRING_BASE.den + entry.id - FIRST_ID), cost: String(entry.cost)}; } return {...base, ...own, titleStrId: String(STRING_BASE.clothing + entry.id - FIRST_ID), value: String(entry.cost), colors: entry.colors.join(","), isGiftable: "1"}; @@ -155,6 +165,7 @@ const fromRow = row => ({ artMode: LOOKS.includes(row.art_mode) ? row.art_mode : "sticker", pending: row.pending === 1, artPending: row.art_pending === 1, + audio: /^[0-9a-f]{8,64}$/.test(row.audio || "") ? row.audio : null, }); // a player-made item (lib/playeritems.js) has the maker's account id; @@ -179,6 +190,10 @@ const addOwnerColumn = () => { if (!columns.includes("art_pending")) { db().exec("ALTER TABLE custom_items ADD COLUMN art_pending INTEGER"); } + // a music track uploaded by staff (lib/custommusic.js): its file's version + if (!columns.includes("audio")) { + db().exec("ALTER TABLE custom_items ADD COLUMN audio TEXT"); + } }; const load = () => { @@ -269,10 +284,12 @@ const create = (kind, fields, by, ownerId = null, pending = false) => { return values; } const id = Math.max(FIRST_ID - 1, ...items.get(kind).keys()) + 1; + // `fields.audio`: an uploaded music track's version (lib/custommusic.js) + const audio = kind === "den" && /^[0-9a-f]{8,64}$/.test(String(fields.audio || "")) ? String(fields.audio) : null; const entry = {kind, id, base: baseId, colors: [], ...values, createdBy: String(by), createdAt: now(), ownerId, artMode: "sticker", pending, artPending: false, - seq: made++}; - db().prepare("INSERT INTO custom_items (kind, id, base, name, cost, members, colors, in_shop, created_by, created_at, owner_id, pending) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") - .run(kind, id, baseId, entry.name, entry.cost, entry.members ? 1 : 0, entry.colors.join(","), entry.inShop ? 1 : 0, entry.createdBy, entry.createdAt, ownerId, pending ? 1 : 0); + audio, seq: made++}; + db().prepare("INSERT INTO custom_items (kind, id, base, name, cost, members, colors, in_shop, created_by, created_at, owner_id, pending, audio) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + .run(kind, id, baseId, entry.name, entry.cost, entry.members ? 1 : 0, entry.colors.join(","), entry.inShop ? 1 : 0, entry.createdBy, entry.createdAt, ownerId, pending ? 1 : 0, audio); items.get(kind).set(id, entry); if (!pending) { apply(entry); @@ -362,6 +379,9 @@ const remove = (kind, id) => { db().prepare("DELETE FROM custom_items WHERE kind = ? AND id = ?").run(kind, entry.id); items.get(kind).delete(entry.id); defpacks.unsetCustomItem(kind, entry.id); + if (entry.audio) { + audioGone(entry); + } return {item: entry}; }; @@ -371,6 +391,10 @@ const list = () => KINDS.flatMap(kind => [...items.get(kind).values()]) module.exports = { KINDS, FIRST_ID, + audioName, + onAudioGone: callback => { + audioGone = callback; + }, load, create, update, diff --git a/lib/custommusic.js b/lib/custommusic.js new file mode 100644 index 0000000..69bf1c2 --- /dev/null +++ b/lib/custommusic.js @@ -0,0 +1,174 @@ +"use strict"; + +const crypto = require("crypto"); +const fs = require("fs"); +const path = require("path"); +const {logText} = require("./logtext.js"); +const cdn = require("./cdn.js"); +const config = require("../config.js"); +const customItems = require("./customitems.js"); +const extraMusic = require("./extramusic.js"); + +// music tracks staff upload in the panels (permission "music.upload", +// lib/roles.js): an MP3 made a den music item while the server runs, as the +// extra tracks of lib/extramusic.js are. each is a custom den item +// (lib/customitems.js) made from the game's "Jam On", with an `audio` +// version of its own: games are sent its def row at once, players get it +// from the item finder (or the shop, if staff put it on sale), and deleting +// it takes it from everyone, as with any custom item. +// +// a den music item plays "audio/.mp3" from the content server +// (DenItemHolder); for these the server answers with the file uploaded, kept +// in data/custom-music/-.mp3 (the data folder is the one +// place the server writes to). + +const MAX_BYTES = 15 * 1024 * 1024; +const MIN_BYTES = 4 * 1024; +const VERSION_LENGTH = 12; +const FOLDER = () => path.join(config.dataDir, "custom-music"); +const fileOf = entry => path.join(FOLDER(), `${entry.id}-${entry.audio}.mp3`); + +// MPEG audio layer III, as the game's Flash plays it: an ID3v2 tag may come +// first; then a frame header whose next frame is where its length says. +const BITRATES = { + 1: [0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320], + 2: [0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160], +}; +const SAMPLE_RATES = {3: [44100, 48000, 32000], 2: [22050, 24000, 16000], 0: [11025, 12000, 8000]}; + +// the frame at `at`: {length, sampleRate} or null. +const frameAt = (data, at) => { + if (at + 4 > data.length || data[at] !== 0xFF || (data[at + 1] & 0xE0) !== 0xE0) { + return null; + } + const version = (data[at + 1] >> 3) & 3; // 3 MPEG-1, 2 MPEG-2, 0 MPEG-2.5 + const layer = (data[at + 1] >> 1) & 3; // 1 is layer III + const bitrateIndex = data[at + 2] >> 4; + const rateIndex = (data[at + 2] >> 2) & 3; + const padding = (data[at + 2] >> 1) & 1; + if (version === 1 || layer !== 1 || bitrateIndex === 0 || bitrateIndex === 15 || rateIndex === 3) { + return null; + } + const bitrate = BITRATES[version === 3 ? 1 : 2][bitrateIndex] * 1000; + const sampleRate = SAMPLE_RATES[version][rateIndex]; + return {length: Math.floor(((version === 3 ? 144 : 72) * bitrate) / sampleRate) + padding, sampleRate}; +}; + +// null for an MP3 the game can play, or what is wrong with it. +const problem = data => { + if (!Buffer.isBuffer(data) || data.length < MIN_BYTES) { + return "That file is too small to be a song."; + } + if (data.length > MAX_BYTES) { + return `That file is too big: ${MAX_BYTES / 1024 / 1024} MB at most.`; + } + let at = 0; + if (data.toString("latin1", 0, 3) === "ID3" && data.length > 10) { + at = 10 + (((data[6] & 0x7F) << 21) | ((data[7] & 0x7F) << 14) | ((data[8] & 0x7F) << 7) | (data[9] & 0x7F)) + (data[5] & 0x10 ? 10 : 0); + } + // the first frame: the header, then two more where their lengths say + for (const end = Math.min(data.length, at + 64 * 1024); at < end; at++) { + const first = frameAt(data, at); + const second = first && frameAt(data, at + first.length); + const third = second && frameAt(data, at + first.length + second.length); + if (third) { + return null; + } + } + return "That is not an MP3 file (MPEG audio layer III)."; +}; + +const tracks = () => customItems.list().filter(entry => entry.kind === "den" && entry.audio); + +// content path ("1830/audio/") -> entry, made again when the tracks change. +let paths = null; +const pathMap = () => { + if (!paths) { + paths = new Map(tracks().map(entry => [cdn.hashedPath(`audio/${customItems.audioName(entry)}.mp3`), entry])); + } + return paths; +}; + +// what to answer a content request with when it is for one of these +// tracks: {body, type}; undefined when it is not. +const asset = async relPath => { + const entry = pathMap().get(relPath); + if (!entry) { + return undefined; + } + const body = await fs.promises.readFile(fileOf(entry)).catch(() => null); + return {body, type: "audio/mpeg"}; +}; + +// an upload made a track: {item} or {error}. `fields`: {name, cost, inShop, members}. +const add = (data, fields, by) => { + const wrong = problem(data); + if (wrong) { + return {error: wrong}; + } + const audio = crypto.createHash("sha256").update(data).digest("hex").slice(0, VERSION_LENGTH); + fs.mkdirSync(FOLDER(), {recursive: true, mode: 0o700}); + // written before the item exists, so no game asks for a file not there yet + const waiting = path.join(FOLDER(), `new-${audio}-${process.pid}.mp3`); + fs.writeFileSync(waiting, data, {mode: 0o600}); + const result = customItems.create("den", {name: fields.name, cost: fields.cost, members: fields.members, + inShop: fields.inShop === true, base: extraMusic.TEMPLATE, audio}, by); + if (result.error) { + fs.rmSync(waiting, {force: true}); + return result; + } + fs.renameSync(waiting, fileOf(result.item)); + paths = null; + console.log(`[custom music] ${logText(by)} uploaded "${logText(result.item.name)}" (den item ${logText(result.item.id)}, ${logText(data.length)} bytes)`); + return result; +}; + +// the file of a track, for staff to listen to before or after: {data} or null. +const file = id => { + const entry = tracks().find(track => track.id === Number(id)); + if (!entry) { + return null; + } + const data = (() => { + try { + return fs.readFileSync(fileOf(entry)); + } + catch (err) { + return null; + } + })(); + return data ? {data, entry} : null; +}; + +// for the panels: newest first +const list = () => tracks().map(entry => ({id: entry.id, name: entry.name, cost: entry.cost, inShop: entry.inShop, + createdBy: entry.createdBy, createdAt: entry.createdAt, + bytes: (() => { + try { + return fs.statSync(fileOf(entry)).size; + } + catch (err) { + return 0; + } + })()})); + +// a track's item was deleted (lib/itemcontrol.js deleteCustom): its file goes too. +customItems.onAudioGone(entry => { + fs.rmSync(fileOf(entry), {force: true}); + paths = null; +}); + +// at start: files of tracks no longer there (and of uploads cut short) go. +const load = () => { + const kept = new Set(tracks().map(entry => path.basename(fileOf(entry)))); + let gone = 0; + for (const name of fs.existsSync(FOLDER()) ? fs.readdirSync(FOLDER()) : []) { + if (!kept.has(name)) { + fs.rmSync(path.join(FOLDER(), name), {force: true}); + gone++; + } + } + console.log(`[custom music] ${logText(kept.size)} uploaded tracks${gone ? `, ${logText(gone)} stray files removed` : ""}`); +}; + +module.exports = {MAX_BYTES, problem, asset, add, file, list, load}; diff --git a/lib/http.js b/lib/http.js index 7a2e47b..b90c408 100644 --- a/lib/http.js +++ b/lib/http.js @@ -24,6 +24,8 @@ const manage = require("./manage.js"); const modpanel = require("./modpanel.js"); const badges = require("./badges.js"); const extraMusic = require("./extramusic.js"); +const customMusic = require("./custommusic.js"); +const itemControl = require("./itemcontrol.js"); const masterpieces = require("./masterpieces.js"); const panel = require("./panel.js"); const totp = require("./totp.js"); @@ -230,6 +232,75 @@ const itemArtUpload = async (req, res, url, from, signedIn = null, adminSignedIn send(res, 200, result); }; +// music tracks uploaded in the panels (lib/custommusic.js), at +// /mod/api/music (staff with "music.upload") and /admin/api/music: +// GET: the tracks; GET ?play=: one's file, to listen to; +// POST ?name=&cost=&inShop=1, the MP3 as the body: a new track; +// POST ?delete=: the track deleted, and taken from every player. +// `signedIn` (req) -> account: the moderator panel's session; +// `adminSignedIn` (req) -> boolean: the admin panel's, written down as "admin". +const musicUploads = limits.limiter(30, 3600000); // tracks an hour, per staff member +const musicRoute = async (req, res, url, from, signedIn = null, adminSignedIn = null) => { + res.setHeader("Cache-Control", "no-store"); + const asAdmin = Boolean(adminSignedIn && adminSignedIn(req)); + const account = asAdmin ? null : signedIn ? signedIn(req) : null; + if (!asAdmin && !(account && roles.can(account, "music.upload"))) { + send(res, 403, {error: "Your role cannot upload music."}); + return; + } + const by = asAdmin ? "admin" : account.username; + if (req.method === "GET" || req.method === "HEAD") { + if (url.searchParams.has("play")) { + const track = customMusic.file(url.searchParams.get("play")); + if (!track) { + send(res, 404, "Not found", "text/plain"); + return; + } + res.setHeader("Content-Security-Policy", "default-src 'none'; sandbox"); + res.setHeader("X-Content-Type-Options", "nosniff"); + send(res, 200, track.data, "audio/mpeg"); + return; + } + send(res, 200, {tracks: customMusic.list(), maxBytes: customMusic.MAX_BYTES}); + return; + } + if (req.method !== "POST" || !limits.originAllowed(req)) { + send(res, 403, {error: "forbidden"}); + return; + } + if (url.searchParams.has("delete")) { + const entry = customItems.get("den", url.searchParams.get("delete")); + if (!entry || !entry.audio) { + send(res, 404, {error: "No such uploaded track."}); + return; + } + const result = itemControl.deleteCustom("den", entry.id, world); + if (result.ok) { + modlog.record(by, "music deleted", "", `den ${entry.id} ${entry.name}`); + } + send(res, result.ok ? 200 : 400, result); + return; + } + if (!asAdmin && !musicUploads.allow(account.id)) { + send(res, 429, {error: "Too many uploads. Try again later."}); + return; + } + const data = await readBody(req, customMusic.MAX_BYTES + 1).catch(() => null); + if (!data) { + send(res, 413, {error: `That file is too big: ${customMusic.MAX_BYTES / 1024 / 1024} MB at most.`}); + return; + } + const result = customMusic.add(data, {name: url.searchParams.get("name"), cost: url.searchParams.get("cost") || 0, + inShop: url.searchParams.get("inShop") === "1"}, by); + if (result.error) { + send(res, 400, result); + return; + } + modlog.record(by, "music uploaded", "", `den ${result.item.id} ${result.item.name}`); + send(res, 200, {ok: `"${result.item.name}" is den item ${result.item.id}. Players can get it from the item finder now${result.item.inShop ? ", and the shop" : ""}.`, + item: {id: result.item.id, name: result.item.name}}); +}; + // a picture still waiting for a moderator, which /itemart/ does not serve: // to its maker (the account page) and to staff who review or make items. const itemArtView = (req, res, url, signedIn, adminSignedIn) => { @@ -731,6 +802,10 @@ const handle = async (req, res) => { await itemArtUpload(req, res, url, from, null, panel.signedIn); return; } + if (url.pathname === "/admin/api/music") { + await musicRoute(req, res, url, from, null, panel.signedIn); + return; + } await panel.handle(req, res, url, from, {send, readJson}); return; } @@ -757,6 +832,10 @@ const handle = async (req, res) => { await itemArtUpload(req, res, url, from, modpanel.signedInAs); return; } + if (isMod && url.pathname === "/mod/api/music") { + await musicRoute(req, res, url, from, modpanel.signedInAs); + return; + } if (isMod) { await modpanel.handle(req, res, url, from, {send, readJson}); return; @@ -1043,8 +1122,9 @@ const handle = async (req, res) => { send(res, body ? 200 : 404, body || "Not found", body ? MIME[".swf"] : "text/plain"); return; } - // the extra den music tracks, which the CDN does not have as files. - const extra = await extraMusic.asset(relPath); + // the extra den music tracks, which the CDN does not have as files, and + // those uploaded by staff. + const extra = await extraMusic.asset(relPath) || await customMusic.asset(relPath); if (extra) { send(res, extra.body ? 200 : 404, extra.body || "Not found", extra.body ? extra.type : "text/plain"); return; diff --git a/lib/modpanel.js b/lib/modpanel.js index 04d46d7..ffd9ea8 100644 --- a/lib/modpanel.js +++ b/lib/modpanel.js @@ -395,7 +395,7 @@ const PAGE_HEADERS = { "X-Frame-Options": "DENY", "X-Content-Type-Options": "nosniff", "Referrer-Policy": "no-referrer", - "Content-Security-Policy": captcha.policy("default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; font-src 'self'; " + "Content-Security-Policy": captcha.policy("default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; media-src 'self'; font-src 'self'; " + "form-action 'none'; frame-ancestors 'none'; base-uri 'none'"), }; diff --git a/lib/panel.js b/lib/panel.js index 21a783f..f476813 100644 --- a/lib/panel.js +++ b/lib/panel.js @@ -600,7 +600,7 @@ const PAGE_HEADERS = { "X-Frame-Options": "DENY", "X-Content-Type-Options": "nosniff", "Referrer-Policy": "no-referrer", - "Content-Security-Policy": "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; font-src 'self'; " + "Content-Security-Policy": "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; media-src 'self'; font-src 'self'; " + "form-action 'none'; frame-ancestors 'none'; base-uri 'none'", }; diff --git a/lib/roles.js b/lib/roles.js index 74b37a7..87045f7 100644 --- a/lib/roles.js +++ b/lib/roles.js @@ -32,11 +32,12 @@ const PERMISSIONS = { "items.create": "make new items from the game's art, change and delete them, lock items out of the item finder and take items from players (!item, !lock, !unlock, !locked, !take)", "room.effects": "run temporary room-wide size and colour effects (!roomfx)", "theater.control": "play or stop a YouTube video for everyone in the theatre", + "music.upload": "upload MP3s as den music tracks, and delete those uploaded (the Music page)", }; const HELPER = ["server.view", "player.view", "staff.list", "player.warn", "player.goto", "chat.silence"]; const MODERATOR = [...HELPER, "player.kick", "player.summon", "staff.invisible", "player.ban", "player.names", "address.ban", "items.approve", "items.restrict", "theater.control"]; -const ADMIN = [...MODERATOR, "items.create", "room.effects"]; +const ADMIN = [...MODERATOR, "items.create", "room.effects", "music.upload"]; // a developer makes things, and moderates nobody. const DEVELOPER = ["server.view", "staff.list", "items.create", "room.effects", "theater.control"]; diff --git a/public/admin.html b/public/admin.html index b337ab4..714ce8b 100644 --- a/public/admin.html +++ b/public/admin.html @@ -413,6 +413,7 @@ + @@ -735,6 +736,31 @@ + + +
+

Lost access

+

A new random password, shown once. They are signed out everywhere and should change it on their account page.

+
+
+ +

Adventure level

@@ -1529,6 +1538,7 @@ label.append(box, ` ${badge.label}`); return label; })); + $("temp-two-step-off").disabled = !current || !current.twoStep; $("guide-toggle").textContent = current && current.guide ? "Remove guide" : "Make guide"; $("discord-name").textContent = current && current.discord ? `Discord: ${current.discord}` : "No Discord linked"; $("admin-dashboard-access").textContent = current && current.adminDashboardAccess ? "Remove Discord dashboard access" : "Allow Discord dashboard access"; @@ -1566,7 +1576,8 @@ selected = username; current = null; setStatus($("detail-status"), ""); - for (const id of ["new-name", "ban-hours", "silence-hours", "message-text", "gems-amount", "gift-value", "gift-text"]) { + $("temp-password-row").hidden = true; + for (const id of ["temp-password", "new-name", "ban-hours", "silence-hours", "message-text", "gems-amount", "gift-value", "gift-text"]) { $(id).value = ""; } hashArg = username; @@ -1720,6 +1731,7 @@ case "permissions": return permissionChoices(); case "adminDashboardAccess": return {on: !(current || {}).adminDashboardAccess}; case "guide": return {on: !(current || {}).guide}; + case "tempPassword": return {twoStepOff: $("temp-two-step-off").checked}; case "message": return {text: $("message-text").value}; case "gems": return {amount: numberFrom("gems-amount")}; case "gift": return giftBody(); @@ -1731,6 +1743,10 @@ } }; + $("temp-password-copy").addEventListener("click", () => { + navigator.clipboard.writeText($("temp-password").value).catch(() => $("temp-password").select()); + }); + // the "N ago" buttons fill the last-seen box and press Set. $("detail").addEventListener("click", event => { const quick = event.target.closest("button[data-ago]"); @@ -1760,6 +1776,9 @@ if (action === "banDevices" && !window.confirm(`Hardware ban ${selected}? Everyone on their hardware is put out.`)) { return; } + if (action === "tempPassword" && !window.confirm(`Give ${selected} a new random password? Their old one stops working and they are signed out everywhere.`)) { + return; + } button.disabled = true; const result = await api("account", {username: selected, action, ...actionBody(action)}); button.disabled = false; @@ -1767,6 +1786,11 @@ if (result.ok && action === "delete") { selected = null; } + if (result.ok && action === "tempPassword") { + $("temp-password").value = result.data.password; + $("temp-password-row").hidden = false; + $("temp-two-step-off").checked = false; + } if (result.ok && action === "rename") { selected = result.data.username; $("new-name").value = ""; From 8368692b99f7174c63b5231b70063b0451f0ed26 Mon Sep 17 00:00:00 2001 From: Burrowdev Date: Tue, 20 Oct 2026 23:44:16 +0300 Subject: [PATCH 10/11] feat(modpanel): the mod log as its own page The admin panel's "What they did" table, in the moderator panel too: the latest 300 entries. A new permission, staff.log, which moderators and admins have by default, guards the page and its route. Co-Authored-By: Claude Opus 5.5 --- lib/modpanel.js | 11 +++++++++++ lib/roles.js | 3 ++- public/mod.html | 25 +++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/lib/modpanel.js b/lib/modpanel.js index ffd9ea8..fd6bb26 100644 --- a/lib/modpanel.js +++ b/lib/modpanel.js @@ -670,6 +670,17 @@ const handle = async (req, res, url, from, web) => { return; } + // the mod log (lib/modlog.js): what staff did, as the admin panel's + // Moderators page shows it. + if (route === "GET /mod/api/modlog") { + if (!roles.can(mod, "staff.log")) { + reply(403, {error: "Your role does not allow that."}); + return; + } + reply(200, {log: modlog.recent()}); + return; + } + // everything else shows players and what they did: not for a role that // only makes things. if (!roles.can(mod, "player.view")) { diff --git a/lib/roles.js b/lib/roles.js index 87045f7..ac33c27 100644 --- a/lib/roles.js +++ b/lib/roles.js @@ -33,10 +33,11 @@ const PERMISSIONS = { "room.effects": "run temporary room-wide size and colour effects (!roomfx)", "theater.control": "play or stop a YouTube video for everyone in the theatre", "music.upload": "upload MP3s as den music tracks, and delete those uploaded (the Music page)", + "staff.log": "read the mod log: what staff did, in the panels and the game (the Mod log page)", }; const HELPER = ["server.view", "player.view", "staff.list", "player.warn", "player.goto", "chat.silence"]; -const MODERATOR = [...HELPER, "player.kick", "player.summon", "staff.invisible", "player.ban", "player.names", "address.ban", "items.approve", "items.restrict", "theater.control"]; +const MODERATOR = [...HELPER, "player.kick", "player.summon", "staff.invisible", "player.ban", "player.names", "address.ban", "items.approve", "items.restrict", "theater.control", "staff.log"]; const ADMIN = [...MODERATOR, "items.create", "room.effects", "music.upload"]; // a developer makes things, and moderates nobody. const DEVELOPER = ["server.view", "staff.list", "items.create", "room.effects", "theater.control"]; diff --git a/public/mod.html b/public/mod.html index b474f0f..ef93b98 100644 --- a/public/mod.html +++ b/public/mod.html @@ -475,6 +475,7 @@ + @@ -992,6 +993,17 @@
+ +