diff --git a/config.js b/config.js index 882f37b..f957977 100644 --- a/config.js +++ b/config.js @@ -327,6 +327,9 @@ module.exports = { cacheMissesPerTenMinutes: 1500, // a first visit with an empty cache fetches several hundred files upstreamTimeoutMs: 20000, maxCacheBytes: Number(env.AJ_MAX_CACHE_MB || 8192) * 1024 * 1024, + // how often at most the packs with custom items in are made again after + // one changes (lib/defpacks.js clientPack); the tests use 0, made at once. + packRemakeMs: env.AJ_PACK_REMAKE_MS === undefined ? 15000 : Number(env.AJ_PACK_REMAKE_MS), maxConnections: 2000, maxHttpConnections: 2000, // open at once, in all maxHttpConnectionsPerAddress: 40, // the game opens a good many at once while it loads diff --git a/lib/amf3.js b/lib/amf3.js index 13b0cab..4c1c412 100644 --- a/lib/amf3.js +++ b/lib/amf3.js @@ -177,40 +177,89 @@ class Amf3Reader { // writer for what the reader above produces from a defPack: plain objects, // arrays (with or without string keys), strings, numbers, booleans and null. // nothing is written as a reference except repeated strings. +// written into one buffer that grows: a pack is megabytes of small values, +// and a Buffer for each of them took seconds for the item pack. class Amf3Writer { constructor() { - this.parts = []; + this.buf = Buffer.allocUnsafe(1 << 16); + this.pos = 0; this.strings = new Map(); } + room(bytes) { + if (this.pos + bytes > this.buf.length) { + let size = this.buf.length * 2; + while (size < this.pos + bytes) { + size *= 2; + } + const bigger = Buffer.allocUnsafe(size); + this.buf.copy(bigger, 0, 0, this.pos); + this.buf = bigger; + } + } + + byte(value) { + this.room(1); + this.buf[this.pos++] = value; + } + u29(value) { + this.room(4); + const {buf} = this; if (value < 0x80) { - this.parts.push(Buffer.from([value])); + buf[this.pos++] = value; } else if (value < 0x4000) { - this.parts.push(Buffer.from([(value >> 7) | 0x80, value & 0x7f])); + buf[this.pos++] = (value >> 7) | 0x80; + buf[this.pos++] = value & 0x7f; } else if (value < 0x200000) { - this.parts.push(Buffer.from([(value >> 14) | 0x80, ((value >> 7) & 0x7f) | 0x80, value & 0x7f])); + buf[this.pos++] = (value >> 14) | 0x80; + buf[this.pos++] = ((value >> 7) & 0x7f) | 0x80; + buf[this.pos++] = value & 0x7f; } else { - this.parts.push(Buffer.from([(value >> 22) | 0x80, ((value >> 15) & 0x7f) | 0x80, ((value >> 8) & 0x7f) | 0x80, value & 0xff])); + buf[this.pos++] = (value >> 22) | 0x80; + buf[this.pos++] = ((value >> 15) & 0x7f) | 0x80; + buf[this.pos++] = ((value >> 8) & 0x7f) | 0x80; + buf[this.pos++] = value & 0xff; } } string(text) { if (text === "") { this.u29(1); + return; } - else if (this.strings.has(text)) { - this.u29(this.strings.get(text) << 1); + const known = this.strings.get(text); + if (known !== undefined) { + this.u29(known << 1); + return; } - else { - this.strings.set(text, this.strings.size); - const bytes = Buffer.from(text, "utf8"); - this.u29((bytes.length << 1) | 1); - this.parts.push(bytes); + this.strings.set(text, this.strings.size); + // nearly every string of a pack is short and ASCII: its bytes are its + // characters, copied here rather than through Buffer's utf8 writer. + if (text.length < 64) { + let ascii = true; + for (let i = 0; i < text.length; i++) { + if (text.charCodeAt(i) > 0x7f) { + ascii = false; + break; + } + } + if (ascii) { + this.u29((text.length << 1) | 1); + this.room(text.length); + for (let i = 0; i < text.length; i++) { + this.buf[this.pos++] = text.charCodeAt(i); + } + return; + } } + const length = Buffer.byteLength(text, "utf8"); + this.u29((length << 1) | 1); + this.room(length); + this.pos += this.buf.write(text, this.pos, "utf8"); } pairs(entries) { @@ -222,45 +271,55 @@ class Amf3Writer { } value(value) { - if (value === undefined) { - this.parts.push(Buffer.from([0x00])); + // most values of a pack are strings: asked first + if (typeof value === "string") { + this.byte(0x06); + this.string(value); + } + else if (value === undefined) { + this.byte(0x00); } else if (value === null) { - this.parts.push(Buffer.from([0x01])); + this.byte(0x01); } else if (typeof value === "boolean") { - this.parts.push(Buffer.from([value ? 0x03 : 0x02])); + this.byte(value ? 0x03 : 0x02); } else if (typeof value === "number") { if (Number.isInteger(value) && value >= -0x10000000 && value < 0x10000000) { - this.parts.push(Buffer.from([0x04])); + this.byte(0x04); this.u29(value & 0x1fffffff); } else { - const bytes = Buffer.alloc(9); - bytes[0] = 0x05; - bytes.writeDoubleBE(value, 1); - this.parts.push(bytes); + this.byte(0x05); + this.room(8); + this.pos = this.buf.writeDoubleBE(value, this.pos); } } - else if (typeof value === "string") { - this.parts.push(Buffer.from([0x06])); - this.string(value); - } else if (Array.isArray(value)) { - this.parts.push(Buffer.from([0x09])); + this.byte(0x09); this.u29((value.length << 1) | 1); - // string keys set on the array beside its numbered elements. - this.pairs(Object.entries(value).filter(([key]) => !/^\d+$/.test(key))); + // string keys set on the array beside its numbered elements (an array + // with as many keys as elements has none). + const keys = Object.keys(value); + this.pairs(keys.length === value.length ? [] : keys.filter(key => !/^\d+$/.test(key)).map(key => [key, value[key]])); for (const element of value) { this.value(element); } } else if (typeof value === "object" && !Buffer.isBuffer(value)) { // A dynamic object of no class, with its traits written in place. - this.parts.push(Buffer.from([0x0a, 0x0b])); + this.byte(0x0a); + this.byte(0x0b); + this.string(""); + // its own keys in Object.keys order, without an array of entries for each object + for (const key in value) { + if (Object.hasOwn(value, key)) { + this.string(key); + this.value(value[key]); + } + } this.string(""); - this.pairs(Object.entries(value)); } else { throw new Error("Cannot write this value as AMF3"); @@ -273,6 +332,6 @@ module.exports = { encode: value => { const writer = new Amf3Writer(); writer.value(value); - return Buffer.concat(writer.parts); + return writer.buf.subarray(0, writer.pos); }, }; diff --git a/lib/customitems.js b/lib/customitems.js index 8c56128..1290c2f 100644 --- a/lib/customitems.js +++ b/lib/customitems.js @@ -53,11 +53,11 @@ const SCHEMA = `CREATE TABLE IF NOT EXISTS custom_items ( const items = new Map(KINDS.map(kind => [kind, new Map()])); // kind -> id -> entry let made = 0; // entries in the order they were made (`seq`), for lists -let changes = 0; // goes up with every change a game must be told of; an entry's `changed` is the count at its last let broadcast = () => {}; // (cmd, ...params) to every game playing; set by lib/world.js -// address key -> {at, clothing, den, names}: `changes` when that address -// last fetched the packs with custom items in, the oldest within -// PACKS_SEEN_MS (players behind one address fetch at different times). +// address key -> {at, clothing, den, names}: the version (defpacks +// customVersion) of the packs with custom items in that address last +// fetched, the oldest within PACKS_SEEN_MS (players behind one address +// fetch at different times). an entry's `changed` is the version of its last change. const packsSeen = new Map(); const db = () => accounts.database(); @@ -87,7 +87,7 @@ const rowOf = entry => { const apply = entry => { defpacks.setCustomItem(entry.kind, rowOf(entry), entry.name, entry.inShop); - entry.changed = ++changes; + entry.changed = defpacks.customVersion(); }; // what "xdi" carries of an entry: the fields the client reads, its name and @@ -317,7 +317,7 @@ const setArtMode = (kind, id, mode) => { } entry.artMode = mode; db().prepare("UPDATE custom_items SET art_mode = ? WHERE kind = ? AND id = ?").run(mode, kind, entry.id); - entry.changed = ++changes; + entry.changed = defpacks.customChanged(); announce(entry); return {item: entry}; }; @@ -332,7 +332,7 @@ const setArt = (kind, id, art, pending = false) => { entry.art = art || null; entry.artPending = Boolean(art && pending); db().prepare("UPDATE custom_items SET art = ?, art_pending = ? WHERE kind = ? AND id = ?").run(entry.art, entry.artPending ? 1 : 0, kind, entry.id); - entry.changed = ++changes; + entry.changed = defpacks.customChanged(); announce(entry); }; @@ -400,20 +400,19 @@ module.exports = { const def = baseDef(entry.kind, entry.base); return def ? defpacks.string(entry.kind === "den" ? def.nameStrId : def.titleStrId) || String(def.abbrName || entry.base) : String(entry.base); }, - // an address fetched a pack with custom items in: "clothing", "den" or - // "names" (lib/defpacks.js customPackKind). called before the pack is made, - // so that a change while it is made counts as after. - sawPack(key, packKind) { + // an address was sent a pack with custom items in: "clothing", "den" or + // "names" (lib/defpacks.js customPackKind), made at `version`. + sawPack(key, packKind, version) { const time = Date.now(); const seen = packsSeen.get(key); if (seen && time - seen.at < PACKS_SEEN_MS) { if (seen[packKind] === undefined) { - seen[packKind] = changes; + seen[packKind] = version; } return; } packsSeen.delete(key); - packsSeen.set(key, {at: time, [packKind]: changes}); + packsSeen.set(key, {at: time, [packKind]: version}); // the oldest are first for (const [oldKey, old] of packsSeen) { if (packsSeen.size <= MAX_PACKS_SEEN && time - old.at < PACKS_SEEN_MS) { diff --git a/lib/defpacks.js b/lib/defpacks.js index a26c39f..4ff8cf6 100644 --- a/lib/defpacks.js +++ b/lib/defpacks.js @@ -370,26 +370,51 @@ const CLIENT_PACKS = new Map([ [DEF_NAMEBAR_BADGE, () => {}], ]); const clientPackPaths = new Map([...CLIENT_PACKS.keys()].map(id => [cdn.hashedPath(`defPacks/${id}`), id])); -const clientPackBodies = new Map(); // id -> {day, allOnSale, body} +const clientPackBodies = new Map(); // id -> {day, allOnSale, body, version, madeAt} +const clientPacksChanged = new Set(); // ids whose body is older than a custom item change +const clientPacksMaking = new Map(); // id -> the promise of the body being made +// a pack is made again after a custom item change at most every +// config.packRemakeMs: the item pack takes ~70 ms of the one thread at 5000 +// items (~300 ms at 50000), and players make items many times a minute. +// until then the body made before goes out, with the version it has; a +// game logging in is sent what changed after it (lib/customitems.js sendAll). +const deflateRaw = require("util").promisify(zlib.deflateRaw); // the packs custom items are in: their rows, and their names const CUSTOM_PACK_KINDS = new Map([[DEF_ITEM, "clothing"], [DEF_DEN_ITEM, "den"], [DEF_LOCALIZATION_EN, "names"]]); -// the pack at `relPath` as clients should get it, or null when it is not -// one of the above (and comes straight from the CDN). +const makeClientPack = async id => { + // the version before the rows are read: a change while it is made counts + // as after, and is sent at login (more than needed, never less) + const version = customVersion; + const pack = await loadChanged(id); + CLIENT_PACKS.get(id)(pack); + // deflated on libuv's threads, not this one + const body = await deflateRaw(amf3.encode(pack)); + const made = {day: sales.today(), allOnSale: sales.everythingOnSale(), body, version, madeAt: Date.now()}; + clientPackBodies.set(id, made); + if (customVersion === version) { + clientPacksChanged.delete(id); + } + return made; +}; + +// the pack at `relPath` as clients should get it, {body, version}, or null +// when it is not one of the above (and comes straight from the CDN). +// `version`: customVersion() of the custom items in it. const clientPack = async relPath => { const id = clientPackPaths.get(relPath); if (id === undefined) { return null; } const made = clientPackBodies.get(id); - if (made && made.day === sales.today() && made.allOnSale === sales.everythingOnSale()) { - return made.body; + if (made && made.day === sales.today() && made.allOnSale === sales.everythingOnSale() + && (!clientPacksChanged.has(id) || Date.now() - made.madeAt < config.packRemakeMs)) { + return made; } - const pack = await loadChanged(id); - CLIENT_PACKS.get(id)(pack); - const body = zlib.deflateRawSync(amf3.encode(pack)); - clientPackBodies.set(id, {day: sales.today(), allOnSale: sales.everythingOnSale(), body}); - return body; + if (!clientPacksMaking.has(id)) { + clientPacksMaking.set(id, makeClientPack(id).finally(() => clientPacksMaking.delete(id))); + } + return clientPacksMaking.get(id); }; // most packs are tables: an object of rows keyed by row index. @@ -537,8 +562,8 @@ const setCustomItem = (kind, row, name, inShop = true) => { clothingIds = clothingIds.includes(id) ? clothingIds : [...clothingIds, id]; } customInShop[kind][inShop ? "add" : "delete"](id); - clientPackBodies.delete(pack); - clientPackBodies.delete(DEF_LOCALIZATION_EN); + clientPacksChanged.add(pack); + clientPacksChanged.add(DEF_LOCALIZATION_EN); customVersion++; }; @@ -560,7 +585,7 @@ const unsetCustomItem = (kind, id) => { clothingIds = clothingIds.filter(one => one !== number); } customInShop[kind].delete(number); - clientPackBodies.delete(pack); + clientPacksChanged.add(pack); customVersion++; }; @@ -571,6 +596,9 @@ module.exports = { // the custom items on sale, the newest first. customShopIds: kind => [...customInShop[kind]].reverse(), customVersion: () => customVersion, + // a change of a custom item that is not in the packs (its picture): one + // more version, for a game logging in to be told of it + customChanged: () => ++customVersion, // the avatarUseFlags every clothing row has in the clients' pack (any animal) EVERY_AVATAR, // "clothing", "den" or "names" for a pack path with custom items in (lib/customitems.js sawPack), else null. diff --git a/lib/http.js b/lib/http.js index 5f7e0b3..7a2e47b 100644 --- a/lib/http.js +++ b/lib/http.js @@ -1049,14 +1049,14 @@ const handle = async (req, res) => { send(res, extra.body ? 200 : 404, extra.body || "Not found", extra.body ? extra.type : "text/plain"); return; } - // what custom items this game will have from its packs (customItems.sendAll) - const packKind = defpacks.customPackKind(relPath); - if (packKind) { - customItems.sawPack(from.key, packKind); - } const clientPack = await defpacks.clientPack(relPath); if (clientPack) { - send(res, 200, clientPack, "application/octet-stream"); + // what custom items this game has from its packs (customItems.sendAll) + const packKind = defpacks.customPackKind(relPath); + if (packKind) { + customItems.sawPack(from.key, packKind, clientPack.version); + } + send(res, 200, clientPack.body, "application/octet-stream"); return; } // A room file in a seasonal theme ("/roomDefs/theme_ { lou.client.socket.destroy(); }; +// the packs made again at most every AJ_PACK_REMAKE_MS after a change: one +// fetched before then is the one made before, and a game whose address +// fetched it is sent at login the item it lacks. +const REMAKE_MS = 5000; +const third = async () => { + const fetchedEarlier = "127.0.0.3"; + await packsFrom(fetchedEarlier); + const lou = await arrive("Lou", people.Lou); + lou.client.xt("xcm", lou.roomId, "make", "clothing", made.base.id, "Lou Hat"); + const answer = await lou.client.untilXt("xcm"); + assert.strictEqual(answer[2], "1", answer[3]); + const louHat = JSON.parse(answer[6]).find(entry => entry.name === "Lou Hat"); + assert.ok(!byId(await pack(1000), louHat.id)); // the pack of before, for a moment + const atLogin = await defsAtLogin("Pat", people.Pat, fetchedEarlier); + assert.deepStrictEqual(atLogin.clothing.filter(row => row.id === String(louHat.id)).map(row => row.name), ["Lou Hat"]); + await new Promise(resolve => setTimeout(resolve, REMAKE_MS + 200)); + assert.strictEqual(byId(await pack(1000), louHat.id).titleStrId, String(9100000 + louHat.id - 50000)); + assert.strictEqual((await strings())[9100000 + louHat.id - 50000], "Lou Hat"); + lou.client.socket.destroy(); +}; + (async () => { const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "aj-test-")); try { await harness.withServer("items test", HTTP_PORT, GAME_PORT, first, {dataDir}); await harness.withServer("items test (after a restart)", HTTP_PORT, GAME_PORT, second, {dataDir}); + process.env.AJ_PACK_REMAKE_MS = String(REMAKE_MS); + await harness.withServer("items test (packs made again at most so often)", HTTP_PORT, GAME_PORT, third, {dataDir}); } finally { fs.rmSync(dataDir, {recursive: true, force: true, maxRetries: 10, retryDelay: 200});