diff --git a/lib/accounts.js b/lib/accounts.js index 50d679d..fe501e2 100644 --- a/lib/accounts.js +++ b/lib/accounts.js @@ -78,7 +78,10 @@ const LOGIN_DAYS_KEPT = 90; // and none older than this: they hold addresses let db = null; const byKey = new Map(); // lowercased username -> account const byId = new Map(); -const savedState = new Map(); // account id -> state JSON as last written +// account id -> digest of the state JSON as last written: enough to tell a +// change, without a second copy of every account's state (~13 KB each). +const savedState = new Map(); +const stateDigest = json => crypto.createHash("sha256").update(json).digest("base64"); // A name somebody gave up (a rename, a deleted account) is nobody's to take // for a while: whoever had it is still known by it to the players they met. const NAME_HELD_DAYS = 30; @@ -331,7 +334,7 @@ const insert = (username, salt, hash, createdAt, state) => { const result = db.prepare("INSERT INTO accounts (username, username_key, password_salt, password_hash, created_at, state) VALUES (?, ?, ?, ?, ?, ?)") .run(username, username.toLowerCase(), salt, hash, createdAt, json); account.id = Number(result.lastInsertRowid); - savedState.set(account.id, json); + savedState.set(account.id, stateDigest(json)); remember(account); touch(account); return account; @@ -381,7 +384,7 @@ const load = () => { createdAt: row.created_at, ...JSON.parse(row.state), }); - savedState.set(account.id, row.state); + savedState.set(account.id, stateDigest(row.state)); remember(account); } importLegacyFile(); @@ -475,7 +478,7 @@ const release = account => { const changedState = (account, changed) => { const json = stateJson(account); - if (savedState.get(account.id) !== json) { + if (savedState.get(account.id) !== stateDigest(json)) { changed.push([account.id, json]); return true; } @@ -583,7 +586,7 @@ const flush = ({some = false} = {}) => { throw err; } for (const [id, json] of changed) { - savedState.set(id, json); + savedState.set(id, stateDigest(json)); } }; diff --git a/lib/customitems.js b/lib/customitems.js index c7b26fb..8c56128 100644 --- a/lib/customitems.js +++ b/lib/customitems.js @@ -3,14 +3,15 @@ const {logText} = require("./logtext.js"); const accounts = require("./accounts.js"); const defpacks = require("./defpacks.js"); +const limits = require("./limits.js"); // items the staff make while the server runs, with no restart of the server // or of anybody's game. each is an existing item's art (the clothing piece // or den item it is made from) under an id, name, price and colours of its // own. the server's tables and the packs new games fetch have it at once // (defpacks.setCustomItem); games already playing are sent its def row -// ("xdi", which the patched client adds to its tables), and so is every game -// at login, in case it was started before the item was made. +// ("xdi", which the patched client adds to its tables), and so is a game at +// login if its packs may be older than the item (sendAll). // // an item can be taken out of the shop, renamed and repriced, or deleted: // then it goes from every inventory too (lib/itemcontrol.js). @@ -29,6 +30,21 @@ const CHUNK = 40; // def rows per "xdi" message const TYPE_CLOTHING = "1"; const OFFERED_SPECIAL_TYPES = ["0", "4"]; // den items that can be copied: plain, mannequin const CURRENCY_GEMS = "0"; +// the fields of a def row the patched client reads from "xdi" +// (ItemXtCommManager.addItemDef, DenXtCommManager.addDenItemDef): the rest of +// the base's row is only weight, ~600 bytes a row and 5000 rows at login. +const CLIENT_FIELDS = { + clothing: ["id", "layerDefId", "layerId", "type", "value", "titleStrId", "isGiftable", "membersOnly", "itemStatus", "enviroType", + "avatarUseFlags", "recycleValue", "currencyType", "defense", "attack", "attackMediaRefId", "combatType", "modifierType", + "modifierValue", "criticalHit", "combinedCost", "colors", "availabilityStartTime", "availabilityEndTime"], + den: ["id", "abbrName", "combinedCost", "cost", "currencyType", "enviroType", "flag", "gameDefId", "itemStatus", "layer", + "mannequinAvatarDefId", "mannequinCatId", "membersOnly", "nameStrId", "parentArrayId", "promoType", "recycleValue", + "scalePercent", "sortCat", "specialType", "typeCat", "availabilityStartTime", "availabilityEndTime", "ecoPower"], +}; +// how long a fetch of the packs is taken to describe a game of that address +// that logs in (sendAll). a game logs in minutes after it fetched them. +const PACKS_SEEN_MS = 2 * 3600000; +const MAX_PACKS_SEEN = 100000; // addresses const SCHEMA = `CREATE TABLE IF NOT EXISTS custom_items ( kind TEXT NOT NULL, id INTEGER NOT NULL, base INTEGER NOT NULL, name TEXT NOT NULL, @@ -37,7 +53,12 @@ const SCHEMA = `CREATE TABLE IF NOT EXISTS custom_items ( const items = new Map(KINDS.map(kind => [kind, new Map()])); // kind -> id -> entry let made = 0; // entries in the order they were made (`seq`), for lists +let changes = 0; // goes up with every change a game must be told of; an entry's `changed` is the count at its last let broadcast = () => {}; // (cmd, ...params) to every game playing; set by lib/world.js +// address key -> {at, clothing, den, names}: `changes` when that address +// last fetched the packs with custom items in, the oldest within +// PACKS_SEEN_MS (players behind one address fetch at different times). +const packsSeen = new Map(); const db = () => accounts.database(); const now = () => Math.floor(Date.now() / 1000); @@ -64,7 +85,24 @@ const rowOf = entry => { return {...base, ...own, titleStrId: String(STRING_BASE.clothing + entry.id - FIRST_ID), value: String(entry.cost), colors: entry.colors.join(","), isGiftable: "1"}; }; -const apply = entry => defpacks.setCustomItem(entry.kind, rowOf(entry), entry.name, entry.inShop); +const apply = entry => { + defpacks.setCustomItem(entry.kind, rowOf(entry), entry.name, entry.inShop); + entry.changed = ++changes; +}; + +// what "xdi" carries of an entry: the fields the client reads, its name and +// picture. clothing goes through the clients' pack transform, so the wear +// rules reach custom items too. +const clientRow = entry => { + const row = entry.kind === "clothing" ? defpacks.clientItemRow(rowOf(entry)) : rowOf(entry); + const out = {}; + for (const field of CLIENT_FIELDS[entry.kind]) { + if (row[field] !== undefined) { + out[field] = row[field]; + } + } + return {...out, name: entry.name, art: entry.artPending ? "" : entry.art || "", artMode: entry.artMode || "sticker"}; +}; // "xdi" [.., .., kind, json]: def rows with their names, as JSON. a field // may not hold "%" (lib/sfs.js drops it), so a "%" in a name is written as @@ -75,9 +113,7 @@ const messages = entries => { const out = []; for (const kind of KINDS) { // a picture waiting for a moderator is not shown yet - const rows = entries.filter(entry => entry.kind === kind && !entry.pending) - .map(entry => ({...(kind === "clothing" ? defpacks.clientItemRow(rowOf(entry)) : rowOf(entry)), name: entry.name, - art: entry.artPending ? "" : entry.art || "", artMode: entry.artMode || "sticker"})); + const rows = entries.filter(entry => entry.kind === kind && !entry.pending).map(clientRow); for (let at = 0; at < rows.length; at += CHUNK) { out.push([kind, xtRows(rows.slice(at, at + CHUNK))]); } @@ -281,6 +317,7 @@ const setArtMode = (kind, id, mode) => { } entry.artMode = mode; db().prepare("UPDATE custom_items SET art_mode = ? WHERE kind = ? AND id = ?").run(mode, kind, entry.id); + entry.changed = ++changes; announce(entry); return {item: entry}; }; @@ -295,6 +332,7 @@ const setArt = (kind, id, art, pending = false) => { entry.art = art || null; entry.artPending = Boolean(art && pending); db().prepare("UPDATE custom_items SET art = ?, art_pending = ? WHERE kind = ? AND id = ?").run(entry.art, entry.artPending ? 1 : 0, kind, entry.id); + entry.changed = ++changes; announce(entry); }; @@ -362,9 +400,37 @@ module.exports = { const def = baseDef(entry.kind, entry.base); return def ? defpacks.string(entry.kind === "den" ? def.nameStrId : def.titleStrId) || String(def.abbrName || entry.base) : String(entry.base); }, - // every custom item, for a game logging in. + // an address fetched a pack with custom items in: "clothing", "den" or + // "names" (lib/defpacks.js customPackKind). called before the pack is made, + // so that a change while it is made counts as after. + sawPack(key, packKind) { + const time = Date.now(); + const seen = packsSeen.get(key); + if (seen && time - seen.at < PACKS_SEEN_MS) { + if (seen[packKind] === undefined) { + seen[packKind] = changes; + } + return; + } + packsSeen.delete(key); + packsSeen.set(key, {at: time, [packKind]: changes}); + // the oldest are first + for (const [oldKey, old] of packsSeen) { + if (packsSeen.size <= MAX_PACKS_SEEN && time - old.at < PACKS_SEEN_MS) { + break; + } + packsSeen.delete(oldKey); + } + }, + // the custom items a game logging in may not have: those changed since its + // address fetched the packs, and those with a picture (which the packs do + // not carry); every one when the address fetched none lately. sendAll(session) { - for (const params of messages(list().reverse().filter(entry => !entry.pending))) { + const seen = packsSeen.get(limits.key(session.address)); + const fresh = seen && Date.now() - seen.at < PACKS_SEEN_MS && seen.names !== undefined ? seen : null; + const since = kind => (fresh && fresh[kind] !== undefined ? Math.min(fresh[kind], fresh.names) : -1); + const needed = entry => !entry.pending && (entry.changed > since(entry.kind) || (entry.art && !entry.artPending)); + for (const params of messages(list().reverse().filter(needed))) { session.sendXt("xdi", -1, ...params); } }, diff --git a/lib/defpacks.js b/lib/defpacks.js index 7107186..a26c39f 100644 --- a/lib/defpacks.js +++ b/lib/defpacks.js @@ -371,6 +371,8 @@ const CLIENT_PACKS = new Map([ ]); const clientPackPaths = new Map([...CLIENT_PACKS.keys()].map(id => [cdn.hashedPath(`defPacks/${id}`), id])); const clientPackBodies = new Map(); // id -> {day, allOnSale, body} +// the packs custom items are in: their rows, and their names +const CUSTOM_PACK_KINDS = new Map([[DEF_ITEM, "clothing"], [DEF_DEN_ITEM, "den"], [DEF_LOCALIZATION_EN, "names"]]); // the pack at `relPath` as clients should get it, or null when it is not // one of the above (and comes straight from the CDN). @@ -569,6 +571,10 @@ module.exports = { // the custom items on sale, the newest first. customShopIds: kind => [...customInShop[kind]].reverse(), customVersion: () => customVersion, + // the avatarUseFlags every clothing row has in the clients' pack (any animal) + EVERY_AVATAR, + // "clothing", "den" or "names" for a pack path with custom items in (lib/customitems.js sawPack), else null. + customPackKind: relPath => CUSTOM_PACK_KINDS.get(clientPackPaths.get(relPath)) || null, load, itemDef: id => itemDefs.get(Number(id)), clothingIds: () => clothingIds, diff --git a/lib/http.js b/lib/http.js index b9a983c..5f7e0b3 100644 --- a/lib/http.js +++ b/lib/http.js @@ -1049,6 +1049,11 @@ const handle = async (req, res) => { send(res, extra.body ? 200 : 404, extra.body || "Not found", extra.body ? extra.type : "text/plain"); return; } + // what custom items this game will have from its packs (customItems.sendAll) + const packKind = defpacks.customPackKind(relPath); + if (packKind) { + customItems.sawPack(from.key, packKind); + } const clientPack = await defpacks.clientPack(relPath); if (clientPack) { send(res, 200, clientPack, "application/octet-stream"); diff --git a/test/harness.js b/test/harness.js index e802849..bdb18b5 100644 --- a/test/harness.js +++ b/test/harness.js @@ -13,8 +13,11 @@ const os = require("os"); const path = require("path"); class GameClient { - constructor(gamePort) { + // `localAddress`: another loopback address to come from (127.0.0.2), for + // what the server tells apart by address. + constructor(gamePort, localAddress = undefined) { this.gamePort = gamePort; + this.localAddress = localAddress; this.queue = []; this.varUpdates = []; this.askedForGames = 0; @@ -25,7 +28,7 @@ class GameClient { connect() { return new Promise((resolve, reject) => { - this.socket = net.connect(this.gamePort, "127.0.0.1", resolve); + this.socket = net.connect({port: this.gamePort, host: "127.0.0.1", localAddress: this.localAddress}, resolve); this.socket.once("error", reject); this.socket.on("data", chunk => { this.buffer = Buffer.concat([this.buffer, chunk]); diff --git a/test/items.js b/test/items.js index 64f8781..a0e8893 100644 --- a/test/items.js +++ b/test/items.js @@ -8,6 +8,7 @@ const assert = require("assert"); const crypto = require("crypto"); const fs = require("fs"); +const http = require("http"); const os = require("os"); const path = require("path"); const zlib = require("zlib"); @@ -119,6 +120,34 @@ const untilDefs = async (client, kind) => { } } }; +// a game on another address: it fetches the packs with custom items in, and +// at login is sent of them only what they may lack (customItems.sendAll), +// all of it before the chat filter settings ("xfl"). +const ELSEWHERE = "127.0.0.2"; +const packsFrom = address => Promise.all([1000, 1030, 10230].map(id => new Promise((resolve, reject) => { + http.get(`${BASE}/${cdn.hashedPath(`defPacks/${id}`)}`, {localAddress: address}, res => { + assert.strictEqual(res.statusCode, 200); + res.resume(); + res.on("end", resolve); + }).on("error", reject); +}))); +const defsAtLogin = async (username, password, address) => { + const auth = await (await postJson(`${BASE}/authenticate`, {username, password})).json(); + const client = new harness.GameClient(GAME_PORT, address); + await client.connect(); + assert.strictEqual((await client.logIn(username, auth.auth_token)).status, "1"); + const defs = {clothing: [], den: []}; + for (;;) { + const fields = (await client.next()).split("%").slice(2, -1); + if (fields[0] === "xfl") { + client.socket.destroy(); + return defs; + } + if (fields[0] === "xdi") { + defs[fields[2]].push(...JSON.parse(fields[3])); + } + } +}; const nextBox = async client => (await client.untilXt("xtl"))[2]; const shopList = async (client, roomId, shop) => { client.xt("gl", roomId, shop); @@ -194,6 +223,7 @@ const first = async () => { const crownRow = byId(await pack(1000), 50000); assert.strictEqual(crownRow.layerDefId, base.layerDefId); assert.strictEqual(crownRow.avatarUseFlags, "9007199254740991"); // any animal, like all clothing here + assert.strictEqual(row.avatarUseFlags, crownRow.avatarUseFlags); // and so in what games playing are sent assert.strictEqual((await strings())[9100000], "Test Crown"); // first in the shop of oddities, and Pat can buy it in a colour it has @@ -235,6 +265,20 @@ const first = async () => { say(devGame.client, devGame.roomId, `!item clothing ${base.id} Quick Hat`); assert.strictEqual(await nextBox(devGame.client), "Made Quick Hat, clothing 50001. It is in the item finder now."); assert.strictEqual((await untilDefs(pat.client, "clothing"))[0].name, "Quick Hat"); + // a game whose address fetched the packs since has every item from them, + // and is sent none again; then only the one changed after + await packsFrom(ELSEWHERE); + assert.deepStrictEqual(await defsAtLogin("Lou", people.Lou, ELSEWHERE), {clothing: [], den: []}); + assert.match((await dev("/mod/api/items", {action: "update", kind: "clothing", id: 50000, cost: 7})).data.ok, /changed/); + assert.strictEqual((await untilDefs(pat.client, "clothing"))[0].value, "7"); + const changedSince = await defsAtLogin("Lou", people.Lou, ELSEWHERE); + assert.deepStrictEqual(changedSince.clothing.map(row => `${row.id} ${row.name} ${row.value}`), ["50000 Gold Crown 7"]); + assert.deepStrictEqual(changedSince.den, []); + // a row carries what the client reads of it, not the whole base row + const wire = ["id", "layerDefId", "layerId", "type", "value", "titleStrId", "isGiftable", "membersOnly", "itemStatus", "enviroType", + "avatarUseFlags", "recycleValue", "currencyType", "defense", "attack", "attackMediaRefId", "combatType", "modifierType", + "modifierValue", "criticalHit", "combinedCost", "colors", "availabilityStartTime", "availabilityEndTime", "name", "art", "artMode"]; + assert.deepStrictEqual(Object.keys(changedSince.clothing[0]).filter(field => !wire.includes(field)), []); say(devGame.client, devGame.roomId, "!item list"); assert.match(await nextBox(devGame.client), /^Latest items made: 50001 Quick Hat \(clothing\); 50000 Test Chair \(den\); 50000 Gold Crown \(clothing, off sale\)$/); say(devGame.client, devGame.roomId, "!item hat 12 x"); @@ -385,6 +429,10 @@ const first = async () => { const withArt = (await maker("info")).mine.find(sameItem); assert.match(withArt.art, /^p[0-9a-f]{12}$/); assert.deepStrictEqual(told, [[String(hatEntry.id), withArt.art]]); + // the packs have no pictures: a game logging in is sent each one, however new its packs + await packsFrom(ELSEWHERE); + const pictured = (await defsAtLogin("Lou", people.Lou, ELSEWHERE)).clothing; + assert.deepStrictEqual(pictured.filter(row => row.art).map(row => [row.id, row.art]), [[String(hatEntry.id), withArt.art]]); const served = await fetch(`${BASE}/itemart/clothing/${hatEntry.id}.png?v=${withArt.art}`); assert.deepStrictEqual([served.status, served.headers.get("content-type"), served.headers.get("x-content-type-options")], [200, "image/png", "nosniff"]); const bytes = Buffer.from(await served.arrayBuffer());