perf(defpacks): make the item packs again without holding the game up

A custom item change made the item and name packs again on the next
fetch, on the one thread every game shares: ~360 ms at 5000 items,
~1.6 s at 50000, after every change, players making them many times a
minute. Nearly all of it was the AMF3 writer.

- the writer fills one growing buffer instead of a Buffer per value
  (byte-identical output, checked against the old one on every cached
  pack): the item pack 230 -> 52 ms at 5000 items, 1524 -> 231 ms at 50000.
- a changed pack is made again at most every AJ_PACK_REMAKE_MS (15 s);
  until then the one made before goes out with the custom item version
  it has, and customItems.sendAll sends a game logging in what its packs
  lack by that version. one remake at a time; deflated on libuv's threads.
- tests make the packs again at once (AJ_PACK_REMAKE_MS=0), and a run of
  the items test checks the throttled case.

Busy evening simulated in-process (a change every 3 s, packs fetched
6 a second, 30 s): longest hold-up 329 -> 61 ms at 5000 items, 1648 ->
359 ms at 50000 (p99 240 -> 6 ms).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
BurrowdevandClaude Opus 5.5 committed 2026-10-20 14:44:07 +03:00
1 parent c5b1bc37d9
commit 04b8d325ea
7 files changed
+178 -64

No files matched your search

+90 -31
View File
@@ -177,40 +177,89 @@ class Amf3Reader {
// writer for what the reader above produces from a defPack: plain objects,
// arrays (with or without string keys), strings, numbers, booleans and null.
// nothing is written as a reference except repeated strings.
// written into one buffer that grows: a pack is megabytes of small values,
// and a Buffer for each of them took seconds for the item pack.
class Amf3Writer {
constructor() {
this.parts = [];
this.buf = Buffer.allocUnsafe(1 << 16);
this.pos = 0;
this.strings = new Map();
}
room(bytes) {
if (this.pos + bytes > this.buf.length) {
let size = this.buf.length * 2;
while (size < this.pos + bytes) {
size *= 2;
}
const bigger = Buffer.allocUnsafe(size);
this.buf.copy(bigger, 0, 0, this.pos);
this.buf = bigger;
}
}
byte(value) {
this.room(1);
this.buf[this.pos++] = value;
}
u29(value) {
this.room(4);
const {buf} = this;
if (value < 0x80) {
this.parts.push(Buffer.from([value]));
buf[this.pos++] = value;
}
else if (value < 0x4000) {
this.parts.push(Buffer.from([(value >> 7) | 0x80, value & 0x7f]));
buf[this.pos++] = (value >> 7) | 0x80;
buf[this.pos++] = value & 0x7f;
}
else if (value < 0x200000) {
this.parts.push(Buffer.from([(value >> 14) | 0x80, ((value >> 7) & 0x7f) | 0x80, value & 0x7f]));
buf[this.pos++] = (value >> 14) | 0x80;
buf[this.pos++] = ((value >> 7) & 0x7f) | 0x80;
buf[this.pos++] = value & 0x7f;
}
else {
this.parts.push(Buffer.from([(value >> 22) | 0x80, ((value >> 15) & 0x7f) | 0x80, ((value >> 8) & 0x7f) | 0x80, value & 0xff]));
buf[this.pos++] = (value >> 22) | 0x80;
buf[this.pos++] = ((value >> 15) & 0x7f) | 0x80;
buf[this.pos++] = ((value >> 8) & 0x7f) | 0x80;
buf[this.pos++] = value & 0xff;
}
}
string(text) {
if (text === "") {
this.u29(1);
return;
}
else if (this.strings.has(text)) {
this.u29(this.strings.get(text) << 1);
const known = this.strings.get(text);
if (known !== undefined) {
this.u29(known << 1);
return;
}
else {
this.strings.set(text, this.strings.size);
const bytes = Buffer.from(text, "utf8");
this.u29((bytes.length << 1) | 1);
this.parts.push(bytes);
this.strings.set(text, this.strings.size);
// nearly every string of a pack is short and ASCII: its bytes are its
// characters, copied here rather than through Buffer's utf8 writer.
if (text.length < 64) {
let ascii = true;
for (let i = 0; i < text.length; i++) {
if (text.charCodeAt(i) > 0x7f) {
ascii = false;
break;
}
}
if (ascii) {
this.u29((text.length << 1) | 1);
this.room(text.length);
for (let i = 0; i < text.length; i++) {
this.buf[this.pos++] = text.charCodeAt(i);
}
return;
}
}
const length = Buffer.byteLength(text, "utf8");
this.u29((length << 1) | 1);
this.room(length);
this.pos += this.buf.write(text, this.pos, "utf8");
}
pairs(entries) {
@@ -222,45 +271,55 @@ class Amf3Writer {
}
value(value) {
if (value === undefined) {
this.parts.push(Buffer.from([0x00]));
// most values of a pack are strings: asked first
if (typeof value === "string") {
this.byte(0x06);
this.string(value);
}
else if (value === undefined) {
this.byte(0x00);
}
else if (value === null) {
this.parts.push(Buffer.from([0x01]));
this.byte(0x01);
}
else if (typeof value === "boolean") {
this.parts.push(Buffer.from([value ? 0x03 : 0x02]));
this.byte(value ? 0x03 : 0x02);
}
else if (typeof value === "number") {
if (Number.isInteger(value) && value >= -0x10000000 && value < 0x10000000) {
this.parts.push(Buffer.from([0x04]));
this.byte(0x04);
this.u29(value & 0x1fffffff);
}
else {
const bytes = Buffer.alloc(9);
bytes[0] = 0x05;
bytes.writeDoubleBE(value, 1);
this.parts.push(bytes);
this.byte(0x05);
this.room(8);
this.pos = this.buf.writeDoubleBE(value, this.pos);
}
}
else if (typeof value === "string") {
this.parts.push(Buffer.from([0x06]));
this.string(value);
}
else if (Array.isArray(value)) {
this.parts.push(Buffer.from([0x09]));
this.byte(0x09);
this.u29((value.length << 1) | 1);
// string keys set on the array beside its numbered elements.
this.pairs(Object.entries(value).filter(([key]) => !/^\d+$/.test(key)));
// string keys set on the array beside its numbered elements (an array
// with as many keys as elements has none).
const keys = Object.keys(value);
this.pairs(keys.length === value.length ? [] : keys.filter(key => !/^\d+$/.test(key)).map(key => [key, value[key]]));
for (const element of value) {
this.value(element);
}
}
else if (typeof value === "object" && !Buffer.isBuffer(value)) {
// A dynamic object of no class, with its traits written in place.
this.parts.push(Buffer.from([0x0a, 0x0b]));
this.byte(0x0a);
this.byte(0x0b);
this.string("");
// its own keys in Object.keys order, without an array of entries for each object
for (const key in value) {
if (Object.hasOwn(value, key)) {
this.string(key);
this.value(value[key]);
}
}
this.string("");
this.pairs(Object.entries(value));
}
else {
throw new Error("Cannot write this value as AMF3");
@@ -273,6 +332,6 @@ module.exports = {
encode: value => {
const writer = new Amf3Writer();
writer.value(value);
return Buffer.concat(writer.parts);
return writer.buf.subarray(0, writer.pos);
},
};