pack.js 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137
  1. import { Unpackr, mult10, C1Type, typedArrays, addExtension as unpackAddExtension } from './unpack.js'
  2. let textEncoder
  3. try {
  4. textEncoder = new TextEncoder()
  5. } catch (error) {}
  6. let extensions, extensionClasses
  7. const hasNodeBuffer = typeof Buffer !== 'undefined'
  8. const ByteArrayAllocate = hasNodeBuffer ?
  9. function(length) { return Buffer.allocUnsafeSlow(length) } : Uint8Array
  10. const ByteArray = hasNodeBuffer ? Buffer : Uint8Array
  11. const MAX_BUFFER_SIZE = hasNodeBuffer ? 0x100000000 : 0x7fd00000
  12. let target, keysTarget
  13. let targetView
  14. let position = 0
  15. let safeEnd
  16. let bundledStrings = null
  17. let writeStructSlots
  18. const MAX_BUNDLE_SIZE = 0x5500 // maximum characters such that the encoded bytes fits in 16 bits.
  19. const hasNonLatin = /[\u0080-\uFFFF]/
  20. export const RECORD_SYMBOL = Symbol('record-id')
  21. export class Packr extends Unpackr {
  22. constructor(options) {
  23. super(options)
  24. this.offset = 0
  25. let typeBuffer
  26. let start
  27. let hasSharedUpdate
  28. let structures
  29. let referenceMap
  30. let encodeUtf8 = ByteArray.prototype.utf8Write ? function(string, position) {
  31. return target.utf8Write(string, position, target.byteLength - position)
  32. } : (textEncoder && textEncoder.encodeInto) ?
  33. function(string, position) {
  34. return textEncoder.encodeInto(string, target.subarray(position)).written
  35. } : false
  36. let packr = this
  37. if (!options)
  38. options = {}
  39. let isSequential = options && options.sequential
  40. let hasSharedStructures = options.structures || options.saveStructures
  41. let maxSharedStructures = options.maxSharedStructures
  42. if (maxSharedStructures == null)
  43. maxSharedStructures = hasSharedStructures ? 32 : 0
  44. if (maxSharedStructures > 8160)
  45. throw new Error('Maximum maxSharedStructure is 8160')
  46. if (options.structuredClone && options.moreTypes == undefined) {
  47. this.moreTypes = true
  48. }
  49. let maxOwnStructures = options.maxOwnStructures
  50. if (maxOwnStructures == null)
  51. maxOwnStructures = hasSharedStructures ? 32 : 64
  52. if (!this.structures && options.useRecords != false)
  53. this.structures = []
  54. // two byte record ids for shared structures
  55. let useTwoByteRecords = maxSharedStructures > 32 || (maxOwnStructures + maxSharedStructures > 64)
  56. let sharedLimitId = maxSharedStructures + 0x40
  57. let maxStructureId = maxSharedStructures + maxOwnStructures + 0x40
  58. if (maxStructureId > 8256) {
  59. throw new Error('Maximum maxSharedStructure + maxOwnStructure is 8192')
  60. }
  61. let recordIdsToRemove = []
  62. let transitionsCount = 0
  63. let serializationsSinceTransitionRebuild = 0
  64. this.pack = this.encode = function(value, encodeOptions) {
  65. if (!target) {
  66. target = new ByteArrayAllocate(8192)
  67. targetView = target.dataView || (target.dataView = new DataView(target.buffer, 0, 8192))
  68. position = 0
  69. }
  70. safeEnd = target.length - 10
  71. if (safeEnd - position < 0x800) {
  72. // don't start too close to the end,
  73. target = new ByteArrayAllocate(target.length)
  74. targetView = target.dataView || (target.dataView = new DataView(target.buffer, 0, target.length))
  75. safeEnd = target.length - 10
  76. position = 0
  77. } else
  78. position = (position + 7) & 0x7ffffff8 // Word align to make any future copying of this buffer faster
  79. start = position
  80. if (encodeOptions & RESERVE_START_SPACE) position += (encodeOptions & 0xff)
  81. referenceMap = packr.structuredClone ? new Map() : null
  82. if (packr.bundleStrings && typeof value !== 'string') {
  83. bundledStrings = []
  84. bundledStrings.size = Infinity // force a new bundle start on first string
  85. } else
  86. bundledStrings = null
  87. structures = packr.structures
  88. if (structures) {
  89. if (structures.uninitialized)
  90. structures = packr._mergeStructures(packr.getStructures())
  91. let sharedLength = structures.sharedLength || 0
  92. if (sharedLength > maxSharedStructures) {
  93. //if (maxSharedStructures <= 32 && structures.sharedLength > 32) // TODO: could support this, but would need to update the limit ids
  94. throw new Error('Shared structures is larger than maximum shared structures, try increasing maxSharedStructures to ' + structures.sharedLength)
  95. }
  96. if (!structures.transitions) {
  97. // rebuild our structure transitions
  98. structures.transitions = Object.create(null)
  99. for (let i = 0; i < sharedLength; i++) {
  100. let keys = structures[i]
  101. if (!keys)
  102. continue
  103. let nextTransition, transition = structures.transitions
  104. for (let j = 0, l = keys.length; j < l; j++) {
  105. let key = keys[j]
  106. nextTransition = transition[key]
  107. if (!nextTransition) {
  108. nextTransition = transition[key] = Object.create(null)
  109. }
  110. transition = nextTransition
  111. }
  112. transition[RECORD_SYMBOL] = i + 0x40
  113. }
  114. this.lastNamedStructuresLength = sharedLength
  115. }
  116. if (!isSequential) {
  117. structures.nextId = sharedLength + 0x40
  118. }
  119. }
  120. if (hasSharedUpdate)
  121. hasSharedUpdate = false
  122. let encodingError;
  123. try {
  124. if (packr.randomAccessStructure && value && value.constructor && value.constructor === Object)
  125. writeStruct(value);
  126. else
  127. pack(value)
  128. let lastBundle = bundledStrings;
  129. if (bundledStrings)
  130. writeBundles(start, pack, 0)
  131. if (referenceMap && referenceMap.idsToInsert) {
  132. let idsToInsert = referenceMap.idsToInsert.sort((a, b) => a.offset > b.offset ? 1 : -1);
  133. let i = idsToInsert.length;
  134. let incrementPosition = -1;
  135. while (lastBundle && i > 0) {
  136. let insertionPoint = idsToInsert[--i].offset + start;
  137. if (insertionPoint < (lastBundle.stringsPosition + start) && incrementPosition === -1)
  138. incrementPosition = 0;
  139. if (insertionPoint > (lastBundle.position + start)) {
  140. if (incrementPosition >= 0)
  141. incrementPosition += 6;
  142. } else {
  143. if (incrementPosition >= 0) {
  144. // update the bundle reference now
  145. targetView.setUint32(lastBundle.position + start,
  146. targetView.getUint32(lastBundle.position + start) + incrementPosition)
  147. incrementPosition = -1; // reset
  148. }
  149. lastBundle = lastBundle.previous;
  150. i++;
  151. }
  152. }
  153. if (incrementPosition >= 0 && lastBundle) {
  154. // update the bundle reference now
  155. targetView.setUint32(lastBundle.position + start,
  156. targetView.getUint32(lastBundle.position + start) + incrementPosition)
  157. }
  158. position += idsToInsert.length * 6;
  159. if (position > safeEnd)
  160. makeRoom(position)
  161. packr.offset = position
  162. let serialized = insertIds(target.subarray(start, position), idsToInsert)
  163. referenceMap = null
  164. return serialized
  165. }
  166. packr.offset = position // update the offset so next serialization doesn't write over our buffer, but can continue writing to same buffer sequentially
  167. if (encodeOptions & REUSE_BUFFER_MODE) {
  168. target.start = start
  169. target.end = position
  170. return target
  171. }
  172. return target.subarray(start, position) // position can change if we call pack again in saveStructures, so we get the buffer now
  173. } catch(error) {
  174. encodingError = error;
  175. throw error;
  176. } finally {
  177. if (structures) {
  178. resetStructures();
  179. if (hasSharedUpdate && packr.saveStructures) {
  180. let sharedLength = structures.sharedLength || 0
  181. // we can't rely on start/end with REUSE_BUFFER_MODE since they will (probably) change when we save
  182. let returnBuffer = target.subarray(start, position)
  183. let newSharedData = prepareStructures(structures, packr);
  184. if (!encodingError) { // TODO: If there is an encoding error, should make the structures as uninitialized so they get rebuilt next time
  185. if (packr.saveStructures(newSharedData, newSharedData.isCompatible) === false) {
  186. // get updated structures and try again if the update failed
  187. return packr.pack(value, encodeOptions)
  188. }
  189. packr.lastNamedStructuresLength = sharedLength
  190. // don't keep large buffers around
  191. if (target.length > 0x40000000) target = null
  192. return returnBuffer
  193. }
  194. }
  195. }
  196. // don't keep large buffers around, they take too much memory and cause problems (limit at 1GB)
  197. if (target.length > 0x40000000) target = null
  198. if (encodeOptions & RESET_BUFFER_MODE)
  199. position = start
  200. }
  201. }
  202. const resetStructures = () => {
  203. if (serializationsSinceTransitionRebuild < 10)
  204. serializationsSinceTransitionRebuild++
  205. let sharedLength = structures.sharedLength || 0
  206. if (structures.length > sharedLength && !isSequential)
  207. structures.length = sharedLength
  208. if (transitionsCount > 10000) {
  209. // force a rebuild occasionally after a lot of transitions so it can get cleaned up
  210. structures.transitions = null
  211. serializationsSinceTransitionRebuild = 0
  212. transitionsCount = 0
  213. if (recordIdsToRemove.length > 0)
  214. recordIdsToRemove = []
  215. } else if (recordIdsToRemove.length > 0 && !isSequential) {
  216. for (let i = 0, l = recordIdsToRemove.length; i < l; i++) {
  217. recordIdsToRemove[i][RECORD_SYMBOL] = 0
  218. }
  219. recordIdsToRemove = []
  220. }
  221. }
  222. const packArray = (value) => {
  223. var length = value.length
  224. if (length < 0x10) {
  225. target[position++] = 0x90 | length
  226. } else if (length < 0x10000) {
  227. target[position++] = 0xdc
  228. target[position++] = length >> 8
  229. target[position++] = length & 0xff
  230. } else {
  231. target[position++] = 0xdd
  232. targetView.setUint32(position, length)
  233. position += 4
  234. }
  235. for (let i = 0; i < length; i++) {
  236. pack(value[i])
  237. }
  238. }
  239. const pack = (value) => {
  240. if (position > safeEnd)
  241. target = makeRoom(position)
  242. var type = typeof value
  243. var length
  244. if (type === 'string') {
  245. let strLength = value.length
  246. if (bundledStrings && strLength >= 4 && strLength < 0x1000) {
  247. if ((bundledStrings.size += strLength) > MAX_BUNDLE_SIZE) {
  248. let extStart
  249. let maxBytes = (bundledStrings[0] ? bundledStrings[0].length * 3 + bundledStrings[1].length : 0) + 10
  250. if (position + maxBytes > safeEnd)
  251. target = makeRoom(position + maxBytes)
  252. let lastBundle
  253. if (bundledStrings.position) { // here we use the 0x62 extension to write the last bundle and reserve space for the reference pointer to the next/current bundle
  254. lastBundle = bundledStrings
  255. target[position] = 0xc8 // ext 16
  256. position += 3 // reserve for the writing bundle size
  257. target[position++] = 0x62 // 'b'
  258. extStart = position - start
  259. position += 4 // reserve for writing bundle reference
  260. writeBundles(start, pack, 0) // write the last bundles
  261. targetView.setUint16(extStart + start - 3, position - start - extStart)
  262. } else { // here we use the 0x62 extension just to reserve the space for the reference pointer to the bundle (will be updated once the bundle is written)
  263. target[position++] = 0xd6 // fixext 4
  264. target[position++] = 0x62 // 'b'
  265. extStart = position - start
  266. position += 4 // reserve for writing bundle reference
  267. }
  268. bundledStrings = ['', ''] // create new ones
  269. bundledStrings.previous = lastBundle;
  270. bundledStrings.size = 0
  271. bundledStrings.position = extStart
  272. }
  273. let twoByte = hasNonLatin.test(value)
  274. bundledStrings[twoByte ? 0 : 1] += value
  275. target[position++] = 0xc1
  276. pack(twoByte ? -strLength : strLength);
  277. return
  278. }
  279. let headerSize
  280. // first we estimate the header size, so we can write to the correct location
  281. if (strLength < 0x20) {
  282. headerSize = 1
  283. } else if (strLength < 0x100) {
  284. headerSize = 2
  285. } else if (strLength < 0x10000) {
  286. headerSize = 3
  287. } else {
  288. headerSize = 5
  289. }
  290. let maxBytes = strLength * 3
  291. if (position + maxBytes > safeEnd)
  292. target = makeRoom(position + maxBytes)
  293. if (strLength < 0x40 || !encodeUtf8) {
  294. let i, c1, c2, strPosition = position + headerSize
  295. for (i = 0; i < strLength; i++) {
  296. c1 = value.charCodeAt(i)
  297. if (c1 < 0x80) {
  298. target[strPosition++] = c1
  299. } else if (c1 < 0x800) {
  300. target[strPosition++] = c1 >> 6 | 0xc0
  301. target[strPosition++] = c1 & 0x3f | 0x80
  302. } else if (
  303. (c1 & 0xfc00) === 0xd800 &&
  304. ((c2 = value.charCodeAt(i + 1)) & 0xfc00) === 0xdc00
  305. ) {
  306. c1 = 0x10000 + ((c1 & 0x03ff) << 10) + (c2 & 0x03ff)
  307. i++
  308. target[strPosition++] = c1 >> 18 | 0xf0
  309. target[strPosition++] = c1 >> 12 & 0x3f | 0x80
  310. target[strPosition++] = c1 >> 6 & 0x3f | 0x80
  311. target[strPosition++] = c1 & 0x3f | 0x80
  312. } else {
  313. target[strPosition++] = c1 >> 12 | 0xe0
  314. target[strPosition++] = c1 >> 6 & 0x3f | 0x80
  315. target[strPosition++] = c1 & 0x3f | 0x80
  316. }
  317. }
  318. length = strPosition - position - headerSize
  319. } else {
  320. length = encodeUtf8(value, position + headerSize)
  321. }
  322. if (length < 0x20) {
  323. target[position++] = 0xa0 | length
  324. } else if (length < 0x100) {
  325. if (headerSize < 2) {
  326. target.copyWithin(position + 2, position + 1, position + 1 + length)
  327. }
  328. target[position++] = 0xd9
  329. target[position++] = length
  330. } else if (length < 0x10000) {
  331. if (headerSize < 3) {
  332. target.copyWithin(position + 3, position + 2, position + 2 + length)
  333. }
  334. target[position++] = 0xda
  335. target[position++] = length >> 8
  336. target[position++] = length & 0xff
  337. } else {
  338. if (headerSize < 5) {
  339. target.copyWithin(position + 5, position + 3, position + 3 + length)
  340. }
  341. target[position++] = 0xdb
  342. targetView.setUint32(position, length)
  343. position += 4
  344. }
  345. position += length
  346. } else if (type === 'number') {
  347. if (value >>> 0 === value) {// positive integer, 32-bit or less
  348. // positive uint
  349. if (value < 0x20 || (value < 0x80 && this.useRecords === false) || (value < 0x40 && !this.randomAccessStructure)) {
  350. target[position++] = value
  351. } else if (value < 0x100) {
  352. target[position++] = 0xcc
  353. target[position++] = value
  354. } else if (value < 0x10000) {
  355. target[position++] = 0xcd
  356. target[position++] = value >> 8
  357. target[position++] = value & 0xff
  358. } else {
  359. target[position++] = 0xce
  360. targetView.setUint32(position, value)
  361. position += 4
  362. }
  363. } else if (value >> 0 === value) { // negative integer
  364. if (value >= -0x20) {
  365. target[position++] = 0x100 + value
  366. } else if (value >= -0x80) {
  367. target[position++] = 0xd0
  368. target[position++] = value + 0x100
  369. } else if (value >= -0x8000) {
  370. target[position++] = 0xd1
  371. targetView.setInt16(position, value)
  372. position += 2
  373. } else {
  374. target[position++] = 0xd2
  375. targetView.setInt32(position, value)
  376. position += 4
  377. }
  378. } else {
  379. let useFloat32
  380. if ((useFloat32 = this.useFloat32) > 0 && value < 0x100000000 && value >= -0x80000000) {
  381. target[position++] = 0xca
  382. targetView.setFloat32(position, value)
  383. let xShifted
  384. if (useFloat32 < 4 ||
  385. // this checks for rounding of numbers that were encoded in 32-bit float to nearest significant decimal digit that could be preserved
  386. ((xShifted = value * mult10[((target[position] & 0x7f) << 1) | (target[position + 1] >> 7)]) >> 0) === xShifted) {
  387. position += 4
  388. return
  389. } else
  390. position-- // move back into position for writing a double
  391. }
  392. target[position++] = 0xcb
  393. targetView.setFloat64(position, value)
  394. position += 8
  395. }
  396. } else if (type === 'object' || type === 'function') {
  397. if (!value)
  398. target[position++] = 0xc0
  399. else {
  400. if (referenceMap) {
  401. let referee = referenceMap.get(value)
  402. if (referee) {
  403. if (!referee.id) {
  404. let idsToInsert = referenceMap.idsToInsert || (referenceMap.idsToInsert = [])
  405. referee.id = idsToInsert.push(referee)
  406. }
  407. target[position++] = 0xd6 // fixext 4
  408. target[position++] = 0x70 // "p" for pointer
  409. targetView.setUint32(position, referee.id)
  410. position += 4
  411. return
  412. } else
  413. referenceMap.set(value, { offset: position - start })
  414. }
  415. let constructor = value.constructor
  416. if (constructor === Object) {
  417. writeObject(value)
  418. } else if (constructor === Array) {
  419. packArray(value)
  420. } else if (constructor === Map) {
  421. if (this.mapAsEmptyObject) target[position++] = 0x80
  422. else {
  423. length = value.size
  424. if (length < 0x10) {
  425. target[position++] = 0x80 | length
  426. } else if (length < 0x10000) {
  427. target[position++] = 0xde
  428. target[position++] = length >> 8
  429. target[position++] = length & 0xff
  430. } else {
  431. target[position++] = 0xdf
  432. targetView.setUint32(position, length)
  433. position += 4
  434. }
  435. for (let [key, entryValue] of value) {
  436. pack(key)
  437. pack(entryValue)
  438. }
  439. }
  440. } else {
  441. for (let i = 0, l = extensions.length; i < l; i++) {
  442. let extensionClass = extensionClasses[i]
  443. if (value instanceof extensionClass) {
  444. let extension = extensions[i]
  445. if (extension.write) {
  446. if (extension.type) {
  447. target[position++] = 0xd4 // one byte "tag" extension
  448. target[position++] = extension.type
  449. target[position++] = 0
  450. }
  451. let writeResult = extension.write.call(this, value)
  452. if (writeResult === value) { // avoid infinite recursion
  453. if (Array.isArray(value)) {
  454. packArray(value)
  455. } else {
  456. writeObject(value)
  457. }
  458. } else {
  459. pack(writeResult)
  460. }
  461. return
  462. }
  463. let currentTarget = target
  464. let currentTargetView = targetView
  465. let currentPosition = position
  466. target = null
  467. let result
  468. try {
  469. result = extension.pack.call(this, value, (size) => {
  470. // restore target and use it
  471. target = currentTarget
  472. currentTarget = null
  473. position += size
  474. if (position > safeEnd)
  475. makeRoom(position)
  476. return {
  477. target, targetView, position: position - size
  478. }
  479. }, pack)
  480. } finally {
  481. // restore current target information (unless already restored)
  482. if (currentTarget) {
  483. target = currentTarget
  484. targetView = currentTargetView
  485. position = currentPosition
  486. safeEnd = target.length - 10
  487. }
  488. }
  489. if (result) {
  490. if (result.length + position > safeEnd)
  491. makeRoom(result.length + position)
  492. position = writeExtensionData(result, target, position, extension.type)
  493. }
  494. return
  495. }
  496. }
  497. // check isArray after extensions, because extensions can extend Array
  498. if (Array.isArray(value)) {
  499. packArray(value)
  500. } else {
  501. // use this as an alternate mechanism for expressing how to serialize
  502. if (value.toJSON) {
  503. const json = value.toJSON()
  504. // if for some reason value.toJSON returns itself it'll loop forever
  505. if (json !== value)
  506. return pack(json)
  507. }
  508. // if there is a writeFunction, use it, otherwise just encode as undefined
  509. if (type === 'function')
  510. return pack(this.writeFunction && this.writeFunction(value));
  511. // no extension found, write as plain object
  512. writeObject(value)
  513. }
  514. }
  515. }
  516. } else if (type === 'boolean') {
  517. target[position++] = value ? 0xc3 : 0xc2
  518. } else if (type === 'bigint') {
  519. if (value < 0x8000000000000000 && value >= -0x8000000000000000) {
  520. // use a signed int as long as it fits
  521. target[position++] = 0xd3
  522. targetView.setBigInt64(position, value)
  523. } else if (value < 0x10000000000000000 && value > 0) {
  524. // if we can fit an unsigned int, use that
  525. target[position++] = 0xcf
  526. targetView.setBigUint64(position, value)
  527. } else {
  528. // overflow
  529. if (this.largeBigIntToFloat) {
  530. target[position++] = 0xcb
  531. targetView.setFloat64(position, Number(value))
  532. } else if (this.largeBigIntToString) {
  533. return pack(value.toString());
  534. } else if (this.useBigIntExtension || this.moreTypes) {
  535. let empty = value < 0 ? BigInt(-1) : BigInt(0)
  536. let array
  537. if (value >> BigInt(0x10000) === empty) {
  538. let mask = BigInt(0x10000000000000000) - BigInt(1) // literal would overflow
  539. let chunks = []
  540. while (true) {
  541. chunks.push(value & mask)
  542. if ((value >> BigInt(63)) === empty) break
  543. value >>= BigInt(64)
  544. }
  545. array = new Uint8Array(new BigUint64Array(chunks).buffer)
  546. array.reverse()
  547. } else {
  548. let invert = value < 0
  549. let string = (invert ? ~value : value).toString(16)
  550. if (string.length % 2) {
  551. string = '0' + string
  552. } else if (parseInt(string.charAt(0), 16) >= 8) {
  553. string = '00' + string
  554. }
  555. if (hasNodeBuffer) {
  556. array = Buffer.from(string, 'hex')
  557. } else {
  558. array = new Uint8Array(string.length / 2)
  559. for (let i = 0; i < array.length; i++) {
  560. array[i] = parseInt(string.slice(i * 2, i * 2 + 2), 16)
  561. }
  562. }
  563. if (invert) {
  564. for (let i = 0; i < array.length; i++) array[i] = ~array[i]
  565. }
  566. }
  567. if (array.length + position > safeEnd)
  568. makeRoom(array.length + position)
  569. position = writeExtensionData(array, target, position, 0x42)
  570. return
  571. } else {
  572. throw new RangeError(value + ' was too large to fit in MessagePack 64-bit integer format, use' +
  573. ' useBigIntExtension, or set largeBigIntToFloat to convert to float-64, or set' +
  574. ' largeBigIntToString to convert to string')
  575. }
  576. }
  577. position += 8
  578. } else if (type === 'undefined') {
  579. if (this.encodeUndefinedAsNil)
  580. target[position++] = 0xc0
  581. else {
  582. target[position++] = 0xd4 // a number of implementations use fixext1 with type 0, data 0 to denote undefined, so we follow suite
  583. target[position++] = 0
  584. target[position++] = 0
  585. }
  586. } else {
  587. throw new Error('Unknown type: ' + type)
  588. }
  589. }
  590. const writePlainObject = (this.variableMapSize || this.coercibleKeyAsNumber || this.skipValues) ? (object) => {
  591. // this method is slightly slower, but generates "preferred serialization" (optimally small for smaller objects)
  592. let keys;
  593. if (this.skipValues) {
  594. keys = [];
  595. for (let key in object) {
  596. if ((typeof object.hasOwnProperty !== 'function' || object.hasOwnProperty(key)) &&
  597. !this.skipValues.includes(object[key]))
  598. keys.push(key);
  599. }
  600. } else {
  601. keys = Object.keys(object)
  602. }
  603. let length = keys.length
  604. if (length < 0x10) {
  605. target[position++] = 0x80 | length
  606. } else if (length < 0x10000) {
  607. target[position++] = 0xde
  608. target[position++] = length >> 8
  609. target[position++] = length & 0xff
  610. } else {
  611. target[position++] = 0xdf
  612. targetView.setUint32(position, length)
  613. position += 4
  614. }
  615. let key
  616. if (this.coercibleKeyAsNumber) {
  617. for (let i = 0; i < length; i++) {
  618. key = keys[i]
  619. let num = Number(key)
  620. pack(isNaN(num) ? key : num)
  621. pack(object[key])
  622. }
  623. } else {
  624. for (let i = 0; i < length; i++) {
  625. pack(key = keys[i])
  626. pack(object[key])
  627. }
  628. }
  629. } :
  630. (object) => {
  631. target[position++] = 0xde // always using map 16, so we can preallocate and set the length afterwards
  632. let objectOffset = position - start
  633. position += 2
  634. let size = 0
  635. for (let key in object) {
  636. if (typeof object.hasOwnProperty !== 'function' || object.hasOwnProperty(key)) {
  637. pack(key)
  638. pack(object[key])
  639. size++
  640. }
  641. }
  642. if (size > 0xffff) {
  643. throw new Error('Object is too large to serialize with fast 16-bit map size,' +
  644. ' use the "variableMapSize" option to serialize this object');
  645. }
  646. target[objectOffset++ + start] = size >> 8
  647. target[objectOffset + start] = size & 0xff
  648. }
  649. const writeRecord = this.useRecords === false ? writePlainObject :
  650. (options.progressiveRecords && !useTwoByteRecords) ? // this is about 2% faster for highly stable structures, since it only requires one for-in loop (but much more expensive when new structure needs to be written)
  651. (object) => {
  652. let nextTransition, transition = structures.transitions || (structures.transitions = Object.create(null))
  653. let objectOffset = position++ - start
  654. let wroteKeys
  655. for (let key in object) {
  656. if (typeof object.hasOwnProperty !== 'function' || object.hasOwnProperty(key)) {
  657. nextTransition = transition[key]
  658. if (nextTransition)
  659. transition = nextTransition
  660. else {
  661. // record doesn't exist, create full new record and insert it
  662. let keys = Object.keys(object)
  663. let lastTransition = transition
  664. transition = structures.transitions
  665. let newTransitions = 0
  666. for (let i = 0, l = keys.length; i < l; i++) {
  667. let key = keys[i]
  668. nextTransition = transition[key]
  669. if (!nextTransition) {
  670. nextTransition = transition[key] = Object.create(null)
  671. newTransitions++
  672. }
  673. transition = nextTransition
  674. }
  675. if (objectOffset + start + 1 == position) {
  676. // first key, so we don't need to insert, we can just write record directly
  677. position--
  678. newRecord(transition, keys, newTransitions)
  679. } else // otherwise we need to insert the record, moving existing data after the record
  680. insertNewRecord(transition, keys, objectOffset, newTransitions)
  681. wroteKeys = true
  682. transition = lastTransition[key]
  683. }
  684. pack(object[key])
  685. }
  686. }
  687. if (!wroteKeys) {
  688. let recordId = transition[RECORD_SYMBOL]
  689. if (recordId)
  690. target[objectOffset + start] = recordId
  691. else
  692. insertNewRecord(transition, Object.keys(object), objectOffset, 0)
  693. }
  694. } :
  695. (object) => {
  696. let nextTransition, transition = structures.transitions || (structures.transitions = Object.create(null))
  697. let newTransitions = 0
  698. for (let key in object) if (typeof object.hasOwnProperty !== 'function' || object.hasOwnProperty(key)) {
  699. nextTransition = transition[key]
  700. if (!nextTransition) {
  701. nextTransition = transition[key] = Object.create(null)
  702. newTransitions++
  703. }
  704. transition = nextTransition
  705. }
  706. let recordId = transition[RECORD_SYMBOL]
  707. if (recordId) {
  708. if (recordId >= 0x60 && useTwoByteRecords) {
  709. target[position++] = ((recordId -= 0x60) & 0x1f) + 0x60
  710. target[position++] = recordId >> 5
  711. } else
  712. target[position++] = recordId
  713. } else {
  714. newRecord(transition, transition.__keys__ || Object.keys(object), newTransitions)
  715. }
  716. // now write the values
  717. for (let key in object)
  718. if (typeof object.hasOwnProperty !== 'function' || object.hasOwnProperty(key)) {
  719. pack(object[key])
  720. }
  721. }
  722. // create reference to useRecords if useRecords is a function
  723. const checkUseRecords = typeof this.useRecords == 'function' && this.useRecords;
  724. const writeObject = checkUseRecords ? (object) => {
  725. checkUseRecords(object) ? writeRecord(object) : writePlainObject(object)
  726. } : writeRecord
  727. const makeRoom = (end) => {
  728. let newSize
  729. if (end > 0x1000000) {
  730. // special handling for really large buffers
  731. if ((end - start) > MAX_BUFFER_SIZE)
  732. throw new Error('Packed buffer would be larger than maximum buffer size')
  733. newSize = Math.min(MAX_BUFFER_SIZE,
  734. Math.round(Math.max((end - start) * (end > 0x4000000 ? 1.25 : 2), 0x400000) / 0x1000) * 0x1000)
  735. } else // faster handling for smaller buffers
  736. newSize = ((Math.max((end - start) << 2, target.length - 1) >> 12) + 1) << 12
  737. let newBuffer = new ByteArrayAllocate(newSize)
  738. targetView = newBuffer.dataView || (newBuffer.dataView = new DataView(newBuffer.buffer, 0, newSize))
  739. end = Math.min(end, target.length)
  740. if (target.copy)
  741. target.copy(newBuffer, 0, start, end)
  742. else
  743. newBuffer.set(target.slice(start, end))
  744. position -= start
  745. start = 0
  746. safeEnd = newBuffer.length - 10
  747. return target = newBuffer
  748. }
  749. const newRecord = (transition, keys, newTransitions) => {
  750. let recordId = structures.nextId
  751. if (!recordId)
  752. recordId = 0x40
  753. if (recordId < sharedLimitId && this.shouldShareStructure && !this.shouldShareStructure(keys)) {
  754. recordId = structures.nextOwnId
  755. if (!(recordId < maxStructureId))
  756. recordId = sharedLimitId
  757. structures.nextOwnId = recordId + 1
  758. } else {
  759. if (recordId >= maxStructureId)// cycle back around
  760. recordId = sharedLimitId
  761. structures.nextId = recordId + 1
  762. }
  763. let highByte = keys.highByte = recordId >= 0x60 && useTwoByteRecords ? (recordId - 0x60) >> 5 : -1
  764. transition[RECORD_SYMBOL] = recordId
  765. transition.__keys__ = keys
  766. structures[recordId - 0x40] = keys
  767. if (recordId < sharedLimitId) {
  768. keys.isShared = true
  769. structures.sharedLength = recordId - 0x3f
  770. hasSharedUpdate = true
  771. if (highByte >= 0) {
  772. target[position++] = (recordId & 0x1f) + 0x60
  773. target[position++] = highByte
  774. } else {
  775. target[position++] = recordId
  776. }
  777. } else {
  778. if (highByte >= 0) {
  779. target[position++] = 0xd5 // fixext 2
  780. target[position++] = 0x72 // "r" record defintion extension type
  781. target[position++] = (recordId & 0x1f) + 0x60
  782. target[position++] = highByte
  783. } else {
  784. target[position++] = 0xd4 // fixext 1
  785. target[position++] = 0x72 // "r" record defintion extension type
  786. target[position++] = recordId
  787. }
  788. if (newTransitions)
  789. transitionsCount += serializationsSinceTransitionRebuild * newTransitions
  790. // record the removal of the id, we can maintain our shared structure
  791. if (recordIdsToRemove.length >= maxOwnStructures)
  792. recordIdsToRemove.shift()[RECORD_SYMBOL] = 0 // we are cycling back through, and have to remove old ones
  793. recordIdsToRemove.push(transition)
  794. pack(keys)
  795. }
  796. }
  797. const insertNewRecord = (transition, keys, insertionOffset, newTransitions) => {
  798. let mainTarget = target
  799. let mainPosition = position
  800. let mainSafeEnd = safeEnd
  801. let mainStart = start
  802. target = keysTarget
  803. position = 0
  804. start = 0
  805. if (!target)
  806. keysTarget = target = new ByteArrayAllocate(8192)
  807. safeEnd = target.length - 10
  808. newRecord(transition, keys, newTransitions)
  809. keysTarget = target
  810. let keysPosition = position
  811. target = mainTarget
  812. position = mainPosition
  813. safeEnd = mainSafeEnd
  814. start = mainStart
  815. if (keysPosition > 1) {
  816. let newEnd = position + keysPosition - 1
  817. if (newEnd > safeEnd)
  818. makeRoom(newEnd)
  819. let insertionPosition = insertionOffset + start
  820. target.copyWithin(insertionPosition + keysPosition, insertionPosition + 1, position)
  821. target.set(keysTarget.slice(0, keysPosition), insertionPosition)
  822. position = newEnd
  823. } else {
  824. target[insertionOffset + start] = keysTarget[0]
  825. }
  826. }
  827. const writeStruct = (object) => {
  828. let newPosition = writeStructSlots(object, target, start, position, structures, makeRoom, (value, newPosition, notifySharedUpdate) => {
  829. if (notifySharedUpdate)
  830. return hasSharedUpdate = true;
  831. position = newPosition;
  832. let startTarget = target;
  833. pack(value);
  834. resetStructures();
  835. if (startTarget !== target) {
  836. return { position, targetView, target }; // indicate the buffer was re-allocated
  837. }
  838. return position;
  839. }, this);
  840. if (newPosition === 0) // bail and go to a msgpack object
  841. return writeObject(object);
  842. position = newPosition;
  843. }
  844. }
  845. useBuffer(buffer) {
  846. // this means we are finished using our own buffer and we can write over it safely
  847. target = buffer
  848. target.dataView || (target.dataView = new DataView(target.buffer, target.byteOffset, target.byteLength))
  849. targetView = target.dataView;
  850. position = 0
  851. }
  852. set position (value) {
  853. position = value;
  854. }
  855. get position() {
  856. return position;
  857. }
  858. clearSharedData() {
  859. if (this.structures)
  860. this.structures = []
  861. if (this.typedStructs)
  862. this.typedStructs = []
  863. }
  864. }
  865. extensionClasses = [ Date, Set, Error, RegExp, ArrayBuffer, Object.getPrototypeOf(Uint8Array.prototype).constructor /*TypedArray*/, DataView, C1Type ]
  866. extensions = [{
  867. pack(date, allocateForWrite, pack) {
  868. let seconds = date.getTime() / 1000
  869. if ((this.useTimestamp32 || date.getMilliseconds() === 0) && seconds >= 0 && seconds < 0x100000000) {
  870. // Timestamp 32
  871. let { target, targetView, position} = allocateForWrite(6)
  872. target[position++] = 0xd6
  873. target[position++] = 0xff
  874. targetView.setUint32(position, seconds)
  875. } else if (seconds > 0 && seconds < 0x100000000) {
  876. // Timestamp 64
  877. let { target, targetView, position} = allocateForWrite(10)
  878. target[position++] = 0xd7
  879. target[position++] = 0xff
  880. targetView.setUint32(position, date.getMilliseconds() * 4000000 + ((seconds / 1000 / 0x100000000) >> 0))
  881. targetView.setUint32(position + 4, seconds)
  882. } else if (isNaN(seconds)) {
  883. if (this.onInvalidDate) {
  884. allocateForWrite(0)
  885. return pack(this.onInvalidDate())
  886. }
  887. // Intentionally invalid timestamp
  888. let { target, targetView, position} = allocateForWrite(3)
  889. target[position++] = 0xd4
  890. target[position++] = 0xff
  891. target[position++] = 0xff
  892. } else {
  893. // Timestamp 96
  894. let { target, targetView, position} = allocateForWrite(15)
  895. target[position++] = 0xc7
  896. target[position++] = 12
  897. target[position++] = 0xff
  898. targetView.setUint32(position, date.getMilliseconds() * 1000000)
  899. targetView.setBigInt64(position + 4, BigInt(Math.floor(seconds)))
  900. }
  901. }
  902. }, {
  903. pack(set, allocateForWrite, pack) {
  904. if (this.setAsEmptyObject) {
  905. allocateForWrite(0);
  906. return pack({})
  907. }
  908. let array = Array.from(set)
  909. let { target, position} = allocateForWrite(this.moreTypes ? 3 : 0)
  910. if (this.moreTypes) {
  911. target[position++] = 0xd4
  912. target[position++] = 0x73 // 's' for Set
  913. target[position++] = 0
  914. }
  915. pack(array)
  916. }
  917. }, {
  918. pack(error, allocateForWrite, pack) {
  919. let { target, position} = allocateForWrite(this.moreTypes ? 3 : 0)
  920. if (this.moreTypes) {
  921. target[position++] = 0xd4
  922. target[position++] = 0x65 // 'e' for error
  923. target[position++] = 0
  924. }
  925. pack([ error.name, error.message, error.cause ])
  926. }
  927. }, {
  928. pack(regex, allocateForWrite, pack) {
  929. let { target, position} = allocateForWrite(this.moreTypes ? 3 : 0)
  930. if (this.moreTypes) {
  931. target[position++] = 0xd4
  932. target[position++] = 0x78 // 'x' for regeXp
  933. target[position++] = 0
  934. }
  935. pack([ regex.source, regex.flags ])
  936. }
  937. }, {
  938. pack(arrayBuffer, allocateForWrite) {
  939. if (this.moreTypes)
  940. writeExtBuffer(arrayBuffer, 0x10, allocateForWrite)
  941. else
  942. writeBuffer(hasNodeBuffer ? Buffer.from(arrayBuffer) : new Uint8Array(arrayBuffer), allocateForWrite)
  943. }
  944. }, {
  945. pack(typedArray, allocateForWrite) {
  946. let constructor = typedArray.constructor
  947. if (constructor !== ByteArray && this.moreTypes)
  948. writeExtBuffer(typedArray, typedArrays.indexOf(constructor.name), allocateForWrite)
  949. else
  950. writeBuffer(typedArray, allocateForWrite)
  951. }
  952. }, {
  953. pack(arrayBuffer, allocateForWrite) {
  954. if (this.moreTypes)
  955. writeExtBuffer(arrayBuffer, 0x11, allocateForWrite)
  956. else
  957. writeBuffer(hasNodeBuffer ? Buffer.from(arrayBuffer) : new Uint8Array(arrayBuffer), allocateForWrite)
  958. }
  959. }, {
  960. pack(c1, allocateForWrite) { // specific 0xC1 object
  961. let { target, position} = allocateForWrite(1)
  962. target[position] = 0xc1
  963. }
  964. }]
  965. function writeExtBuffer(typedArray, type, allocateForWrite, encode) {
  966. let length = typedArray.byteLength
  967. if (length + 1 < 0x100) {
  968. var { target, position } = allocateForWrite(4 + length)
  969. target[position++] = 0xc7
  970. target[position++] = length + 1
  971. } else if (length + 1 < 0x10000) {
  972. var { target, position } = allocateForWrite(5 + length)
  973. target[position++] = 0xc8
  974. target[position++] = (length + 1) >> 8
  975. target[position++] = (length + 1) & 0xff
  976. } else {
  977. var { target, position, targetView } = allocateForWrite(7 + length)
  978. target[position++] = 0xc9
  979. targetView.setUint32(position, length + 1) // plus one for the type byte
  980. position += 4
  981. }
  982. target[position++] = 0x74 // "t" for typed array
  983. target[position++] = type
  984. if (!typedArray.buffer) typedArray = new Uint8Array(typedArray)
  985. target.set(new Uint8Array(typedArray.buffer, typedArray.byteOffset, typedArray.byteLength), position)
  986. }
  987. function writeBuffer(buffer, allocateForWrite) {
  988. let length = buffer.byteLength
  989. var target, position
  990. if (length < 0x100) {
  991. var { target, position } = allocateForWrite(length + 2)
  992. target[position++] = 0xc4
  993. target[position++] = length
  994. } else if (length < 0x10000) {
  995. var { target, position } = allocateForWrite(length + 3)
  996. target[position++] = 0xc5
  997. target[position++] = length >> 8
  998. target[position++] = length & 0xff
  999. } else {
  1000. var { target, position, targetView } = allocateForWrite(length + 5)
  1001. target[position++] = 0xc6
  1002. targetView.setUint32(position, length)
  1003. position += 4
  1004. }
  1005. target.set(buffer, position)
  1006. }
  1007. function writeExtensionData(result, target, position, type) {
  1008. let length = result.length
  1009. switch (length) {
  1010. case 1:
  1011. target[position++] = 0xd4
  1012. break
  1013. case 2:
  1014. target[position++] = 0xd5
  1015. break
  1016. case 4:
  1017. target[position++] = 0xd6
  1018. break
  1019. case 8:
  1020. target[position++] = 0xd7
  1021. break
  1022. case 16:
  1023. target[position++] = 0xd8
  1024. break
  1025. default:
  1026. if (length < 0x100) {
  1027. target[position++] = 0xc7
  1028. target[position++] = length
  1029. } else if (length < 0x10000) {
  1030. target[position++] = 0xc8
  1031. target[position++] = length >> 8
  1032. target[position++] = length & 0xff
  1033. } else {
  1034. target[position++] = 0xc9
  1035. target[position++] = length >> 24
  1036. target[position++] = (length >> 16) & 0xff
  1037. target[position++] = (length >> 8) & 0xff
  1038. target[position++] = length & 0xff
  1039. }
  1040. }
  1041. target[position++] = type
  1042. target.set(result, position)
  1043. position += length
  1044. return position
  1045. }
  1046. function insertIds(serialized, idsToInsert) {
  1047. // insert the ids that need to be referenced for structured clones
  1048. let nextId
  1049. let distanceToMove = idsToInsert.length * 6
  1050. let lastEnd = serialized.length - distanceToMove
  1051. while (nextId = idsToInsert.pop()) {
  1052. let offset = nextId.offset
  1053. let id = nextId.id
  1054. serialized.copyWithin(offset + distanceToMove, offset, lastEnd)
  1055. distanceToMove -= 6
  1056. let position = offset + distanceToMove
  1057. serialized[position++] = 0xd6
  1058. serialized[position++] = 0x69 // 'i'
  1059. serialized[position++] = id >> 24
  1060. serialized[position++] = (id >> 16) & 0xff
  1061. serialized[position++] = (id >> 8) & 0xff
  1062. serialized[position++] = id & 0xff
  1063. lastEnd = offset
  1064. }
  1065. return serialized
  1066. }
  1067. function writeBundles(start, pack, incrementPosition) {
  1068. if (bundledStrings.length > 0) {
  1069. targetView.setUint32(bundledStrings.position + start, position + incrementPosition - bundledStrings.position - start)
  1070. bundledStrings.stringsPosition = position - start;
  1071. let writeStrings = bundledStrings
  1072. bundledStrings = null
  1073. pack(writeStrings[0])
  1074. pack(writeStrings[1])
  1075. }
  1076. }
  1077. export function addExtension(extension) {
  1078. if (extension.Class) {
  1079. if (!extension.pack && !extension.write)
  1080. throw new Error('Extension has no pack or write function')
  1081. if (extension.pack && !extension.type)
  1082. throw new Error('Extension has no type (numeric code to identify the extension)')
  1083. extensionClasses.unshift(extension.Class)
  1084. extensions.unshift(extension)
  1085. }
  1086. unpackAddExtension(extension)
  1087. }
  1088. function prepareStructures(structures, packr) {
  1089. structures.isCompatible = (existingStructures) => {
  1090. let compatible = !existingStructures || ((packr.lastNamedStructuresLength || 0) === existingStructures.length)
  1091. if (!compatible) // we want to merge these existing structures immediately since we already have it and we are in the right transaction
  1092. packr._mergeStructures(existingStructures);
  1093. return compatible;
  1094. }
  1095. return structures
  1096. }
  1097. export function setWriteStructSlots(writeSlots, makeStructures) {
  1098. writeStructSlots = writeSlots;
  1099. prepareStructures = makeStructures;
  1100. }
  1101. let defaultPackr = new Packr({ useRecords: false })
  1102. export const pack = defaultPackr.pack
  1103. export const encode = defaultPackr.pack
  1104. export const Encoder = Packr
  1105. export { FLOAT32_OPTIONS } from './unpack.js'
  1106. import { FLOAT32_OPTIONS } from './unpack.js'
  1107. export const { NEVER, ALWAYS, DECIMAL_ROUND, DECIMAL_FIT } = FLOAT32_OPTIONS
  1108. export const REUSE_BUFFER_MODE = 512
  1109. export const RESET_BUFFER_MODE = 1024
  1110. export const RESERVE_START_SPACE = 2048