Initial commit

This commit is contained in:
2026-04-15 17:08:39 +02:00
parent ae164c47a8
commit 47fd1c2b7a
1819 changed files with 685388 additions and 0 deletions

View File

@@ -0,0 +1,598 @@
import { createFastPforWireDecodeWorkspace, decodeFastPforInt32, ensureFastPforWireEncodedWordsCapacity, } from "./fastPforDecoder";
import { decodeBigEndianInt32sInto } from "./bigEndianDecode";
export { createFastPforWireDecodeWorkspace } from "./fastPforDecoder";
//based on https://github.com/mapbox/pbf/blob/main/index.js
export function decodeVarintInt32(buf, bufferOffset, numValues) {
const dst = new Uint32Array(numValues);
let dstOffset = 0;
let offset = bufferOffset.get();
for (let i = 0; i < dst.length; i++) {
let b = buf[offset++];
let val = b & 0x7f;
if (b < 0x80) {
dst[dstOffset++] = val;
continue;
}
b = buf[offset++];
val |= (b & 0x7f) << 7;
if (b < 0x80) {
dst[dstOffset++] = val;
continue;
}
b = buf[offset++];
val |= (b & 0x7f) << 14;
if (b < 0x80) {
dst[dstOffset++] = val;
continue;
}
b = buf[offset++];
val |= (b & 0x7f) << 21;
if (b < 0x80) {
dst[dstOffset++] = val;
continue;
}
b = buf[offset++];
val |= (b & 0x0f) << 28;
dst[dstOffset++] = val;
}
bufferOffset.set(offset);
return dst;
}
export function decodeVarintInt64(src, offset, numValues) {
const dst = new BigUint64Array(numValues);
for (let i = 0; i < dst.length; i++) {
dst[i] = decodeVarintInt64Value(src, offset);
}
return dst;
}
// Source: https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/util/VarInt.java
function decodeVarintInt64Value(bytes, pos) {
let value = 0n;
let shift = 0;
let index = pos.get();
while (index < bytes.length) {
const b = bytes[index++];
value |= BigInt(b & 0x7f) << BigInt(shift);
if ((b & 0x80) === 0) {
break;
}
shift += 7;
if (shift >= 64) {
throw new Error("Varint too long");
}
}
pos.set(index);
return value;
}
/*
* Since decoding Int64 values to BigInt is more than an order of magnitude slower in the tests then using a Float64,
* this decoding method limits the max size of a Long value to 53 bits
*/
export function decodeVarintFloat64(src, offset, numValues) {
const dst = new Float64Array(numValues);
for (let i = 0; i < numValues; i++) {
dst[i] = decodeVarintFloat64Value(src, offset);
}
return dst;
}
//based on https://github.com/mapbox/pbf/blob/main/index.js
function decodeVarintFloat64Value(buf, offset) {
let val;
let b;
b = buf[offset.get()];
offset.increment();
val = b & 0x7f;
if (b < 0x80)
return val;
b = buf[offset.get()];
offset.increment();
val |= (b & 0x7f) << 7;
if (b < 0x80)
return val;
b = buf[offset.get()];
offset.increment();
val |= (b & 0x7f) << 14;
if (b < 0x80)
return val;
b = buf[offset.get()];
offset.increment();
val |= (b & 0x7f) << 21;
if (b < 0x80)
return val;
b = buf[offset.get()];
val |= (b & 0x0f) << 28;
return decodeVarintRemainder(val, buf, offset);
}
function decodeVarintRemainder(l, buf, offset) {
let h;
let b;
b = buf[offset.get()];
offset.increment();
h = (b & 0x70) >> 4;
if (b < 0x80)
return h * 0x100000000 + (l >>> 0);
b = buf[offset.get()];
offset.increment();
h |= (b & 0x7f) << 3;
if (b < 0x80)
return h * 0x100000000 + (l >>> 0);
b = buf[offset.get()];
offset.increment();
h |= (b & 0x7f) << 10;
if (b < 0x80)
return h * 0x100000000 + (l >>> 0);
b = buf[offset.get()];
offset.increment();
h |= (b & 0x7f) << 17;
if (b < 0x80)
return h * 0x100000000 + (l >>> 0);
b = buf[offset.get()];
offset.increment();
h |= (b & 0x7f) << 24;
if (b < 0x80)
return h * 0x100000000 + (l >>> 0);
b = buf[offset.get()];
offset.increment();
h |= (b & 0x01) << 31;
if (b < 0x80)
return h * 0x100000000 + (l >>> 0);
throw new Error("Expected varint not more than 10 bytes");
}
export function decodeFastPfor(encodedBytes, expectedValueCount, encodedByteLength, offset) {
const workspace = createFastPforWireDecodeWorkspace(encodedByteLength >>> 2);
return decodeFastPforWithWorkspace(encodedBytes, expectedValueCount, encodedByteLength, offset, workspace);
}
export function decodeFastPforWithWorkspace(encodedBytes, expectedValueCount, encodedByteLength, offset, workspace) {
const inputByteOffset = offset.get();
if ((encodedByteLength & 3) !== 0) {
throw new Error(`FastPFOR: invalid encodedByteLength=${encodedByteLength} at offset=${inputByteOffset} (encodedBytes.length=${encodedBytes.length}; expected a multiple of 4 bytes for an int32 big-endian word stream)`);
}
const encodedWordCount = encodedByteLength >>> 2;
const encodedWordBuffer = ensureFastPforWireEncodedWordsCapacity(workspace, encodedWordCount);
decodeBigEndianInt32sInto(encodedBytes, inputByteOffset, encodedByteLength, encodedWordBuffer);
const decodedValues = decodeFastPforInt32(encodedWordBuffer.subarray(0, encodedWordCount), expectedValueCount, workspace.decoderWorkspace);
offset.add(encodedByteLength);
return decodedValues;
}
export function decodeZigZagInt32Value(encoded) {
return (encoded >>> 1) ^ -(encoded & 1);
}
export function decodeZigZagInt64Value(encoded) {
return (encoded >> 1n) ^ -(encoded & 1n);
}
export function decodeZigZagFloat64Value(encoded) {
return encoded % 2 === 1 ? (encoded + 1) / -2 : encoded / 2;
}
export function decodeZigZagInt32(encodedData) {
const decodedValues = new Int32Array(encodedData.length);
for (let i = 0; i < encodedData.length; i++) {
decodedValues[i] = decodeZigZagInt32Value(encodedData[i]);
}
return decodedValues;
}
export function decodeZigZagInt64(encodedData) {
const decodedValues = new BigInt64Array(encodedData.length);
for (let i = 0; i < encodedData.length; i++) {
decodedValues[i] = decodeZigZagInt64Value(encodedData[i]);
}
return decodedValues;
}
export function decodeZigZagFloat64(encodedData) {
for (let i = 0; i < encodedData.length; i++) {
encodedData[i] = decodeZigZagFloat64Value(encodedData[i]);
}
}
export function decodeUnsignedRleInt32(encodedData, numRuns, numTotalValues) {
// If numTotalValues not provided, calculate from runs (nullable case)
if (numTotalValues === undefined) {
numTotalValues = 0;
for (let i = 0; i < numRuns; i++) {
numTotalValues += encodedData[i];
}
}
const decodedValues = new Uint32Array(numTotalValues);
let offset = 0;
for (let i = 0; i < numRuns; i++) {
const runLength = encodedData[i];
const value = encodedData[i + numRuns];
decodedValues.fill(value, offset, offset + runLength);
offset += runLength;
}
return decodedValues;
}
export function decodeUnsignedRleInt64(encodedData, numRuns, numTotalValues) {
// If numTotalValues not provided, calculate from runs (nullable case)
if (numTotalValues === undefined) {
numTotalValues = 0;
for (let i = 0; i < numRuns; i++) {
numTotalValues += Number(encodedData[i]);
}
}
const decodedValues = new BigUint64Array(numTotalValues);
let offset = 0;
for (let i = 0; i < numRuns; i++) {
const runLength = Number(encodedData[i]);
const value = encodedData[i + numRuns];
decodedValues.fill(value, offset, offset + runLength);
offset += runLength;
}
return decodedValues;
}
export function decodeUnsignedRleFloat64(encodedData, numRuns, numTotalValues) {
const decodedValues = new Float64Array(numTotalValues);
let offset = 0;
for (let i = 0; i < numRuns; i++) {
const runLength = encodedData[i];
const value = encodedData[i + numRuns];
decodedValues.fill(value, offset, offset + runLength);
offset += runLength;
}
return decodedValues;
}
/*
* In place decoding of the zigzag encoded delta values.
* Inspired by https://github.com/lemire/JavaFastPFOR/blob/master/src/main/java/me/lemire/integercompression/differential/Delta.java
*/
export function decodeZigZagDeltaInt32(data) {
const decodedValues = new Int32Array(data.length);
decodedValues[0] = decodeZigZagInt32Value(data[0]);
const sz0 = (data.length / 4) * 4;
let i = 1;
if (sz0 >= 4) {
for (; i < sz0 - 4; i += 4) {
const data1 = data[i];
const data2 = data[i + 1];
const data3 = data[i + 2];
const data4 = data[i + 3];
decodedValues[i] = decodeZigZagInt32Value(data1) + decodedValues[i - 1];
decodedValues[i + 1] = decodeZigZagInt32Value(data2) + decodedValues[i];
decodedValues[i + 2] = decodeZigZagInt32Value(data3) + decodedValues[i + 1];
decodedValues[i + 3] = decodeZigZagInt32Value(data4) + decodedValues[i + 2];
}
}
for (; i !== data.length; ++i) {
decodedValues[i] = decodeZigZagInt32Value(data[i]) + decodedValues[i - 1];
}
return decodedValues;
}
export function decodeZigZagDeltaInt64(data) {
const decodedValues = new BigInt64Array(data.length);
decodedValues[0] = decodeZigZagInt64Value(data[0]);
const sz0 = (data.length / 4) * 4;
let i = 1;
if (sz0 >= 4) {
for (; i < sz0 - 4; i += 4) {
const data1 = data[i];
const data2 = data[i + 1];
const data3 = data[i + 2];
const data4 = data[i + 3];
decodedValues[i] = decodeZigZagInt64Value(data1) + decodedValues[i - 1];
decodedValues[i + 1] = decodeZigZagInt64Value(data2) + decodedValues[i];
decodedValues[i + 2] = decodeZigZagInt64Value(data3) + decodedValues[i + 1];
decodedValues[i + 3] = decodeZigZagInt64Value(data4) + decodedValues[i + 2];
}
}
for (; i !== decodedValues.length; ++i) {
decodedValues[i] = decodeZigZagInt64Value(data[i]) + decodedValues[i - 1];
}
return decodedValues;
}
export function decodeZigZagDeltaFloat64(data) {
data[0] = decodeZigZagFloat64Value(data[0]);
const sz0 = (data.length / 4) * 4;
let i = 1;
if (sz0 >= 4) {
for (; i < sz0 - 4; i += 4) {
const data1 = data[i];
const data2 = data[i + 1];
const data3 = data[i + 2];
const data4 = data[i + 3];
data[i] = decodeZigZagFloat64Value(data1) + data[i - 1];
data[i + 1] = decodeZigZagFloat64Value(data2) + data[i];
data[i + 2] = decodeZigZagFloat64Value(data3) + data[i + 1];
data[i + 3] = decodeZigZagFloat64Value(data4) + data[i + 2];
}
}
for (; i !== data.length; ++i) {
data[i] = decodeZigZagFloat64Value(data[i]) + data[i - 1];
}
}
export function decodeZigZagRleInt32(data, numRuns, numTotalValues) {
// If numTotalValues not provided, calculate from runs (nullable case)
if (numTotalValues === undefined) {
numTotalValues = 0;
for (let i = 0; i < numRuns; i++) {
numTotalValues += data[i];
}
}
const decodedValues = new Int32Array(numTotalValues);
let offset = 0;
for (let i = 0; i < numRuns; i++) {
const runLength = data[i];
let value = data[i + numRuns];
value = decodeZigZagInt32Value(value);
decodedValues.fill(value, offset, offset + runLength);
offset += runLength;
}
return decodedValues;
}
export function decodeZigZagRleInt64(data, numRuns, numTotalValues) {
// If numTotalValues not provided, calculate from runs (nullable case)
if (numTotalValues === undefined) {
numTotalValues = 0;
for (let i = 0; i < numRuns; i++) {
numTotalValues += Number(data[i]);
}
}
const decodedValues = new BigInt64Array(numTotalValues);
let offset = 0;
for (let i = 0; i < numRuns; i++) {
const runLength = Number(data[i]);
let value = data[i + numRuns];
value = decodeZigZagInt64Value(value);
decodedValues.fill(value, offset, offset + runLength);
offset += runLength;
}
return decodedValues;
}
export function decodeZigZagRleFloat64(data, numRuns, numTotalValues) {
const decodedValues = new Float64Array(numTotalValues);
let offset = 0;
for (let i = 0; i < numRuns; i++) {
const runLength = data[i];
let value = data[i + numRuns];
value = decodeZigZagFloat64Value(value);
decodedValues.fill(value, offset, offset + runLength);
offset += runLength;
}
return decodedValues;
}
/*
* Inspired by https://github.com/lemire/JavaFastPFOR/blob/master/src/main/java/me/lemire/integercompression/differential/Delta.java
*/
export function fastInverseDelta(data) {
const sz0 = (data.length / 4) * 4;
let i = 1;
if (sz0 >= 4) {
for (let a = data[0]; i < sz0 - 4; i += 4) {
a = data[i] += a;
a = data[i + 1] += a;
a = data[i + 2] += a;
a = data[i + 3] += a;
}
}
while (i !== data.length) {
data[i] += data[i - 1];
++i;
}
}
export function inverseDelta(data) {
let prevValue = 0;
for (let i = 0; i < data.length; i++) {
data[i] += prevValue;
prevValue = data[i];
}
}
/*
* In place decoding of the zigzag delta encoded Vec2.
* Inspired by https://github.com/lemire/JavaFastPFOR/blob/master/src/main/java/me/lemire/integercompression/differential/Delta.java
*/
export function decodeComponentwiseDeltaVec2(data) {
if (data.length < 2)
return new Int32Array(data);
const decodedData = new Int32Array(data.length);
decodedData[0] = decodeZigZagInt32Value(data[0]);
decodedData[1] = decodeZigZagInt32Value(data[1]);
const sz0 = (data.length / 4) * 4;
let i = 2;
if (sz0 >= 4) {
for (; i < sz0 - 4; i += 4) {
const x1 = data[i];
const y1 = data[i + 1];
const x2 = data[i + 2];
const y2 = data[i + 3];
decodedData[i] = decodeZigZagInt32Value(x1) + decodedData[i - 2];
decodedData[i + 1] = decodeZigZagInt32Value(y1) + decodedData[i - 1];
decodedData[i + 2] = decodeZigZagInt32Value(x2) + decodedData[i];
decodedData[i + 3] = decodeZigZagInt32Value(y2) + decodedData[i + 1];
}
}
for (; i !== data.length; i += 2) {
decodedData[i] = decodeZigZagInt32Value(data[i]) + decodedData[i - 2];
decodedData[i + 1] = decodeZigZagInt32Value(data[i + 1]) + decodedData[i - 1];
}
return decodedData;
}
export function decodeComponentwiseDeltaVec2Scaled(data, scale, min, max) {
if (data.length < 2)
return new Int32Array(data);
const decodedData = new Int32Array(data.length);
let previousVertexX = decodeZigZagInt32Value(data[0]);
let previousVertexY = decodeZigZagInt32Value(data[1]);
decodedData[0] = clamp(Math.round(previousVertexX * scale), min, max);
decodedData[1] = clamp(Math.round(previousVertexY * scale), min, max);
const sz0 = data.length / 16;
let i = 2;
if (sz0 >= 4) {
for (; i < sz0 - 4; i += 4) {
const x1 = data[i];
const y1 = data[i + 1];
const currentVertexX = decodeZigZagInt32Value(x1) + previousVertexX;
const currentVertexY = decodeZigZagInt32Value(y1) + previousVertexY;
decodedData[i] = clamp(Math.round(currentVertexX * scale), min, max);
decodedData[i + 1] = clamp(Math.round(currentVertexY * scale), min, max);
const x2 = data[i + 2];
const y2 = data[i + 3];
previousVertexX = decodeZigZagInt32Value(x2) + currentVertexX;
previousVertexY = decodeZigZagInt32Value(y2) + currentVertexY;
decodedData[i + 2] = clamp(Math.round(previousVertexX * scale), min, max);
decodedData[i + 3] = clamp(Math.round(previousVertexY * scale), min, max);
}
}
for (; i !== data.length; i += 2) {
previousVertexX += decodeZigZagInt32Value(data[i]);
previousVertexY += decodeZigZagInt32Value(data[i + 1]);
decodedData[i] = clamp(Math.round(previousVertexX * scale), min, max);
decodedData[i + 1] = clamp(Math.round(previousVertexY * scale), min, max);
}
return decodedData;
}
function clamp(n, min, max) {
return Math.min(max, Math.max(min, n));
}
/* Transform data to allow util access ------------------------------------------------------------------------ */
export function decodeZigZagDeltaOfDeltaInt32(data) {
const decodedData = new Int32Array(data.length + 1);
decodedData[0] = 0;
decodedData[1] = decodeZigZagInt32Value(data[0]);
let deltaSum = decodedData[1];
for (let i = 2; i !== decodedData.length; ++i) {
const zigZagValue = data[i - 1];
const delta = decodeZigZagInt32Value(zigZagValue);
deltaSum += delta;
decodedData[i] = decodedData[i - 1] + deltaSum;
}
return new Uint32Array(decodedData);
}
export function decodeZigZagRleDeltaInt32(data, numRuns, numTotalValues) {
const decodedValues = new Int32Array(numTotalValues + 1);
decodedValues[0] = 0;
let offset = 1;
let previousValue = decodedValues[0];
for (let i = 0; i < numRuns; i++) {
const runLength = data[i];
let value = data[i + numRuns];
value = decodeZigZagInt32Value(value);
for (let j = offset; j < offset + runLength; j++) {
decodedValues[j] = value + previousValue;
previousValue = decodedValues[j];
}
offset += runLength;
}
return decodedValues;
}
export function decodeRleDeltaInt32(data, numRuns, numTotalValues) {
const decodedValues = new Uint32Array(numTotalValues + 1);
decodedValues[0] = 0;
let offset = 1;
let previousValue = decodedValues[0];
for (let i = 0; i < numRuns; i++) {
const runLength = data[i];
const value = data[i + numRuns];
for (let j = offset; j < offset + runLength; j++) {
decodedValues[j] = value + previousValue;
previousValue = decodedValues[j];
}
offset += runLength;
}
return decodedValues;
}
/**
* Decode Delta-RLE with multiple runs by fully reconstructing values.
*
* @param data RLE encoded data: [run1, run2, ..., value1, value2, ...]
* @param numRuns Number of runs in the RLE encoding
* @param numValues Total number of values to reconstruct
* @returns Reconstructed values with deltas applied
*/
export function decodeDeltaRleInt32(data, numRuns, numValues) {
const result = new Int32Array(numValues);
let outPos = 0;
let previousValue = 0;
for (let i = 0; i < numRuns; i++) {
const runLength = data[i];
const zigZagDelta = data[i + numRuns];
const delta = decodeZigZagInt32Value(zigZagDelta);
for (let j = 0; j < runLength; j++) {
previousValue += delta;
result[outPos++] = previousValue;
}
}
return result;
}
/**
* Decode Delta-RLE with multiple runs for 64-bit integers.
*/
export function decodeDeltaRleInt64(data, numRuns, numValues) {
const result = new BigInt64Array(numValues);
let outPos = 0;
let previousValue = 0n;
for (let i = 0; i < numRuns; i++) {
const runLength = Number(data[i]);
const zigZagDelta = data[i + numRuns];
const delta = decodeZigZagInt64Value(zigZagDelta);
for (let j = 0; j < runLength; j++) {
previousValue += delta;
result[outPos++] = previousValue;
}
}
return result;
}
export function decodeUnsignedZigZagDeltaInt32(data) {
const decodedValues = new Uint32Array(data.length);
decodedValues[0] = decodeZigZagInt32Value(data[0]) >>> 0;
for (let i = 1; i < data.length; i++) {
decodedValues[i] = (decodedValues[i - 1] + decodeZigZagInt32Value(data[i])) >>> 0;
}
return decodedValues;
}
export function decodeUnsignedZigZagDeltaInt64(data) {
const decodedValues = new BigUint64Array(data.length);
decodedValues[0] = BigInt.asUintN(64, decodeZigZagInt64Value(data[0]));
for (let i = 1; i < data.length; i++) {
decodedValues[i] = BigInt.asUintN(64, decodedValues[i - 1] + decodeZigZagInt64Value(data[i]));
}
return decodedValues;
}
export function decodeUnsignedComponentwiseDeltaVec2(data) {
if (data.length < 2) {
return new Uint32Array(data);
}
const decodedData = new Uint32Array(data.length);
decodedData[0] = decodeZigZagInt32Value(data[0]) >>> 0;
decodedData[1] = decodeZigZagInt32Value(data[1]) >>> 0;
for (let i = 2; i < data.length; i += 2) {
decodedData[i] = (decodedData[i - 2] + decodeZigZagInt32Value(data[i])) >>> 0;
decodedData[i + 1] = (decodedData[i - 1] + decodeZigZagInt32Value(data[i + 1])) >>> 0;
}
return decodedData;
}
export function decodeUnsignedComponentwiseDeltaVec2Scaled(data, scale, min, max) {
const scaledValues = decodeComponentwiseDeltaVec2Scaled(data, scale, min, max);
return new Uint32Array(scaledValues);
}
export function decodeUnsignedConstRleInt32(data) {
return data[1];
}
export function decodeZigZagConstRleInt32(data) {
return decodeZigZagInt32Value(data[1]);
}
export function decodeZigZagSequenceRleInt32(data) {
/* base value and delta value are equal */
if (data.length === 2) {
const value = decodeZigZagInt32Value(data[1]);
return [value, value];
}
/* base value and delta value are not equal -> 2 runs and 2 values*/
const base = decodeZigZagInt32Value(data[2]);
const delta = decodeZigZagInt32Value(data[3]);
return [base, delta];
}
export function decodeUnsignedConstRleInt64(data) {
return data[1];
}
export function decodeZigZagConstRleInt64(data) {
return decodeZigZagInt64Value(data[1]);
}
export function decodeZigZagSequenceRleInt64(data) {
/* base value and delta value are equal */
if (data.length === 2) {
const value = decodeZigZagInt64Value(data[1]);
return [value, value];
}
/* base value and delta value are not equal -> 2 runs and 2 values*/
const base = decodeZigZagInt64Value(data[2]);
const delta = decodeZigZagInt64Value(data[3]);
return [base, delta];
}
//# sourceMappingURL=integerDecodingUtils.js.map