import { PhysicalLevelTechnique } from "../metadata/tile/physicalLevelTechnique"; import IntWrapper from "./intWrapper"; import { decodeComponentwiseDeltaVec2, decodeComponentwiseDeltaVec2Scaled, decodeDeltaRleInt32, decodeDeltaRleInt64, decodeFastPfor, decodeUnsignedComponentwiseDeltaVec2, decodeUnsignedComponentwiseDeltaVec2Scaled, decodeUnsignedConstRleInt32, decodeUnsignedConstRleInt64, decodeUnsignedRleInt32, decodeUnsignedRleInt64, decodeUnsignedRleFloat64, decodeUnsignedZigZagDeltaInt32, decodeUnsignedZigZagDeltaInt64, decodeVarintInt32, decodeVarintInt64, decodeVarintFloat64, decodeZigZagInt32, decodeZigZagInt64, decodeZigZagFloat64, decodeZigZagConstRleInt32, decodeZigZagConstRleInt64, decodeZigZagDeltaInt32, decodeZigZagDeltaInt64, decodeZigZagDeltaFloat64, decodeZigZagSequenceRleInt32, decodeZigZagSequenceRleInt64, decodeZigZagInt32Value, decodeZigZagInt64Value, fastInverseDelta, inverseDelta, decodeRleDeltaInt32, decodeZigZagDeltaOfDeltaInt32, decodeZigZagRleDeltaInt32, decodeZigZagRleInt32, decodeZigZagRleInt64, decodeZigZagRleFloat64, } from "./integerDecodingUtils"; import { LogicalLevelTechnique } from "../metadata/tile/logicalLevelTechnique"; import BitVector from "../vector/flat/bitVector"; import { VectorType } from "../vector/vectorType"; import { unpackNullable } from "./unpackNullableUtils"; export function decodeSignedInt32Stream(data, offset, streamMetadata, scalingData, nullabilityBuffer) { const values = decodePhysicalLevelTechnique(data, offset, streamMetadata); return decodeSignedInt32(values, streamMetadata, scalingData, nullabilityBuffer); } export function decodeUnsignedInt32Stream(data, offset, streamMetadata, scalingData, nullabilityBuffer) { const values = decodePhysicalLevelTechnique(data, offset, streamMetadata); return decodeUnsignedInt32(values, streamMetadata, scalingData, nullabilityBuffer); } export function decodeLengthStreamToOffsetBuffer(data, offset, streamMetadata) { const values = decodePhysicalLevelTechnique(data, offset, streamMetadata); return decodeLengthToOffsetBuffer(values, streamMetadata); } function decodePhysicalLevelTechnique(data, offset, streamMetadata) { const physicalLevelTechnique = streamMetadata.physicalLevelTechnique; switch (physicalLevelTechnique) { case PhysicalLevelTechnique.FAST_PFOR: return decodeFastPfor(data, streamMetadata.numValues, streamMetadata.byteLength, offset); case PhysicalLevelTechnique.VARINT: return decodeVarintInt32(data, offset, streamMetadata.numValues); case PhysicalLevelTechnique.NONE: { const dataOffset = offset.get(); const byteLength = streamMetadata.byteLength; offset.add(byteLength); const slice = data.subarray(dataOffset, offset.get()); return new Uint32Array(slice); } default: throw new Error(`Specified physicalLevelTechnique ${physicalLevelTechnique} is not supported (yet).`); } } export function decodeSignedConstInt32Stream(data, offset, streamMetadata) { const values = decodePhysicalLevelTechnique(data, offset, streamMetadata); if (values.length === 1) { return decodeZigZagInt32Value(values[0]); } return decodeZigZagConstRleInt32(values); } export function decodeUnsignedConstInt32Stream(data, offset, streamMetadata) { const values = decodePhysicalLevelTechnique(data, offset, streamMetadata); if (values.length === 1) { return values[0]; } return decodeUnsignedConstRleInt32(values); } export function decodeSequenceInt32Stream(data, offset, streamMetadata) { const values = decodePhysicalLevelTechnique(data, offset, streamMetadata); return decodeZigZagSequenceRleInt32(values); } export function decodeSequenceInt64Stream(data, offset, streamMetadata) { const values = decodeVarintInt64(data, offset, streamMetadata.numValues); return decodeZigZagSequenceRleInt64(values); } export function decodeSignedInt64Stream(data, offset, streamMetadata, nullabilityBuffer) { const values = decodeVarintInt64(data, offset, streamMetadata.numValues); return decodeSignedInt64(values, streamMetadata, nullabilityBuffer); } export function decodeUnsignedInt64Stream(data, offset, streamMetadata, nullabilityBuffer) { const values = decodeVarintInt64(data, offset, streamMetadata.numValues); return decodeUnsignedInt64(values, streamMetadata, nullabilityBuffer); } export function decodeSignedInt64AsFloat64Stream(data, offset, streamMetadata) { const values = decodeVarintFloat64(data, offset, streamMetadata.numValues); return decodeFloat64Values(values, streamMetadata, true); } export function decodeUnsignedInt64AsFloat64Stream(data, offset, streamMetadata) { const values = decodeVarintFloat64(data, offset, streamMetadata.numValues); return decodeFloat64Values(values, streamMetadata, false); } export function decodeSignedConstInt64Stream(data, offset, streamMetadata) { const values = decodeVarintInt64(data, offset, streamMetadata.numValues); if (values.length === 1) { return decodeZigZagInt64Value(values[0]); } return decodeZigZagConstRleInt64(values); } export function decodeUnsignedConstInt64Stream(data, offset, streamMetadata) { const values = decodeVarintInt64(data, offset, streamMetadata.numValues); if (values.length === 1) { return values[0]; } return decodeUnsignedConstRleInt64(values); } /** * This method decodes integer streams. * Currently the encoder uses only fixed combinations of encodings. * For performance reasons it is also uses a fixed combination of the encodings on the decoding side. * The following encodings and combinations are used: * - Morton Delta -> always sorted so not ZigZag encoding needed * - Delta -> currently always in combination with ZigZag encoding * - Rle -> in combination with ZigZag encoding if data type is signed * - Delta Rle * - Componentwise Delta -> always ZigZag encoding is used */ function decodeSignedInt32(values, streamMetadata, scalingData, nullabilityBuffer) { let decodedValues; switch (streamMetadata.logicalLevelTechnique1) { case LogicalLevelTechnique.DELTA: if (streamMetadata.logicalLevelTechnique2 === LogicalLevelTechnique.RLE) { const rleMetadata = streamMetadata; if (!nullabilityBuffer) { return decodeDeltaRleInt32(values, rleMetadata.runs, rleMetadata.numRleValues); } values = decodeUnsignedRleInt32(values, rleMetadata.runs, rleMetadata.numRleValues); decodedValues = decodeZigZagDeltaInt32(values); } else { decodedValues = decodeZigZagDeltaInt32(values); } break; case LogicalLevelTechnique.RLE: decodedValues = decodeZigZagRleInt32(values, streamMetadata.runs, streamMetadata.numRleValues); break; case LogicalLevelTechnique.MORTON: fastInverseDelta(values); decodedValues = new Int32Array(values); break; case LogicalLevelTechnique.COMPONENTWISE_DELTA: if (scalingData && !nullabilityBuffer) { return decodeComponentwiseDeltaVec2Scaled(values, scalingData.scale, scalingData.min, scalingData.max); } decodedValues = decodeComponentwiseDeltaVec2(values); break; case LogicalLevelTechnique.NONE: decodedValues = decodeZigZagInt32(values); break; default: throw new Error(`The specified Logical level technique is not supported: ${streamMetadata.logicalLevelTechnique1}`); } if (nullabilityBuffer) { return unpackNullable(decodedValues, nullabilityBuffer, 0); } return decodedValues; } function decodeUnsignedInt32(values, streamMetadata, scalingData, nullabilityBuffer) { let decodedValues; switch (streamMetadata.logicalLevelTechnique1) { case LogicalLevelTechnique.DELTA: if (streamMetadata.logicalLevelTechnique2 === LogicalLevelTechnique.RLE) { const rleMetadata = streamMetadata; const deltaValues = decodeUnsignedRleInt32(values, rleMetadata.runs, rleMetadata.numRleValues); decodedValues = decodeUnsignedZigZagDeltaInt32(deltaValues); } else { decodedValues = decodeUnsignedZigZagDeltaInt32(values); } break; case LogicalLevelTechnique.RLE: decodedValues = decodeUnsignedRleInt32(values, streamMetadata.runs, streamMetadata.numRleValues); break; case LogicalLevelTechnique.MORTON: fastInverseDelta(values); decodedValues = values; break; case LogicalLevelTechnique.COMPONENTWISE_DELTA: if (scalingData && !nullabilityBuffer) { decodedValues = decodeUnsignedComponentwiseDeltaVec2Scaled(values, scalingData.scale, scalingData.min, scalingData.max); } else { decodedValues = decodeUnsignedComponentwiseDeltaVec2(values); } break; case LogicalLevelTechnique.NONE: decodedValues = values; break; default: throw new Error(`The specified Logical level technique is not supported: ${streamMetadata.logicalLevelTechnique1}`); } if (nullabilityBuffer) { return unpackNullable(decodedValues, nullabilityBuffer, 0); } return decodedValues; } function decodeSignedInt64(values, streamMetadata, nullabilityBuffer) { let decodedValues; switch (streamMetadata.logicalLevelTechnique1) { case LogicalLevelTechnique.DELTA: if (streamMetadata.logicalLevelTechnique2 === LogicalLevelTechnique.RLE) { const rleMetadata = streamMetadata; if (!nullabilityBuffer) { return decodeDeltaRleInt64(values, rleMetadata.runs, rleMetadata.numRleValues); } values = decodeUnsignedRleInt64(values, rleMetadata.runs, rleMetadata.numRleValues); decodedValues = decodeZigZagDeltaInt64(values); } else { decodedValues = decodeZigZagDeltaInt64(values); } break; case LogicalLevelTechnique.RLE: decodedValues = decodeZigZagRleInt64(values, streamMetadata.runs, streamMetadata.numRleValues); break; case LogicalLevelTechnique.NONE: decodedValues = decodeZigZagInt64(values); break; default: throw new Error(`The specified Logical level technique is not supported: ${streamMetadata.logicalLevelTechnique1}`); } if (nullabilityBuffer) { return unpackNullable(decodedValues, nullabilityBuffer, 0n); } return decodedValues; } function decodeUnsignedInt64(values, streamMetadata, nullabilityBuffer) { let decodedValues; switch (streamMetadata.logicalLevelTechnique1) { case LogicalLevelTechnique.DELTA: if (streamMetadata.logicalLevelTechnique2 === LogicalLevelTechnique.RLE) { const rleMetadata = streamMetadata; const deltaValues = decodeUnsignedRleInt64(values, rleMetadata.runs, rleMetadata.numRleValues); decodedValues = decodeUnsignedZigZagDeltaInt64(deltaValues); } else { decodedValues = decodeUnsignedZigZagDeltaInt64(values); } break; case LogicalLevelTechnique.RLE: decodedValues = decodeUnsignedRleInt64(values, streamMetadata.runs, streamMetadata.numRleValues); break; case LogicalLevelTechnique.NONE: decodedValues = values; break; default: throw new Error(`The specified Logical level technique is not supported: ${streamMetadata.logicalLevelTechnique1}`); } if (nullabilityBuffer) { return unpackNullable(decodedValues, nullabilityBuffer, 0n); } return decodedValues; } function decodeFloat64Values(values, streamMetadata, isSigned) { switch (streamMetadata.logicalLevelTechnique1) { case LogicalLevelTechnique.DELTA: if (streamMetadata.logicalLevelTechnique2 === LogicalLevelTechnique.RLE) { const rleMetadata = streamMetadata; values = decodeUnsignedRleFloat64(values, rleMetadata.runs, rleMetadata.numRleValues); } decodeZigZagDeltaFloat64(values); return values; case LogicalLevelTechnique.RLE: return decodeRleFloat64(values, streamMetadata, isSigned); case LogicalLevelTechnique.NONE: if (isSigned) { decodeZigZagFloat64(values); } return values; default: throw new Error(`The specified Logical level technique is not supported: ${streamMetadata.logicalLevelTechnique1}`); } } function decodeLengthToOffsetBuffer(values, streamMetadata) { if (streamMetadata.logicalLevelTechnique1 === LogicalLevelTechnique.DELTA && streamMetadata.logicalLevelTechnique2 === LogicalLevelTechnique.NONE) { return decodeZigZagDeltaOfDeltaInt32(values); } if (streamMetadata.logicalLevelTechnique1 === LogicalLevelTechnique.RLE && streamMetadata.logicalLevelTechnique2 === LogicalLevelTechnique.NONE) { const rleMetadata = streamMetadata; return decodeRleDeltaInt32(values, rleMetadata.runs, rleMetadata.numRleValues); } if (streamMetadata.logicalLevelTechnique1 === LogicalLevelTechnique.NONE && streamMetadata.logicalLevelTechnique2 === LogicalLevelTechnique.NONE) { //TODO: use fastInverseDelta again and check what are the performance problems in zoom 14 //fastInverseDelta(values); inverseDelta(values); const offsets = new Uint32Array(streamMetadata.numValues + 1); offsets[0] = 0; offsets.set(values, 1); return offsets; } if (streamMetadata.logicalLevelTechnique1 === LogicalLevelTechnique.DELTA && streamMetadata.logicalLevelTechnique2 === LogicalLevelTechnique.RLE) { const rleMetadata = streamMetadata; const decodedValues = decodeZigZagRleDeltaInt32(values, rleMetadata.runs, rleMetadata.numRleValues); fastInverseDelta(decodedValues); return new Uint32Array(decodedValues); } throw new Error("Only delta encoding is supported for transforming length to offset streams yet."); } export function getVectorType(streamMetadata, sizeOrNullabilityBuffer, data, offset, varintWidth = "int32") { const logicalLevelTechnique1 = streamMetadata.logicalLevelTechnique1; if (logicalLevelTechnique1 === LogicalLevelTechnique.RLE) { return streamMetadata.runs === 1 ? VectorType.CONST : VectorType.FLAT; } if (logicalLevelTechnique1 !== LogicalLevelTechnique.DELTA || streamMetadata.logicalLevelTechnique2 !== LogicalLevelTechnique.RLE) { return streamMetadata.numValues === 1 ? VectorType.CONST : VectorType.FLAT; } const numFeatures = sizeOrNullabilityBuffer instanceof BitVector ? sizeOrNullabilityBuffer.size() : sizeOrNullabilityBuffer; const rleMetadata = streamMetadata; if (rleMetadata.numRleValues !== numFeatures) { return VectorType.FLAT; } // Single run is always a sequence if (rleMetadata.runs === 1) { return VectorType.SEQUENCE; } if (rleMetadata.runs !== 2) { return streamMetadata.numValues === 1 ? VectorType.CONST : VectorType.FLAT; } // Two runs can be a sequence if both deltas are equal to 1 const savedOffset = offset.get(); if (streamMetadata.physicalLevelTechnique === PhysicalLevelTechnique.VARINT) { if (isDeltaRleSequenceVarintWidth(data, offset, varintWidth)) { return VectorType.SEQUENCE; } return streamMetadata.numValues === 1 ? VectorType.CONST : VectorType.FLAT; } const byteOffset = offset.get(); const values = new Int32Array(data.buffer, data.byteOffset + byteOffset, 4); offset.set(savedOffset); // Check if both deltas are encoded 1 const zigZagOne = 2; if (values[2] === zigZagOne && values[3] === zigZagOne) { return VectorType.SEQUENCE; } return streamMetadata.numValues === 1 ? VectorType.CONST : VectorType.FLAT; } function isDeltaRleSequenceVarintWidth(data, offset, varintWidth) { const peekOffset = new IntWrapper(offset.get()); if (varintWidth === "int64") { const values = decodeVarintInt64(data, peekOffset, 4); return values[2] === 2n && values[3] === 2n; } const values = decodeVarintInt32(data, peekOffset, 4); return values[2] === 2 && values[3] === 2; } function decodeRleFloat64(data, streamMetadata, isSigned) { return isSigned ? decodeZigZagRleFloat64(data, streamMetadata.runs, streamMetadata.numRleValues) : decodeUnsignedRleFloat64(data, streamMetadata.runs, streamMetadata.numRleValues); } //# sourceMappingURL=integerStreamDecoder.js.map