import { PhysicalStreamType } from "../metadata/tile/physicalStreamType"; import { LogicalLevelTechnique } from "../metadata/tile/logicalLevelTechnique"; import { PhysicalLevelTechnique } from "../metadata/tile/physicalLevelTechnique"; import { DictionaryType } from "../metadata/tile/dictionaryType"; import { LengthType } from "../metadata/tile/lengthType"; import { OffsetType } from "../metadata/tile/offsetType"; import IntWrapper from "./intWrapper"; import { ComplexType, ScalarType } from "../metadata/tileset/tilesetMetadata"; import { encodeBooleanRle, encodeStrings, createStringLengths } from "../encoding/encodingUtils"; import { encodeVarintInt32Value, encodeVarintInt32 } from "../encoding/integerEncodingUtils"; /** * Creates basic stream metadata with logical techniques. */ export function createStreamMetadata(logicalTechnique1, logicalTechnique2 = LogicalLevelTechnique.NONE, numValues = 3) { return { physicalStreamType: PhysicalStreamType.DATA, logicalStreamType: { dictionaryType: DictionaryType.NONE }, logicalLevelTechnique1: logicalTechnique1, logicalLevelTechnique2: logicalTechnique2, physicalLevelTechnique: PhysicalLevelTechnique.VARINT, numValues, byteLength: 10, decompressedCount: numValues, }; } /** * Creates RLE-encoded stream metadata. */ export function createRleMetadata(logicalTechnique1, logicalTechnique2, runs, numRleValues) { return { physicalStreamType: PhysicalStreamType.DATA, logicalStreamType: { dictionaryType: DictionaryType.NONE }, logicalLevelTechnique1: logicalTechnique1, logicalLevelTechnique2: logicalTechnique2, physicalLevelTechnique: PhysicalLevelTechnique.VARINT, numValues: runs * 2, byteLength: 10, decompressedCount: numRleValues, runs, numRleValues, }; } /** * Creates column metadata for STRUCT type columns. */ export function createColumnMetadataForStruct(columnName, childFields) { const children = childFields.map((fieldConfig) => ({ name: fieldConfig.name, nullable: true, scalarField: { physicalType: fieldConfig.type ?? ScalarType.STRING, type: "physicalType", }, type: "scalarField", })); return { name: columnName, nullable: false, complexType: { physicalType: ComplexType.STRUCT, children, type: "physicalType", }, type: "complexType", }; } /** * Creates a single stream with metadata and data. */ export function createStream(physicalType, data, options = {}) { const count = options.count ?? 0; return buildEncodedStream({ physicalStreamType: physicalType, logicalStreamType: options.logical ?? {}, logicalLevelTechnique1: LogicalLevelTechnique.NONE, logicalLevelTechnique2: LogicalLevelTechnique.NONE, physicalLevelTechnique: options.technique ?? PhysicalLevelTechnique.NONE, numValues: count, byteLength: data.length, decompressedCount: count, }, data); } /** * Encodes FSST-compressed strings into a complete stream. * This uses hardcoded test data: ["cat", "dog", "cat"] * @returns Encoded Uint8Array that can be passed to decodeString */ export function encodeFsstStrings() { const symbolTable = new Uint8Array([99, 97, 116, 100, 111, 103]); // "catdog" const symbolLengths = new Uint32Array([3, 3]); const compressedDictionary = new Uint8Array([0, 1]); const dictionaryLengths = new Uint32Array([3, 3]); const offsets = new Uint32Array([0, 1, 0]); // "cat", "dog", "cat" const numValues = 3; return concatenateBuffers(createStream(PhysicalStreamType.PRESENT, encodeBooleanRle(new Array(numValues).fill(true)), { technique: PhysicalLevelTechnique.VARINT, count: numValues, }), createStream(PhysicalStreamType.DATA, symbolTable, { logical: { dictionaryType: DictionaryType.FSST }, }), createStream(PhysicalStreamType.LENGTH, encodeVarintInt32(symbolLengths), { logical: { lengthType: LengthType.SYMBOL }, technique: PhysicalLevelTechnique.VARINT, count: symbolLengths.length, }), createStream(PhysicalStreamType.OFFSET, encodeVarintInt32(offsets), { logical: { offsetType: OffsetType.STRING }, technique: PhysicalLevelTechnique.VARINT, count: offsets.length, }), createStream(PhysicalStreamType.LENGTH, encodeVarintInt32(dictionaryLengths), { logical: { lengthType: LengthType.DICTIONARY }, technique: PhysicalLevelTechnique.VARINT, count: dictionaryLengths.length, }), createStream(PhysicalStreamType.DATA, compressedDictionary, { logical: { dictionaryType: DictionaryType.SINGLE }, })); } /** * Encodes a shared dictionary for struct fields. * @param dictionaryStrings - Array of unique strings in the dictionary * @param options - Encoding options * @returns Object containing length and data streams */ export function encodeSharedDictionary(dictionaryStrings, options = {}) { const { useFsst = false, dictionaryType = DictionaryType.SHARED } = options; const encodedDictionary = encodeStrings(dictionaryStrings); const dictionaryLengths = createStringLengths(dictionaryStrings); const lengthStream = createStream(PhysicalStreamType.LENGTH, encodeVarintInt32(new Uint32Array(dictionaryLengths)), { logical: { lengthType: LengthType.DICTIONARY }, technique: PhysicalLevelTechnique.VARINT, count: dictionaryLengths.length, }); const dataStream = createStream(PhysicalStreamType.DATA, encodedDictionary, { logical: { dictionaryType: dictionaryType }, count: encodedDictionary.length, }); if (useFsst) { const symbolTable = new Uint8Array([99, 97, 116, 100, 111, 103]); // "catdog" const symbolLengths = new Uint32Array([3, 3]); const symbolLengthStream = createStream(PhysicalStreamType.LENGTH, encodeVarintInt32(symbolLengths), { logical: { lengthType: LengthType.SYMBOL }, technique: PhysicalLevelTechnique.VARINT, count: symbolLengths.length, }); const symbolDataStream = createStream(PhysicalStreamType.DATA, symbolTable, { logical: { dictionaryType: DictionaryType.FSST }, count: symbolTable.length, }); return { lengthStream, dataStream, symbolLengthStream, symbolDataStream }; } return { lengthStream, dataStream }; } /** * Encodes streams for a struct field. * @param offsetIndices - Indices into the shared dictionary * @param presentValues - Boolean array indicating which values are present * @param isPresent - Whether the field itself is present * @returns Encoded streams for the field */ export function encodeStructField(offsetIndices, presentValues, isPresent = true) { if (!isPresent) { return encodeNumStreams(0); } const numStreamsEncoded = encodeNumStreams(2); const encodedPresent = createPresentStream(presentValues); const encodedOffsets = createOffsetStream(offsetIndices); return concatenateBuffers(numStreamsEncoded, encodedPresent, encodedOffsets); } function encodeNumStreams(numStreams) { const buffer = new Uint8Array(5); const offset = new IntWrapper(0); encodeVarintInt32Value(numStreams, buffer, offset); return buffer.slice(0, offset.get()); } function createPresentStream(presentValues) { const metadata = { physicalStreamType: PhysicalStreamType.PRESENT, logicalStreamType: { dictionaryType: DictionaryType.NONE }, logicalLevelTechnique1: LogicalLevelTechnique.NONE, logicalLevelTechnique2: LogicalLevelTechnique.NONE, physicalLevelTechnique: PhysicalLevelTechnique.VARINT, numValues: presentValues.length, byteLength: 0, decompressedCount: presentValues.length, }; return buildEncodedStream(metadata, encodeBooleanRle(presentValues)); } function createOffsetStream(offsetIndices) { const metadata = { physicalStreamType: PhysicalStreamType.OFFSET, logicalStreamType: { offsetType: OffsetType.STRING }, logicalLevelTechnique1: LogicalLevelTechnique.NONE, logicalLevelTechnique2: LogicalLevelTechnique.NONE, physicalLevelTechnique: PhysicalLevelTechnique.VARINT, numValues: offsetIndices.length, byteLength: 0, decompressedCount: offsetIndices.length, }; return buildEncodedStream(metadata, encodeVarintInt32(new Uint32Array(offsetIndices))); } /** * Builds a complete encoded stream by combining metadata and data. */ export function buildEncodedStream(streamMetadata, encodedData) { const updatedMetadata = { ...streamMetadata, byteLength: encodedData.length, }; const metadataBuffer = encodeStreamMetadata(updatedMetadata); const result = new Uint8Array(metadataBuffer.length + encodedData.length); result.set(metadataBuffer, 0); result.set(encodedData, metadataBuffer.length); return result; } /** * Encodes stream metadata into binary format. * - Byte 1: Stream type (physical type in upper 4 bits, logical subtype in lower 4 bits) * - Byte 2: Encodings (llt1[5-7], llt2[2-4], plt[0-1]) * - Varints: numValues, byteLength * - If RLE: Varints: runs, numRleValues */ export function encodeStreamMetadata(metadata) { const buffer = new Uint8Array(100); let writeOffset = 0; // Byte 1: Stream type buffer[writeOffset++] = encodeStreamTypeByte(metadata); // Byte 2: Encoding techniques buffer[writeOffset++] = encodeEncodingsByte(metadata); // Variable-length fields const offset = new IntWrapper(writeOffset); encodeVarintInt32Value(metadata.numValues, buffer, offset); encodeVarintInt32Value(metadata.byteLength, buffer, offset); // RLE-specific fields if (isRleMetadata(metadata)) { encodeVarintInt32Value(metadata.runs, buffer, offset); encodeVarintInt32Value(metadata.numRleValues, buffer, offset); } return buffer.slice(0, offset.get()); } function encodeStreamTypeByte(metadata) { const physicalTypeIndex = Object.values(PhysicalStreamType).indexOf(metadata.physicalStreamType); const lowerNibble = getLogicalSubtypeValue(metadata); return (physicalTypeIndex << 4) | lowerNibble; } function getLogicalSubtypeValue(metadata) { const { physicalStreamType, logicalStreamType } = metadata; switch (physicalStreamType) { case PhysicalStreamType.DATA: return logicalStreamType.dictionaryType !== undefined ? Object.values(DictionaryType).indexOf(logicalStreamType.dictionaryType) : 0; case PhysicalStreamType.OFFSET: return logicalStreamType.offsetType !== undefined ? Object.values(OffsetType).indexOf(logicalStreamType.offsetType) : 0; case PhysicalStreamType.LENGTH: return logicalStreamType.lengthType !== undefined ? Object.values(LengthType).indexOf(logicalStreamType.lengthType) : 0; default: return 0; } } function encodeEncodingsByte(metadata) { const llt1Index = Object.values(LogicalLevelTechnique).indexOf(metadata.logicalLevelTechnique1); const llt2Index = Object.values(LogicalLevelTechnique).indexOf(metadata.logicalLevelTechnique2); const pltIndex = Object.values(PhysicalLevelTechnique).indexOf(metadata.physicalLevelTechnique); return (llt1Index << 5) | (llt2Index << 2) | pltIndex; } function isRleMetadata(metadata) { return "runs" in metadata && "numRleValues" in metadata; } /** * Concatenates multiple Uint8Array buffers into a single buffer. */ export function concatenateBuffers(...buffers) { const totalLength = buffers.reduce((sum, buf) => sum + buf.length, 0); const result = new Uint8Array(totalLength); let offset = 0; for (const buffer of buffers) { result.set(buffer, offset); offset += buffer.length; } return result; } //# sourceMappingURL=decodingTestUtils.js.map