import { decodeStreamMetadata } from "../metadata/tile/streamMetadataDecoder"; import { StringFlatVector } from "../vector/flat/stringFlatVector"; import { StringDictionaryVector } from "../vector/dictionary/stringDictionaryVector"; import BitVector from "../vector/flat/bitVector"; import { PhysicalStreamType } from "../metadata/tile/physicalStreamType"; import { DictionaryType } from "../metadata/tile/dictionaryType"; import { LengthType } from "../metadata/tile/lengthType"; import { decodeUnsignedInt32Stream, decodeLengthStreamToOffsetBuffer } from "./integerStreamDecoder"; import { ScalarType } from "../metadata/tileset/tilesetMetadata"; import { decodeVarintInt32 } from "./integerDecodingUtils"; import { decodeBooleanRle, skipColumn } from "./decodingUtils"; import { StringFsstDictionaryVector } from "../vector/fsst-dictionary/stringFsstDictionaryVector"; export function decodeString(name, data, offset, numStreams, bitVector) { let dictionaryLengthStream = null; let offsetStream = null; let dictionaryStream = null; let symbolLengthStream = null; let symbolTableStream = null; let nullabilityBuffer = bitVector ?? null; let plainLengthStream = null; let plainDataStream = null; for (let i = 0; i < numStreams; i++) { const streamMetadata = decodeStreamMetadata(data, offset); switch (streamMetadata.physicalStreamType) { case PhysicalStreamType.PRESENT: { const presentData = decodeBooleanRle(data, streamMetadata.numValues, streamMetadata.byteLength, offset); const presentStream = new BitVector(presentData, streamMetadata.numValues); nullabilityBuffer = bitVector ?? presentStream; break; } case PhysicalStreamType.OFFSET: { offsetStream = decodeUnsignedInt32Stream(data, offset, streamMetadata, undefined, nullabilityBuffer); break; } case PhysicalStreamType.LENGTH: { const lengthStream = decodeLengthStreamToOffsetBuffer(data, offset, streamMetadata); if (LengthType.DICTIONARY === streamMetadata.logicalStreamType.lengthType) { dictionaryLengthStream = lengthStream; } else if (LengthType.SYMBOL === streamMetadata.logicalStreamType.lengthType) { symbolLengthStream = lengthStream; } else { // Plain string encoding uses VAR_BINARY length type plainLengthStream = lengthStream; } break; } case PhysicalStreamType.DATA: { const dataStream = data.subarray(offset.get(), offset.get() + streamMetadata.byteLength); offset.add(streamMetadata.byteLength); const dictType = streamMetadata.logicalStreamType.dictionaryType; if (DictionaryType.FSST === dictType) { symbolTableStream = dataStream; } else if (DictionaryType.SINGLE === dictType || DictionaryType.SHARED === dictType) { dictionaryStream = dataStream; } else if (DictionaryType.NONE === dictType) { plainDataStream = dataStream; } break; } } } return (decodeFsstDictionaryVector(name, symbolTableStream, offsetStream, dictionaryLengthStream, dictionaryStream, symbolLengthStream, nullabilityBuffer) ?? decodeDictionaryVector(name, dictionaryStream, offsetStream, dictionaryLengthStream, nullabilityBuffer) ?? decodePlainStringVector(name, plainLengthStream, plainDataStream, offsetStream, nullabilityBuffer)); } function decodeFsstDictionaryVector(name, symbolTableStream, offsetStream, dictionaryLengthStream, dictionaryStream, symbolLengthStream, nullabilityBuffer) { if (!symbolTableStream) { return null; } return new StringFsstDictionaryVector(name, offsetStream, dictionaryLengthStream, dictionaryStream, symbolLengthStream, symbolTableStream, nullabilityBuffer); } function decodeDictionaryVector(name, dictionaryStream, offsetStream, dictionaryLengthStream, nullabilityBuffer) { if (!dictionaryStream) { return null; } return nullabilityBuffer ? new StringDictionaryVector(name, offsetStream, dictionaryLengthStream, dictionaryStream, nullabilityBuffer) : new StringDictionaryVector(name, offsetStream, dictionaryLengthStream, dictionaryStream); } function decodePlainStringVector(name, plainLengthStream, plainDataStream, offsetStream, nullabilityBuffer) { if (!plainLengthStream || !plainDataStream) { return null; } if (offsetStream) { return nullabilityBuffer ? new StringDictionaryVector(name, offsetStream, plainLengthStream, plainDataStream, nullabilityBuffer) : new StringDictionaryVector(name, offsetStream, plainLengthStream, plainDataStream); } if (nullabilityBuffer && nullabilityBuffer.size() !== plainLengthStream.length - 1) { const sparseOffsetStream = new Uint32Array(nullabilityBuffer.size()); let valueIndex = 0; for (let i = 0; i < nullabilityBuffer.size(); i++) { if (nullabilityBuffer.get(i)) { sparseOffsetStream[i] = valueIndex++; } else { sparseOffsetStream[i] = 0; } } return new StringDictionaryVector(name, sparseOffsetStream, plainLengthStream, plainDataStream, nullabilityBuffer); } return nullabilityBuffer ? new StringFlatVector(name, plainLengthStream, plainDataStream, nullabilityBuffer) : new StringFlatVector(name, plainLengthStream, plainDataStream); } export function decodeSharedDictionary(data, offset, column, numFeatures, propertyColumnNames) { let dictionaryOffsetBuffer = null; let dictionaryBuffer = null; let symbolOffsetBuffer = null; let symbolTableBuffer = null; let dictionaryStreamDecoded = false; while (!dictionaryStreamDecoded) { const streamMetadata = decodeStreamMetadata(data, offset); switch (streamMetadata.physicalStreamType) { case PhysicalStreamType.LENGTH: if (LengthType.DICTIONARY === streamMetadata.logicalStreamType.lengthType) { dictionaryOffsetBuffer = decodeLengthStreamToOffsetBuffer(data, offset, streamMetadata); } else { symbolOffsetBuffer = decodeLengthStreamToOffsetBuffer(data, offset, streamMetadata); } break; case PhysicalStreamType.DATA: if (DictionaryType.SINGLE === streamMetadata.logicalStreamType.dictionaryType || DictionaryType.SHARED === streamMetadata.logicalStreamType.dictionaryType) { dictionaryBuffer = data.subarray(offset.get(), offset.get() + streamMetadata.byteLength); dictionaryStreamDecoded = true; } else { symbolTableBuffer = data.subarray(offset.get(), offset.get() + streamMetadata.byteLength); } offset.add(streamMetadata.byteLength); break; } } const childFields = column.complexType.children; const stringDictionaryVectors = []; let i = 0; for (const childField of childFields) { const numStreams = decodeVarintInt32(data, offset, 1)[0]; if (numStreams === 0) { /* Column is not present in the tile */ continue; } const columnName = childField.name ? `${column.name}${childField.name}` : column.name; if (propertyColumnNames) { if (!propertyColumnNames.has(columnName)) { //TODO: add size of sub column to Mlt for faster skipping skipColumn(numStreams, data, offset); continue; } } if (numStreams !== 2 || childField.type !== "scalarField" || childField.scalarField.physicalType !== ScalarType.STRING) { throw new Error("Currently only optional string fields are implemented for a struct."); } const presentStreamMetadata = decodeStreamMetadata(data, offset); const presentStream = decodeBooleanRle(data, presentStreamMetadata.numValues, presentStreamMetadata.byteLength, offset); const offsetStreamMetadata = decodeStreamMetadata(data, offset); const offsetCount = offsetStreamMetadata.decompressedCount; const isNullable = offsetCount !== numFeatures; const offsetStream = decodeUnsignedInt32Stream(data, offset, offsetStreamMetadata, undefined, isNullable ? new BitVector(presentStream, presentStreamMetadata.numValues) : undefined); stringDictionaryVectors[i++] = symbolTableBuffer ? new StringFsstDictionaryVector(columnName, offsetStream, dictionaryOffsetBuffer, dictionaryBuffer, symbolOffsetBuffer, symbolTableBuffer, new BitVector(presentStream, presentStreamMetadata.numValues)) : new StringDictionaryVector(columnName, offsetStream, dictionaryOffsetBuffer, dictionaryBuffer, new BitVector(presentStream, presentStreamMetadata.numValues)); } return stringDictionaryVectors; } //# sourceMappingURL=stringDecoder.js.map