284 lines
12 KiB
JavaScript
284 lines
12 KiB
JavaScript
import { PhysicalStreamType } from "../metadata/tile/physicalStreamType";
|
|
import { LogicalLevelTechnique } from "../metadata/tile/logicalLevelTechnique";
|
|
import { PhysicalLevelTechnique } from "../metadata/tile/physicalLevelTechnique";
|
|
import { DictionaryType } from "../metadata/tile/dictionaryType";
|
|
import { LengthType } from "../metadata/tile/lengthType";
|
|
import { OffsetType } from "../metadata/tile/offsetType";
|
|
import IntWrapper from "./intWrapper";
|
|
import { ComplexType, ScalarType } from "../metadata/tileset/tilesetMetadata";
|
|
import { encodeBooleanRle, encodeStrings, createStringLengths } from "../encoding/encodingUtils";
|
|
import { encodeVarintInt32Value, encodeVarintInt32 } from "../encoding/integerEncodingUtils";
|
|
/**
|
|
* Creates basic stream metadata with logical techniques.
|
|
*/
|
|
export function createStreamMetadata(logicalTechnique1, logicalTechnique2 = LogicalLevelTechnique.NONE, numValues = 3) {
|
|
return {
|
|
physicalStreamType: PhysicalStreamType.DATA,
|
|
logicalStreamType: { dictionaryType: DictionaryType.NONE },
|
|
logicalLevelTechnique1: logicalTechnique1,
|
|
logicalLevelTechnique2: logicalTechnique2,
|
|
physicalLevelTechnique: PhysicalLevelTechnique.VARINT,
|
|
numValues,
|
|
byteLength: 10,
|
|
decompressedCount: numValues,
|
|
};
|
|
}
|
|
/**
|
|
* Creates RLE-encoded stream metadata.
|
|
*/
|
|
export function createRleMetadata(logicalTechnique1, logicalTechnique2, runs, numRleValues) {
|
|
return {
|
|
physicalStreamType: PhysicalStreamType.DATA,
|
|
logicalStreamType: { dictionaryType: DictionaryType.NONE },
|
|
logicalLevelTechnique1: logicalTechnique1,
|
|
logicalLevelTechnique2: logicalTechnique2,
|
|
physicalLevelTechnique: PhysicalLevelTechnique.VARINT,
|
|
numValues: runs * 2,
|
|
byteLength: 10,
|
|
decompressedCount: numRleValues,
|
|
runs,
|
|
numRleValues,
|
|
};
|
|
}
|
|
/**
|
|
* Creates column metadata for STRUCT type columns.
|
|
*/
|
|
export function createColumnMetadataForStruct(columnName, childFields) {
|
|
const children = childFields.map((fieldConfig) => ({
|
|
name: fieldConfig.name,
|
|
nullable: true,
|
|
scalarField: {
|
|
physicalType: fieldConfig.type ?? ScalarType.STRING,
|
|
type: "physicalType",
|
|
},
|
|
type: "scalarField",
|
|
}));
|
|
return {
|
|
name: columnName,
|
|
nullable: false,
|
|
complexType: {
|
|
physicalType: ComplexType.STRUCT,
|
|
children,
|
|
type: "physicalType",
|
|
},
|
|
type: "complexType",
|
|
};
|
|
}
|
|
/**
|
|
* Creates a single stream with metadata and data.
|
|
*/
|
|
export function createStream(physicalType, data, options = {}) {
|
|
const count = options.count ?? 0;
|
|
return buildEncodedStream({
|
|
physicalStreamType: physicalType,
|
|
logicalStreamType: options.logical ?? {},
|
|
logicalLevelTechnique1: LogicalLevelTechnique.NONE,
|
|
logicalLevelTechnique2: LogicalLevelTechnique.NONE,
|
|
physicalLevelTechnique: options.technique ?? PhysicalLevelTechnique.NONE,
|
|
numValues: count,
|
|
byteLength: data.length,
|
|
decompressedCount: count,
|
|
}, data);
|
|
}
|
|
/**
|
|
* Encodes FSST-compressed strings into a complete stream.
|
|
* This uses hardcoded test data: ["cat", "dog", "cat"]
|
|
* @returns Encoded Uint8Array that can be passed to decodeString
|
|
*/
|
|
export function encodeFsstStrings() {
|
|
const symbolTable = new Uint8Array([99, 97, 116, 100, 111, 103]); // "catdog"
|
|
const symbolLengths = new Uint32Array([3, 3]);
|
|
const compressedDictionary = new Uint8Array([0, 1]);
|
|
const dictionaryLengths = new Uint32Array([3, 3]);
|
|
const offsets = new Uint32Array([0, 1, 0]); // "cat", "dog", "cat"
|
|
const numValues = 3;
|
|
return concatenateBuffers(createStream(PhysicalStreamType.PRESENT, encodeBooleanRle(new Array(numValues).fill(true)), {
|
|
technique: PhysicalLevelTechnique.VARINT,
|
|
count: numValues,
|
|
}), createStream(PhysicalStreamType.DATA, symbolTable, {
|
|
logical: { dictionaryType: DictionaryType.FSST },
|
|
}), createStream(PhysicalStreamType.LENGTH, encodeVarintInt32(symbolLengths), {
|
|
logical: { lengthType: LengthType.SYMBOL },
|
|
technique: PhysicalLevelTechnique.VARINT,
|
|
count: symbolLengths.length,
|
|
}), createStream(PhysicalStreamType.OFFSET, encodeVarintInt32(offsets), {
|
|
logical: { offsetType: OffsetType.STRING },
|
|
technique: PhysicalLevelTechnique.VARINT,
|
|
count: offsets.length,
|
|
}), createStream(PhysicalStreamType.LENGTH, encodeVarintInt32(dictionaryLengths), {
|
|
logical: { lengthType: LengthType.DICTIONARY },
|
|
technique: PhysicalLevelTechnique.VARINT,
|
|
count: dictionaryLengths.length,
|
|
}), createStream(PhysicalStreamType.DATA, compressedDictionary, {
|
|
logical: { dictionaryType: DictionaryType.SINGLE },
|
|
}));
|
|
}
|
|
/**
|
|
* Encodes a shared dictionary for struct fields.
|
|
* @param dictionaryStrings - Array of unique strings in the dictionary
|
|
* @param options - Encoding options
|
|
* @returns Object containing length and data streams
|
|
*/
|
|
export function encodeSharedDictionary(dictionaryStrings, options = {}) {
|
|
const { useFsst = false, dictionaryType = DictionaryType.SHARED } = options;
|
|
const encodedDictionary = encodeStrings(dictionaryStrings);
|
|
const dictionaryLengths = createStringLengths(dictionaryStrings);
|
|
const lengthStream = createStream(PhysicalStreamType.LENGTH, encodeVarintInt32(new Uint32Array(dictionaryLengths)), {
|
|
logical: { lengthType: LengthType.DICTIONARY },
|
|
technique: PhysicalLevelTechnique.VARINT,
|
|
count: dictionaryLengths.length,
|
|
});
|
|
const dataStream = createStream(PhysicalStreamType.DATA, encodedDictionary, {
|
|
logical: { dictionaryType: dictionaryType },
|
|
count: encodedDictionary.length,
|
|
});
|
|
if (useFsst) {
|
|
const symbolTable = new Uint8Array([99, 97, 116, 100, 111, 103]); // "catdog"
|
|
const symbolLengths = new Uint32Array([3, 3]);
|
|
const symbolLengthStream = createStream(PhysicalStreamType.LENGTH, encodeVarintInt32(symbolLengths), {
|
|
logical: { lengthType: LengthType.SYMBOL },
|
|
technique: PhysicalLevelTechnique.VARINT,
|
|
count: symbolLengths.length,
|
|
});
|
|
const symbolDataStream = createStream(PhysicalStreamType.DATA, symbolTable, {
|
|
logical: { dictionaryType: DictionaryType.FSST },
|
|
count: symbolTable.length,
|
|
});
|
|
return { lengthStream, dataStream, symbolLengthStream, symbolDataStream };
|
|
}
|
|
return { lengthStream, dataStream };
|
|
}
|
|
/**
|
|
* Encodes streams for a struct field.
|
|
* @param offsetIndices - Indices into the shared dictionary
|
|
* @param presentValues - Boolean array indicating which values are present
|
|
* @param isPresent - Whether the field itself is present
|
|
* @returns Encoded streams for the field
|
|
*/
|
|
export function encodeStructField(offsetIndices, presentValues, isPresent = true) {
|
|
if (!isPresent) {
|
|
return encodeNumStreams(0);
|
|
}
|
|
const numStreamsEncoded = encodeNumStreams(2);
|
|
const encodedPresent = createPresentStream(presentValues);
|
|
const encodedOffsets = createOffsetStream(offsetIndices);
|
|
return concatenateBuffers(numStreamsEncoded, encodedPresent, encodedOffsets);
|
|
}
|
|
function encodeNumStreams(numStreams) {
|
|
const buffer = new Uint8Array(5);
|
|
const offset = new IntWrapper(0);
|
|
encodeVarintInt32Value(numStreams, buffer, offset);
|
|
return buffer.slice(0, offset.get());
|
|
}
|
|
function createPresentStream(presentValues) {
|
|
const metadata = {
|
|
physicalStreamType: PhysicalStreamType.PRESENT,
|
|
logicalStreamType: { dictionaryType: DictionaryType.NONE },
|
|
logicalLevelTechnique1: LogicalLevelTechnique.NONE,
|
|
logicalLevelTechnique2: LogicalLevelTechnique.NONE,
|
|
physicalLevelTechnique: PhysicalLevelTechnique.VARINT,
|
|
numValues: presentValues.length,
|
|
byteLength: 0,
|
|
decompressedCount: presentValues.length,
|
|
};
|
|
return buildEncodedStream(metadata, encodeBooleanRle(presentValues));
|
|
}
|
|
function createOffsetStream(offsetIndices) {
|
|
const metadata = {
|
|
physicalStreamType: PhysicalStreamType.OFFSET,
|
|
logicalStreamType: { offsetType: OffsetType.STRING },
|
|
logicalLevelTechnique1: LogicalLevelTechnique.NONE,
|
|
logicalLevelTechnique2: LogicalLevelTechnique.NONE,
|
|
physicalLevelTechnique: PhysicalLevelTechnique.VARINT,
|
|
numValues: offsetIndices.length,
|
|
byteLength: 0,
|
|
decompressedCount: offsetIndices.length,
|
|
};
|
|
return buildEncodedStream(metadata, encodeVarintInt32(new Uint32Array(offsetIndices)));
|
|
}
|
|
/**
|
|
* Builds a complete encoded stream by combining metadata and data.
|
|
*/
|
|
export function buildEncodedStream(streamMetadata, encodedData) {
|
|
const updatedMetadata = {
|
|
...streamMetadata,
|
|
byteLength: encodedData.length,
|
|
};
|
|
const metadataBuffer = encodeStreamMetadata(updatedMetadata);
|
|
const result = new Uint8Array(metadataBuffer.length + encodedData.length);
|
|
result.set(metadataBuffer, 0);
|
|
result.set(encodedData, metadataBuffer.length);
|
|
return result;
|
|
}
|
|
/**
|
|
* Encodes stream metadata into binary format.
|
|
* - Byte 1: Stream type (physical type in upper 4 bits, logical subtype in lower 4 bits)
|
|
* - Byte 2: Encodings (llt1[5-7], llt2[2-4], plt[0-1])
|
|
* - Varints: numValues, byteLength
|
|
* - If RLE: Varints: runs, numRleValues
|
|
*/
|
|
export function encodeStreamMetadata(metadata) {
|
|
const buffer = new Uint8Array(100);
|
|
let writeOffset = 0;
|
|
// Byte 1: Stream type
|
|
buffer[writeOffset++] = encodeStreamTypeByte(metadata);
|
|
// Byte 2: Encoding techniques
|
|
buffer[writeOffset++] = encodeEncodingsByte(metadata);
|
|
// Variable-length fields
|
|
const offset = new IntWrapper(writeOffset);
|
|
encodeVarintInt32Value(metadata.numValues, buffer, offset);
|
|
encodeVarintInt32Value(metadata.byteLength, buffer, offset);
|
|
// RLE-specific fields
|
|
if (isRleMetadata(metadata)) {
|
|
encodeVarintInt32Value(metadata.runs, buffer, offset);
|
|
encodeVarintInt32Value(metadata.numRleValues, buffer, offset);
|
|
}
|
|
return buffer.slice(0, offset.get());
|
|
}
|
|
function encodeStreamTypeByte(metadata) {
|
|
const physicalTypeIndex = Object.values(PhysicalStreamType).indexOf(metadata.physicalStreamType);
|
|
const lowerNibble = getLogicalSubtypeValue(metadata);
|
|
return (physicalTypeIndex << 4) | lowerNibble;
|
|
}
|
|
function getLogicalSubtypeValue(metadata) {
|
|
const { physicalStreamType, logicalStreamType } = metadata;
|
|
switch (physicalStreamType) {
|
|
case PhysicalStreamType.DATA:
|
|
return logicalStreamType.dictionaryType !== undefined
|
|
? Object.values(DictionaryType).indexOf(logicalStreamType.dictionaryType)
|
|
: 0;
|
|
case PhysicalStreamType.OFFSET:
|
|
return logicalStreamType.offsetType !== undefined
|
|
? Object.values(OffsetType).indexOf(logicalStreamType.offsetType)
|
|
: 0;
|
|
case PhysicalStreamType.LENGTH:
|
|
return logicalStreamType.lengthType !== undefined
|
|
? Object.values(LengthType).indexOf(logicalStreamType.lengthType)
|
|
: 0;
|
|
default:
|
|
return 0;
|
|
}
|
|
}
|
|
function encodeEncodingsByte(metadata) {
|
|
const llt1Index = Object.values(LogicalLevelTechnique).indexOf(metadata.logicalLevelTechnique1);
|
|
const llt2Index = Object.values(LogicalLevelTechnique).indexOf(metadata.logicalLevelTechnique2);
|
|
const pltIndex = Object.values(PhysicalLevelTechnique).indexOf(metadata.physicalLevelTechnique);
|
|
return (llt1Index << 5) | (llt2Index << 2) | pltIndex;
|
|
}
|
|
function isRleMetadata(metadata) {
|
|
return "runs" in metadata && "numRleValues" in metadata;
|
|
}
|
|
/**
|
|
* Concatenates multiple Uint8Array buffers into a single buffer.
|
|
*/
|
|
export function concatenateBuffers(...buffers) {
|
|
const totalLength = buffers.reduce((sum, buf) => sum + buf.length, 0);
|
|
const result = new Uint8Array(totalLength);
|
|
let offset = 0;
|
|
for (const buffer of buffers) {
|
|
result.set(buffer, offset);
|
|
offset += buffer.length;
|
|
}
|
|
return result;
|
|
}
|
|
//# sourceMappingURL=decodingTestUtils.js.map
|