Skip to content

Commit

Permalink
Merge pull request #2 from ccri/fixed-size-list
Browse files Browse the repository at this point in the history
Don't read OFFSET vector for FixedSizeList
  • Loading branch information
trxcllnt authored Jan 23, 2018
2 parents 614b688 + 5bb63af commit e33c068
Show file tree
Hide file tree
Showing 5 changed files with 33 additions and 16 deletions.
20 changes: 13 additions & 7 deletions js/src/data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { popcnt_bit_range } from './util/bit';
import { VectorLike, Vector } from './vector';
import { VectorType, TypedArray, TypedArrayConstructor, Dictionary } from './type';
import { Int, Bool, FlatListType, List, FixedSizeList, Struct, Map_ } from './type';
import { DataType, FlatType, ListType, NestedType, DenseUnion, SparseUnion } from './type';
import { DataType, FlatType, ListType, NestedType, SingleNestedType, DenseUnion, SparseUnion } from './type';

export function toTypedArray<T extends TypedArray>(ArrayType: TypedArrayConstructor<T>, values?: T | ArrayLike<number> | Iterable<number> | null): T {
if (!ArrayType && ArrayBuffer.isView(values)) { return values; }
Expand All @@ -46,7 +46,7 @@ export interface DataTypes<T extends DataType> {
/* [Type.Struct]*/ 13: NestedData<Struct>;
/* [Type.Union]*/ 14: UnionData;
/* [Type.FixedSizeBinary]*/ 15: FlatData<T>;
/* [Type.FixedSizeList]*/ 16: ListData<FixedSizeList<T>>;
/* [Type.FixedSizeList]*/ 16: SingleNestedData<FixedSizeList<T>>;
/* [Type.Map]*/ 17: NestedData<Map_>;
/* [Type.DenseUnion]*/ DenseUnion: DenseUnionData;
/*[Type.SparseUnion]*/ SparseUnion: SparseUnionData;
Expand Down Expand Up @@ -195,15 +195,21 @@ export class NestedData<T extends NestedType = NestedType> extends BaseData<T> {
}
}

export class ListData<T extends ListType> extends NestedData<T> {
public /* [VectorType.OFFSET]:*/ 0: Int32Array;
public /*[VectorType.VALIDITY]:*/ 2: Uint8Array;
export class SingleNestedData<T extends SingleNestedType> extends NestedData<T> {
protected _valuesData: Data<T>;
public get values() { return this._valuesData; }
public get valueOffsets() { return this[VectorType.OFFSET]; }
constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, valueOffsets: Iterable<number>, valueChildData: Data<T>, offset?: number, nullCount?: number) {
constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, valueChildData: Data<T>, offset?: number, nullCount?: number) {
super(type, length, nullBitmap, [valueChildData], offset, nullCount);
this._valuesData = valueChildData;
}
}

export class ListData<T extends ListType> extends SingleNestedData<T> {
public /* [VectorType.OFFSET]:*/ 0: Int32Array;
public /*[VectorType.VALIDITY]:*/ 2: Uint8Array;
public get valueOffsets() { return this[VectorType.OFFSET]; }
constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, valueOffsets: Iterable<number>, valueChildData: Data<T>, offset?: number, nullCount?: number) {
super(type, length, nullBitmap, valueChildData, offset, nullCount);
this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets);
}
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
Expand Down
7 changes: 5 additions & 2 deletions js/src/ipc/reader/vector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import { RecordBatch } from '../../recordbatch';
import { TypeVisitor } from '../../visitor';
import { FlatType, NestedType, ListType } from '../../type';
import { Message, FieldMetadata, BufferMetadata } from '../metadata';
import { FlatData, ListData, NestedData, DenseUnionData, SparseUnionData, BoolData, FlatListData, DictionaryData } from '../../data';
import { FlatData, ListData, NestedData, SingleNestedData, DenseUnionData, SparseUnionData, BoolData, FlatListData, DictionaryData } from '../../data';
import {
Schema, Field,
Dictionary,
Expand Down Expand Up @@ -89,7 +89,7 @@ export abstract class TypeDataLoader extends TypeVisitor {
public visitStruct (type: Struct) { return this.visitNestedType(type); }
public visitUnion (type: Union) { return this.visitUnionType(type); }
public visitFixedSizeBinary(type: FixedSizeBinary) { return this.visitFlatType(type); }
public visitFixedSizeList (type: FixedSizeList) { return this.visitListType(type); }
public visitFixedSizeList (type: FixedSizeList) { return this.visitFixedSizeListType(type); }
public visitMap (type: Map_) { return this.visitNestedType(type); }
public visitDictionary (type: Dictionary) {
return new DictionaryData(type, this.dictionaries.get(type.id)!, this.visit(type.indicies));
Expand Down Expand Up @@ -117,6 +117,9 @@ export abstract class TypeDataLoader extends TypeVisitor {
protected visitListType<T extends ListType>(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
return new ListData<T>(type, length, this.readNullBitmap(type, nullCount), this.readOffsets(type), this.visit(type.children![0].type), 0, nullCount);
}
protected visitFixedSizeListType<T extends FixedSizeList>(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
return new SingleNestedData<T>(type, length, this.readNullBitmap(type, nullCount), this.visit(type.children![0].type), 0, nullCount);
}
protected visitNestedType<T extends NestedType>(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
return new NestedData<T>(type, length, this.readNullBitmap(type, nullCount), this.visitFields(type.children), 0, nullCount);
}
Expand Down
3 changes: 2 additions & 1 deletion js/src/type.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,9 @@ export type PrimitiveType = NumericType | FixedSizeType;

export type FlatListType = Utf8 | Binary; // <-- these types have `offset`, `data`, and `validity` buffers
export type FlatType = Bool | PrimitiveType | FlatListType; // <-- these types have `data` and `validity` buffers
export type ListType = List<any> | FixedSizeList<any>; // <-- these types have `offset` and `validity` buffers
export type ListType = List<any>; // <-- these types have `offset` and `validity` buffers
export type NestedType = Map_ | Struct | List<any> | FixedSizeList<any> | Union<any>; // <-- these types have `validity` buffer and nested childData
export type SingleNestedType = List<any> | FixedSizeList<any>; // <-- these are nested types that can only have a single child

/**
* *
Expand Down
2 changes: 1 addition & 1 deletion js/src/vector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ export class ListVector<T extends DataType = DataType> extends ListVectorBase<Li
}
}

export class FixedSizeListVector extends ListVectorBase<FixedSizeList> {
export class FixedSizeListVector extends Vector<FixedSizeList> {
constructor(data: Data<FixedSizeList>, view: View<FixedSizeList> = new FixedSizeListView(data)) {
super(data, view);
}
Expand Down
17 changes: 12 additions & 5 deletions js/src/vector/list.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,13 @@ export const decodeUtf8 = ((decoder) =>
decoder.decode.bind(decoder) as (input?: ArrayBufferLike | ArrayBufferView) => string
)(new TextDecoder('utf-8'));

export abstract class ListViewBase<T extends (ListType | FlatListType)> implements View<T> {
export abstract class ListViewBase<T extends (ListType | FlatListType | FixedSizeList)> implements View<T> {
public length: number;
public values: T['TArray'];
public valueOffsets?: Int32Array;
constructor(data: Data<T>) {
this.length = data.length;
this.values = data.values;
this.valueOffsets = data.valueOffsets;
}
public clone(data: Data<T>): this {
return new (<any> this.constructor)(data) as this;
Expand Down Expand Up @@ -64,7 +63,15 @@ export abstract class ListViewBase<T extends (ListType | FlatListType)> implemen
protected abstract setList(values: T['TArray'], index: number, value: T['TValue'], valueOffsets?: Int32Array): void;
}

export class ListView<T extends DataType> extends ListViewBase<List<T>> {
export abstract class VariableListViewBase<T extends (ListType | FlatListType)> extends ListViewBase<T> {
constructor(data: Data<T>) {
super(data)
this.length = data.length;
this.valueOffsets = data.valueOffsets;
}
}

export class ListView<T extends DataType> extends VariableListViewBase<List<T>> {
constructor(data: Data<List<T>>) {
super(data);
this.values = createVector(data.values);
Expand Down Expand Up @@ -101,7 +108,7 @@ export class FixedSizeListView<T extends DataType> extends ListViewBase<FixedSiz
}
}

export class BinaryView extends ListViewBase<Binary> {
export class BinaryView extends VariableListViewBase<Binary> {
protected getList(values: Uint8Array, index: number, valueOffsets: Int32Array) {
return values.subarray(valueOffsets[index], valueOffsets[index + 1]);
}
Expand All @@ -111,7 +118,7 @@ export class BinaryView extends ListViewBase<Binary> {
}
}

export class Utf8View extends ListViewBase<Utf8> {
export class Utf8View extends VariableListViewBase<Utf8> {
protected getList(values: Uint8Array, index: number, valueOffsets: Int32Array) {
return decodeUtf8(values.subarray(valueOffsets[index], valueOffsets[index + 1]));
}
Expand Down

0 comments on commit e33c068

Please sign in to comment.