// Code generated by array/numericbuilder.gen.go.tmpl. DO NOT EDIT. // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package array import ( "bytes" "fmt" "reflect" "strconv" "strings" "sync/atomic" "time" "github.com/apache/arrow/go/v10/arrow" "github.com/apache/arrow/go/v10/arrow/bitutil" "github.com/apache/arrow/go/v10/arrow/internal/debug" "github.com/apache/arrow/go/v10/arrow/memory" "github.com/goccy/go-json" ) type Int64Builder struct { builder data *memory.Buffer rawData []int64 } func NewInt64Builder(mem memory.Allocator) *Int64Builder { return &Int64Builder{builder: builder{refCount: 1, mem: mem}} } func (b *Int64Builder) Type() arrow.DataType { return arrow.PrimitiveTypes.Int64 } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *Int64Builder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *Int64Builder) Append(v int64) { b.Reserve(1) b.UnsafeAppend(v) } func (b *Int64Builder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *Int64Builder) AppendEmptyValue() { b.Append(0) } func (b *Int64Builder) UnsafeAppend(v int64) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *Int64Builder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *Int64Builder) AppendValues(v []int64, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.Int64Traits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *Int64Builder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.Int64Traits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.Int64Traits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *Int64Builder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *Int64Builder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.Int64Traits.BytesRequired(n)) b.rawData = arrow.Int64Traits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Int64 array from the memory buffers used by the builder and resets the Int64Builder // so it can be used to build a new array. func (b *Int64Builder) NewArray() arrow.Array { return b.NewInt64Array() } // NewInt64Array creates a Int64 array from the memory buffers used by the builder and resets the Int64Builder // so it can be used to build a new array. func (b *Int64Builder) NewInt64Array() (a *Int64) { data := b.newData() a = NewInt64Data(data) data.Release() return } func (b *Int64Builder) newData() (data *Data) { bytesRequired := arrow.Int64Traits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(arrow.PrimitiveTypes.Int64, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *Int64Builder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case string: f, err := strconv.ParseInt(v, 10, 8*8) if err != nil { return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(int64(0)), Offset: dec.InputOffset(), } } b.Append(int64(f)) case float64: b.Append(int64(v)) case json.Number: f, err := strconv.ParseInt(v.String(), 10, 8*8) if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(int64(0)), Offset: dec.InputOffset(), } } b.Append(int64(f)) default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(int64(0)), Offset: dec.InputOffset(), } } return nil } func (b *Int64Builder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *Int64Builder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } type Uint64Builder struct { builder data *memory.Buffer rawData []uint64 } func NewUint64Builder(mem memory.Allocator) *Uint64Builder { return &Uint64Builder{builder: builder{refCount: 1, mem: mem}} } func (b *Uint64Builder) Type() arrow.DataType { return arrow.PrimitiveTypes.Uint64 } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *Uint64Builder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *Uint64Builder) Append(v uint64) { b.Reserve(1) b.UnsafeAppend(v) } func (b *Uint64Builder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *Uint64Builder) AppendEmptyValue() { b.Append(0) } func (b *Uint64Builder) UnsafeAppend(v uint64) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *Uint64Builder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *Uint64Builder) AppendValues(v []uint64, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.Uint64Traits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *Uint64Builder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.Uint64Traits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.Uint64Traits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *Uint64Builder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *Uint64Builder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.Uint64Traits.BytesRequired(n)) b.rawData = arrow.Uint64Traits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Uint64 array from the memory buffers used by the builder and resets the Uint64Builder // so it can be used to build a new array. func (b *Uint64Builder) NewArray() arrow.Array { return b.NewUint64Array() } // NewUint64Array creates a Uint64 array from the memory buffers used by the builder and resets the Uint64Builder // so it can be used to build a new array. func (b *Uint64Builder) NewUint64Array() (a *Uint64) { data := b.newData() a = NewUint64Data(data) data.Release() return } func (b *Uint64Builder) newData() (data *Data) { bytesRequired := arrow.Uint64Traits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(arrow.PrimitiveTypes.Uint64, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *Uint64Builder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case string: f, err := strconv.ParseUint(v, 10, 8*8) if err != nil { return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(uint64(0)), Offset: dec.InputOffset(), } } b.Append(uint64(f)) case float64: b.Append(uint64(v)) case json.Number: f, err := strconv.ParseUint(v.String(), 10, 8*8) if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(uint64(0)), Offset: dec.InputOffset(), } } b.Append(uint64(f)) default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(uint64(0)), Offset: dec.InputOffset(), } } return nil } func (b *Uint64Builder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *Uint64Builder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } type Float64Builder struct { builder data *memory.Buffer rawData []float64 } func NewFloat64Builder(mem memory.Allocator) *Float64Builder { return &Float64Builder{builder: builder{refCount: 1, mem: mem}} } func (b *Float64Builder) Type() arrow.DataType { return arrow.PrimitiveTypes.Float64 } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *Float64Builder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *Float64Builder) Append(v float64) { b.Reserve(1) b.UnsafeAppend(v) } func (b *Float64Builder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *Float64Builder) AppendEmptyValue() { b.Append(0) } func (b *Float64Builder) UnsafeAppend(v float64) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *Float64Builder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *Float64Builder) AppendValues(v []float64, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.Float64Traits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *Float64Builder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.Float64Traits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.Float64Traits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *Float64Builder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *Float64Builder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.Float64Traits.BytesRequired(n)) b.rawData = arrow.Float64Traits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Float64 array from the memory buffers used by the builder and resets the Float64Builder // so it can be used to build a new array. func (b *Float64Builder) NewArray() arrow.Array { return b.NewFloat64Array() } // NewFloat64Array creates a Float64 array from the memory buffers used by the builder and resets the Float64Builder // so it can be used to build a new array. func (b *Float64Builder) NewFloat64Array() (a *Float64) { data := b.newData() a = NewFloat64Data(data) data.Release() return } func (b *Float64Builder) newData() (data *Data) { bytesRequired := arrow.Float64Traits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(arrow.PrimitiveTypes.Float64, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *Float64Builder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case string: f, err := strconv.ParseFloat(v, 8*8) if err != nil { return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(float64(0)), Offset: dec.InputOffset(), } } b.Append(float64(f)) case float64: b.Append(float64(v)) case json.Number: f, err := strconv.ParseFloat(v.String(), 8*8) if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(float64(0)), Offset: dec.InputOffset(), } } b.Append(float64(f)) default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(float64(0)), Offset: dec.InputOffset(), } } return nil } func (b *Float64Builder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *Float64Builder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } type Int32Builder struct { builder data *memory.Buffer rawData []int32 } func NewInt32Builder(mem memory.Allocator) *Int32Builder { return &Int32Builder{builder: builder{refCount: 1, mem: mem}} } func (b *Int32Builder) Type() arrow.DataType { return arrow.PrimitiveTypes.Int32 } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *Int32Builder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *Int32Builder) Append(v int32) { b.Reserve(1) b.UnsafeAppend(v) } func (b *Int32Builder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *Int32Builder) AppendEmptyValue() { b.Append(0) } func (b *Int32Builder) UnsafeAppend(v int32) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *Int32Builder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *Int32Builder) AppendValues(v []int32, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.Int32Traits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *Int32Builder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.Int32Traits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.Int32Traits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *Int32Builder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *Int32Builder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.Int32Traits.BytesRequired(n)) b.rawData = arrow.Int32Traits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Int32 array from the memory buffers used by the builder and resets the Int32Builder // so it can be used to build a new array. func (b *Int32Builder) NewArray() arrow.Array { return b.NewInt32Array() } // NewInt32Array creates a Int32 array from the memory buffers used by the builder and resets the Int32Builder // so it can be used to build a new array. func (b *Int32Builder) NewInt32Array() (a *Int32) { data := b.newData() a = NewInt32Data(data) data.Release() return } func (b *Int32Builder) newData() (data *Data) { bytesRequired := arrow.Int32Traits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(arrow.PrimitiveTypes.Int32, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *Int32Builder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case string: f, err := strconv.ParseInt(v, 10, 4*8) if err != nil { return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(int32(0)), Offset: dec.InputOffset(), } } b.Append(int32(f)) case float64: b.Append(int32(v)) case json.Number: f, err := strconv.ParseInt(v.String(), 10, 4*8) if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(int32(0)), Offset: dec.InputOffset(), } } b.Append(int32(f)) default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(int32(0)), Offset: dec.InputOffset(), } } return nil } func (b *Int32Builder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *Int32Builder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } type Uint32Builder struct { builder data *memory.Buffer rawData []uint32 } func NewUint32Builder(mem memory.Allocator) *Uint32Builder { return &Uint32Builder{builder: builder{refCount: 1, mem: mem}} } func (b *Uint32Builder) Type() arrow.DataType { return arrow.PrimitiveTypes.Uint32 } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *Uint32Builder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *Uint32Builder) Append(v uint32) { b.Reserve(1) b.UnsafeAppend(v) } func (b *Uint32Builder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *Uint32Builder) AppendEmptyValue() { b.Append(0) } func (b *Uint32Builder) UnsafeAppend(v uint32) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *Uint32Builder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *Uint32Builder) AppendValues(v []uint32, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.Uint32Traits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *Uint32Builder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.Uint32Traits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.Uint32Traits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *Uint32Builder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *Uint32Builder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.Uint32Traits.BytesRequired(n)) b.rawData = arrow.Uint32Traits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Uint32 array from the memory buffers used by the builder and resets the Uint32Builder // so it can be used to build a new array. func (b *Uint32Builder) NewArray() arrow.Array { return b.NewUint32Array() } // NewUint32Array creates a Uint32 array from the memory buffers used by the builder and resets the Uint32Builder // so it can be used to build a new array. func (b *Uint32Builder) NewUint32Array() (a *Uint32) { data := b.newData() a = NewUint32Data(data) data.Release() return } func (b *Uint32Builder) newData() (data *Data) { bytesRequired := arrow.Uint32Traits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(arrow.PrimitiveTypes.Uint32, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *Uint32Builder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case string: f, err := strconv.ParseUint(v, 10, 4*8) if err != nil { return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(uint32(0)), Offset: dec.InputOffset(), } } b.Append(uint32(f)) case float64: b.Append(uint32(v)) case json.Number: f, err := strconv.ParseUint(v.String(), 10, 4*8) if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(uint32(0)), Offset: dec.InputOffset(), } } b.Append(uint32(f)) default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(uint32(0)), Offset: dec.InputOffset(), } } return nil } func (b *Uint32Builder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *Uint32Builder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } type Float32Builder struct { builder data *memory.Buffer rawData []float32 } func NewFloat32Builder(mem memory.Allocator) *Float32Builder { return &Float32Builder{builder: builder{refCount: 1, mem: mem}} } func (b *Float32Builder) Type() arrow.DataType { return arrow.PrimitiveTypes.Float32 } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *Float32Builder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *Float32Builder) Append(v float32) { b.Reserve(1) b.UnsafeAppend(v) } func (b *Float32Builder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *Float32Builder) AppendEmptyValue() { b.Append(0) } func (b *Float32Builder) UnsafeAppend(v float32) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *Float32Builder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *Float32Builder) AppendValues(v []float32, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.Float32Traits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *Float32Builder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.Float32Traits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.Float32Traits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *Float32Builder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *Float32Builder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.Float32Traits.BytesRequired(n)) b.rawData = arrow.Float32Traits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Float32 array from the memory buffers used by the builder and resets the Float32Builder // so it can be used to build a new array. func (b *Float32Builder) NewArray() arrow.Array { return b.NewFloat32Array() } // NewFloat32Array creates a Float32 array from the memory buffers used by the builder and resets the Float32Builder // so it can be used to build a new array. func (b *Float32Builder) NewFloat32Array() (a *Float32) { data := b.newData() a = NewFloat32Data(data) data.Release() return } func (b *Float32Builder) newData() (data *Data) { bytesRequired := arrow.Float32Traits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(arrow.PrimitiveTypes.Float32, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *Float32Builder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case string: f, err := strconv.ParseFloat(v, 4*8) if err != nil { return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(float32(0)), Offset: dec.InputOffset(), } } b.Append(float32(f)) case float64: b.Append(float32(v)) case json.Number: f, err := strconv.ParseFloat(v.String(), 4*8) if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(float32(0)), Offset: dec.InputOffset(), } } b.Append(float32(f)) default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(float32(0)), Offset: dec.InputOffset(), } } return nil } func (b *Float32Builder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *Float32Builder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } type Int16Builder struct { builder data *memory.Buffer rawData []int16 } func NewInt16Builder(mem memory.Allocator) *Int16Builder { return &Int16Builder{builder: builder{refCount: 1, mem: mem}} } func (b *Int16Builder) Type() arrow.DataType { return arrow.PrimitiveTypes.Int16 } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *Int16Builder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *Int16Builder) Append(v int16) { b.Reserve(1) b.UnsafeAppend(v) } func (b *Int16Builder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *Int16Builder) AppendEmptyValue() { b.Append(0) } func (b *Int16Builder) UnsafeAppend(v int16) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *Int16Builder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *Int16Builder) AppendValues(v []int16, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.Int16Traits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *Int16Builder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.Int16Traits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.Int16Traits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *Int16Builder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *Int16Builder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.Int16Traits.BytesRequired(n)) b.rawData = arrow.Int16Traits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Int16 array from the memory buffers used by the builder and resets the Int16Builder // so it can be used to build a new array. func (b *Int16Builder) NewArray() arrow.Array { return b.NewInt16Array() } // NewInt16Array creates a Int16 array from the memory buffers used by the builder and resets the Int16Builder // so it can be used to build a new array. func (b *Int16Builder) NewInt16Array() (a *Int16) { data := b.newData() a = NewInt16Data(data) data.Release() return } func (b *Int16Builder) newData() (data *Data) { bytesRequired := arrow.Int16Traits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(arrow.PrimitiveTypes.Int16, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *Int16Builder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case string: f, err := strconv.ParseInt(v, 10, 2*8) if err != nil { return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(int16(0)), Offset: dec.InputOffset(), } } b.Append(int16(f)) case float64: b.Append(int16(v)) case json.Number: f, err := strconv.ParseInt(v.String(), 10, 2*8) if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(int16(0)), Offset: dec.InputOffset(), } } b.Append(int16(f)) default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(int16(0)), Offset: dec.InputOffset(), } } return nil } func (b *Int16Builder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *Int16Builder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } type Uint16Builder struct { builder data *memory.Buffer rawData []uint16 } func NewUint16Builder(mem memory.Allocator) *Uint16Builder { return &Uint16Builder{builder: builder{refCount: 1, mem: mem}} } func (b *Uint16Builder) Type() arrow.DataType { return arrow.PrimitiveTypes.Uint16 } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *Uint16Builder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *Uint16Builder) Append(v uint16) { b.Reserve(1) b.UnsafeAppend(v) } func (b *Uint16Builder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *Uint16Builder) AppendEmptyValue() { b.Append(0) } func (b *Uint16Builder) UnsafeAppend(v uint16) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *Uint16Builder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *Uint16Builder) AppendValues(v []uint16, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.Uint16Traits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *Uint16Builder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.Uint16Traits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.Uint16Traits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *Uint16Builder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *Uint16Builder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.Uint16Traits.BytesRequired(n)) b.rawData = arrow.Uint16Traits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Uint16 array from the memory buffers used by the builder and resets the Uint16Builder // so it can be used to build a new array. func (b *Uint16Builder) NewArray() arrow.Array { return b.NewUint16Array() } // NewUint16Array creates a Uint16 array from the memory buffers used by the builder and resets the Uint16Builder // so it can be used to build a new array. func (b *Uint16Builder) NewUint16Array() (a *Uint16) { data := b.newData() a = NewUint16Data(data) data.Release() return } func (b *Uint16Builder) newData() (data *Data) { bytesRequired := arrow.Uint16Traits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(arrow.PrimitiveTypes.Uint16, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *Uint16Builder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case string: f, err := strconv.ParseUint(v, 10, 2*8) if err != nil { return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(uint16(0)), Offset: dec.InputOffset(), } } b.Append(uint16(f)) case float64: b.Append(uint16(v)) case json.Number: f, err := strconv.ParseUint(v.String(), 10, 2*8) if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(uint16(0)), Offset: dec.InputOffset(), } } b.Append(uint16(f)) default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(uint16(0)), Offset: dec.InputOffset(), } } return nil } func (b *Uint16Builder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *Uint16Builder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } type Int8Builder struct { builder data *memory.Buffer rawData []int8 } func NewInt8Builder(mem memory.Allocator) *Int8Builder { return &Int8Builder{builder: builder{refCount: 1, mem: mem}} } func (b *Int8Builder) Type() arrow.DataType { return arrow.PrimitiveTypes.Int8 } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *Int8Builder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *Int8Builder) Append(v int8) { b.Reserve(1) b.UnsafeAppend(v) } func (b *Int8Builder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *Int8Builder) AppendEmptyValue() { b.Append(0) } func (b *Int8Builder) UnsafeAppend(v int8) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *Int8Builder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *Int8Builder) AppendValues(v []int8, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.Int8Traits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *Int8Builder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.Int8Traits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.Int8Traits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *Int8Builder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *Int8Builder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.Int8Traits.BytesRequired(n)) b.rawData = arrow.Int8Traits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Int8 array from the memory buffers used by the builder and resets the Int8Builder // so it can be used to build a new array. func (b *Int8Builder) NewArray() arrow.Array { return b.NewInt8Array() } // NewInt8Array creates a Int8 array from the memory buffers used by the builder and resets the Int8Builder // so it can be used to build a new array. func (b *Int8Builder) NewInt8Array() (a *Int8) { data := b.newData() a = NewInt8Data(data) data.Release() return } func (b *Int8Builder) newData() (data *Data) { bytesRequired := arrow.Int8Traits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(arrow.PrimitiveTypes.Int8, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *Int8Builder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case string: f, err := strconv.ParseInt(v, 10, 1*8) if err != nil { return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(int8(0)), Offset: dec.InputOffset(), } } b.Append(int8(f)) case float64: b.Append(int8(v)) case json.Number: f, err := strconv.ParseInt(v.String(), 10, 1*8) if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(int8(0)), Offset: dec.InputOffset(), } } b.Append(int8(f)) default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(int8(0)), Offset: dec.InputOffset(), } } return nil } func (b *Int8Builder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *Int8Builder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } type Uint8Builder struct { builder data *memory.Buffer rawData []uint8 } func NewUint8Builder(mem memory.Allocator) *Uint8Builder { return &Uint8Builder{builder: builder{refCount: 1, mem: mem}} } func (b *Uint8Builder) Type() arrow.DataType { return arrow.PrimitiveTypes.Uint8 } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *Uint8Builder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *Uint8Builder) Append(v uint8) { b.Reserve(1) b.UnsafeAppend(v) } func (b *Uint8Builder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *Uint8Builder) AppendEmptyValue() { b.Append(0) } func (b *Uint8Builder) UnsafeAppend(v uint8) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *Uint8Builder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *Uint8Builder) AppendValues(v []uint8, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.Uint8Traits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *Uint8Builder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.Uint8Traits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.Uint8Traits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *Uint8Builder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *Uint8Builder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.Uint8Traits.BytesRequired(n)) b.rawData = arrow.Uint8Traits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Uint8 array from the memory buffers used by the builder and resets the Uint8Builder // so it can be used to build a new array. func (b *Uint8Builder) NewArray() arrow.Array { return b.NewUint8Array() } // NewUint8Array creates a Uint8 array from the memory buffers used by the builder and resets the Uint8Builder // so it can be used to build a new array. func (b *Uint8Builder) NewUint8Array() (a *Uint8) { data := b.newData() a = NewUint8Data(data) data.Release() return } func (b *Uint8Builder) newData() (data *Data) { bytesRequired := arrow.Uint8Traits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(arrow.PrimitiveTypes.Uint8, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *Uint8Builder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case string: f, err := strconv.ParseUint(v, 10, 1*8) if err != nil { return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(uint8(0)), Offset: dec.InputOffset(), } } b.Append(uint8(f)) case float64: b.Append(uint8(v)) case json.Number: f, err := strconv.ParseUint(v.String(), 10, 1*8) if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(uint8(0)), Offset: dec.InputOffset(), } } b.Append(uint8(f)) default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(uint8(0)), Offset: dec.InputOffset(), } } return nil } func (b *Uint8Builder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *Uint8Builder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } type TimestampBuilder struct { builder dtype *arrow.TimestampType data *memory.Buffer rawData []arrow.Timestamp } func NewTimestampBuilder(mem memory.Allocator, dtype *arrow.TimestampType) *TimestampBuilder { return &TimestampBuilder{builder: builder{refCount: 1, mem: mem}, dtype: dtype} } func (b *TimestampBuilder) Type() arrow.DataType { return b.dtype } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *TimestampBuilder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *TimestampBuilder) Append(v arrow.Timestamp) { b.Reserve(1) b.UnsafeAppend(v) } func (b *TimestampBuilder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *TimestampBuilder) AppendEmptyValue() { b.Append(0) } func (b *TimestampBuilder) UnsafeAppend(v arrow.Timestamp) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *TimestampBuilder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *TimestampBuilder) AppendValues(v []arrow.Timestamp, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.TimestampTraits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *TimestampBuilder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.TimestampTraits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.TimestampTraits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *TimestampBuilder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *TimestampBuilder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.TimestampTraits.BytesRequired(n)) b.rawData = arrow.TimestampTraits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Timestamp array from the memory buffers used by the builder and resets the TimestampBuilder // so it can be used to build a new array. func (b *TimestampBuilder) NewArray() arrow.Array { return b.NewTimestampArray() } // NewTimestampArray creates a Timestamp array from the memory buffers used by the builder and resets the TimestampBuilder // so it can be used to build a new array. func (b *TimestampBuilder) NewTimestampArray() (a *Timestamp) { data := b.newData() a = NewTimestampData(data) data.Release() return } func (b *TimestampBuilder) newData() (data *Data) { bytesRequired := arrow.TimestampTraits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *TimestampBuilder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case string: loc, _ := b.dtype.GetZone() tm, _, err := arrow.TimestampFromStringInLocation(v, b.dtype.Unit, loc) if err != nil { return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(arrow.Timestamp(0)), Offset: dec.InputOffset(), } } b.Append(tm) case json.Number: n, err := v.Int64() if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(arrow.Timestamp(0)), Offset: dec.InputOffset(), } } b.Append(arrow.Timestamp(n)) case float64: b.Append(arrow.Timestamp(v)) default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(arrow.Timestamp(0)), Offset: dec.InputOffset(), } } return nil } func (b *TimestampBuilder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *TimestampBuilder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } type Time32Builder struct { builder dtype *arrow.Time32Type data *memory.Buffer rawData []arrow.Time32 } func NewTime32Builder(mem memory.Allocator, dtype *arrow.Time32Type) *Time32Builder { return &Time32Builder{builder: builder{refCount: 1, mem: mem}, dtype: dtype} } func (b *Time32Builder) Type() arrow.DataType { return b.dtype } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *Time32Builder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *Time32Builder) Append(v arrow.Time32) { b.Reserve(1) b.UnsafeAppend(v) } func (b *Time32Builder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *Time32Builder) AppendEmptyValue() { b.Append(0) } func (b *Time32Builder) UnsafeAppend(v arrow.Time32) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *Time32Builder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *Time32Builder) AppendValues(v []arrow.Time32, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.Time32Traits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *Time32Builder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.Time32Traits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.Time32Traits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *Time32Builder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *Time32Builder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.Time32Traits.BytesRequired(n)) b.rawData = arrow.Time32Traits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Time32 array from the memory buffers used by the builder and resets the Time32Builder // so it can be used to build a new array. func (b *Time32Builder) NewArray() arrow.Array { return b.NewTime32Array() } // NewTime32Array creates a Time32 array from the memory buffers used by the builder and resets the Time32Builder // so it can be used to build a new array. func (b *Time32Builder) NewTime32Array() (a *Time32) { data := b.newData() a = NewTime32Data(data) data.Release() return } func (b *Time32Builder) newData() (data *Data) { bytesRequired := arrow.Time32Traits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *Time32Builder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case string: tm, err := arrow.Time32FromString(v, b.dtype.Unit) if err != nil { return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(arrow.Time32(0)), Offset: dec.InputOffset(), } } b.Append(tm) case json.Number: n, err := v.Int64() if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(arrow.Time32(0)), Offset: dec.InputOffset(), } } b.Append(arrow.Time32(n)) case float64: b.Append(arrow.Time32(v)) default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(arrow.Time32(0)), Offset: dec.InputOffset(), } } return nil } func (b *Time32Builder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *Time32Builder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } type Time64Builder struct { builder dtype *arrow.Time64Type data *memory.Buffer rawData []arrow.Time64 } func NewTime64Builder(mem memory.Allocator, dtype *arrow.Time64Type) *Time64Builder { return &Time64Builder{builder: builder{refCount: 1, mem: mem}, dtype: dtype} } func (b *Time64Builder) Type() arrow.DataType { return b.dtype } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *Time64Builder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *Time64Builder) Append(v arrow.Time64) { b.Reserve(1) b.UnsafeAppend(v) } func (b *Time64Builder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *Time64Builder) AppendEmptyValue() { b.Append(0) } func (b *Time64Builder) UnsafeAppend(v arrow.Time64) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *Time64Builder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *Time64Builder) AppendValues(v []arrow.Time64, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.Time64Traits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *Time64Builder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.Time64Traits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.Time64Traits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *Time64Builder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *Time64Builder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.Time64Traits.BytesRequired(n)) b.rawData = arrow.Time64Traits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Time64 array from the memory buffers used by the builder and resets the Time64Builder // so it can be used to build a new array. func (b *Time64Builder) NewArray() arrow.Array { return b.NewTime64Array() } // NewTime64Array creates a Time64 array from the memory buffers used by the builder and resets the Time64Builder // so it can be used to build a new array. func (b *Time64Builder) NewTime64Array() (a *Time64) { data := b.newData() a = NewTime64Data(data) data.Release() return } func (b *Time64Builder) newData() (data *Data) { bytesRequired := arrow.Time64Traits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *Time64Builder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case string: tm, err := arrow.Time64FromString(v, b.dtype.Unit) if err != nil { return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(arrow.Time64(0)), Offset: dec.InputOffset(), } } b.Append(tm) case json.Number: n, err := v.Int64() if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(arrow.Time64(0)), Offset: dec.InputOffset(), } } b.Append(arrow.Time64(n)) case float64: b.Append(arrow.Time64(v)) default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(arrow.Time64(0)), Offset: dec.InputOffset(), } } return nil } func (b *Time64Builder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *Time64Builder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } type Date32Builder struct { builder data *memory.Buffer rawData []arrow.Date32 } func NewDate32Builder(mem memory.Allocator) *Date32Builder { return &Date32Builder{builder: builder{refCount: 1, mem: mem}} } func (b *Date32Builder) Type() arrow.DataType { return arrow.PrimitiveTypes.Date32 } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *Date32Builder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *Date32Builder) Append(v arrow.Date32) { b.Reserve(1) b.UnsafeAppend(v) } func (b *Date32Builder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *Date32Builder) AppendEmptyValue() { b.Append(0) } func (b *Date32Builder) UnsafeAppend(v arrow.Date32) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *Date32Builder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *Date32Builder) AppendValues(v []arrow.Date32, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.Date32Traits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *Date32Builder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.Date32Traits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.Date32Traits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *Date32Builder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *Date32Builder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.Date32Traits.BytesRequired(n)) b.rawData = arrow.Date32Traits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Date32 array from the memory buffers used by the builder and resets the Date32Builder // so it can be used to build a new array. func (b *Date32Builder) NewArray() arrow.Array { return b.NewDate32Array() } // NewDate32Array creates a Date32 array from the memory buffers used by the builder and resets the Date32Builder // so it can be used to build a new array. func (b *Date32Builder) NewDate32Array() (a *Date32) { data := b.newData() a = NewDate32Data(data) data.Release() return } func (b *Date32Builder) newData() (data *Data) { bytesRequired := arrow.Date32Traits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(arrow.PrimitiveTypes.Date32, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *Date32Builder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case string: tm, err := time.Parse("2006-01-02", v) if err != nil { return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(arrow.Date32(0)), Offset: dec.InputOffset(), } } b.Append(arrow.Date32FromTime(tm)) case json.Number: n, err := v.Int64() if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(arrow.Date32(0)), Offset: dec.InputOffset(), } } b.Append(arrow.Date32(n)) case float64: b.Append(arrow.Date32(v)) default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(arrow.Date32(0)), Offset: dec.InputOffset(), } } return nil } func (b *Date32Builder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *Date32Builder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } type Date64Builder struct { builder data *memory.Buffer rawData []arrow.Date64 } func NewDate64Builder(mem memory.Allocator) *Date64Builder { return &Date64Builder{builder: builder{refCount: 1, mem: mem}} } func (b *Date64Builder) Type() arrow.DataType { return arrow.PrimitiveTypes.Date64 } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *Date64Builder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *Date64Builder) Append(v arrow.Date64) { b.Reserve(1) b.UnsafeAppend(v) } func (b *Date64Builder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *Date64Builder) AppendEmptyValue() { b.Append(0) } func (b *Date64Builder) UnsafeAppend(v arrow.Date64) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *Date64Builder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *Date64Builder) AppendValues(v []arrow.Date64, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.Date64Traits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *Date64Builder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.Date64Traits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.Date64Traits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *Date64Builder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *Date64Builder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.Date64Traits.BytesRequired(n)) b.rawData = arrow.Date64Traits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Date64 array from the memory buffers used by the builder and resets the Date64Builder // so it can be used to build a new array. func (b *Date64Builder) NewArray() arrow.Array { return b.NewDate64Array() } // NewDate64Array creates a Date64 array from the memory buffers used by the builder and resets the Date64Builder // so it can be used to build a new array. func (b *Date64Builder) NewDate64Array() (a *Date64) { data := b.newData() a = NewDate64Data(data) data.Release() return } func (b *Date64Builder) newData() (data *Data) { bytesRequired := arrow.Date64Traits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(arrow.PrimitiveTypes.Date64, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *Date64Builder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case string: tm, err := time.Parse("2006-01-02", v) if err != nil { return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(arrow.Date64(0)), Offset: dec.InputOffset(), } } b.Append(arrow.Date64FromTime(tm)) case json.Number: n, err := v.Int64() if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(arrow.Date64(0)), Offset: dec.InputOffset(), } } b.Append(arrow.Date64(n)) case float64: b.Append(arrow.Date64(v)) default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(arrow.Date64(0)), Offset: dec.InputOffset(), } } return nil } func (b *Date64Builder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *Date64Builder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } type DurationBuilder struct { builder dtype *arrow.DurationType data *memory.Buffer rawData []arrow.Duration } func NewDurationBuilder(mem memory.Allocator, dtype *arrow.DurationType) *DurationBuilder { return &DurationBuilder{builder: builder{refCount: 1, mem: mem}, dtype: dtype} } func (b *DurationBuilder) Type() arrow.DataType { return b.dtype } // Release decreases the reference count by 1. // When the reference count goes to zero, the memory is freed. func (b *DurationBuilder) Release() { debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") if atomic.AddInt64(&b.refCount, -1) == 0 { if b.nullBitmap != nil { b.nullBitmap.Release() b.nullBitmap = nil } if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } } } func (b *DurationBuilder) Append(v arrow.Duration) { b.Reserve(1) b.UnsafeAppend(v) } func (b *DurationBuilder) AppendNull() { b.Reserve(1) b.UnsafeAppendBoolToBitmap(false) } func (b *DurationBuilder) AppendEmptyValue() { b.Append(0) } func (b *DurationBuilder) UnsafeAppend(v arrow.Duration) { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) b.rawData[b.length] = v b.length++ } func (b *DurationBuilder) UnsafeAppendBoolToBitmap(isValid bool) { if isValid { bitutil.SetBit(b.nullBitmap.Bytes(), b.length) } else { b.nulls++ } b.length++ } // AppendValues will append the values in the v slice. The valid slice determines which values // in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, // all values in v are appended and considered valid. func (b *DurationBuilder) AppendValues(v []arrow.Duration, valid []bool) { if len(v) != len(valid) && len(valid) != 0 { panic("len(v) != len(valid) && len(valid) != 0") } if len(v) == 0 { return } b.Reserve(len(v)) arrow.DurationTraits.Copy(b.rawData[b.length:], v) b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) } func (b *DurationBuilder) init(capacity int) { b.builder.init(capacity) b.data = memory.NewResizableBuffer(b.mem) bytesN := arrow.DurationTraits.BytesRequired(capacity) b.data.Resize(bytesN) b.rawData = arrow.DurationTraits.CastFromBytes(b.data.Bytes()) } // Reserve ensures there is enough space for appending n elements // by checking the capacity and calling Resize if necessary. func (b *DurationBuilder) Reserve(n int) { b.builder.reserve(n, b.Resize) } // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), // additional memory will be allocated. If n is smaller, the allocated memory may reduced. func (b *DurationBuilder) Resize(n int) { nBuilder := n if n < minBuilderCapacity { n = minBuilderCapacity } if b.capacity == 0 { b.init(n) } else { b.builder.resize(nBuilder, b.init) b.data.Resize(arrow.DurationTraits.BytesRequired(n)) b.rawData = arrow.DurationTraits.CastFromBytes(b.data.Bytes()) } } // NewArray creates a Duration array from the memory buffers used by the builder and resets the DurationBuilder // so it can be used to build a new array. func (b *DurationBuilder) NewArray() arrow.Array { return b.NewDurationArray() } // NewDurationArray creates a Duration array from the memory buffers used by the builder and resets the DurationBuilder // so it can be used to build a new array. func (b *DurationBuilder) NewDurationArray() (a *Duration) { data := b.newData() a = NewDurationData(data) data.Release() return } func (b *DurationBuilder) newData() (data *Data) { bytesRequired := arrow.DurationTraits.BytesRequired(b.length) if bytesRequired > 0 && bytesRequired < b.data.Len() { // trim buffers b.data.Resize(bytesRequired) } data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) b.reset() if b.data != nil { b.data.Release() b.data = nil b.rawData = nil } return } func (b *DurationBuilder) unmarshalOne(dec *json.Decoder) error { t, err := dec.Token() if err != nil { return err } switch v := t.(type) { case nil: b.AppendNull() case json.Number: n, err := v.Int64() if err != nil { return &json.UnmarshalTypeError{ Value: v.String(), Type: reflect.TypeOf(arrow.Duration(0)), Offset: dec.InputOffset(), } } b.Append(arrow.Duration(n)) case float64: b.Append(arrow.Duration(v)) case string: // be flexible for specifying durations by accepting forms like // 3h2m0.5s regardless of the unit and converting it to the proper // precision. val, err := time.ParseDuration(v) if err != nil { // if we got an error, maybe it was because the attempt to create // a time.Duration (int64) in nanoseconds would overflow. check if // the string is just a large number followed by the unit suffix if strings.HasSuffix(v, b.dtype.Unit.String()) { value, err := strconv.ParseInt(v[:len(v)-len(b.dtype.Unit.String())], 10, 64) if err == nil { b.Append(arrow.Duration(value)) break } } return &json.UnmarshalTypeError{ Value: v, Type: reflect.TypeOf(arrow.Duration(0)), Offset: dec.InputOffset(), } } switch b.dtype.Unit { case arrow.Nanosecond: b.Append(arrow.Duration(val.Nanoseconds())) case arrow.Microsecond: b.Append(arrow.Duration(val.Microseconds())) case arrow.Millisecond: b.Append(arrow.Duration(val.Milliseconds())) case arrow.Second: b.Append(arrow.Duration(val.Seconds())) } default: return &json.UnmarshalTypeError{ Value: fmt.Sprint(t), Type: reflect.TypeOf(arrow.Duration(0)), Offset: dec.InputOffset(), } } return nil } func (b *DurationBuilder) unmarshal(dec *json.Decoder) error { for dec.More() { if err := b.unmarshalOne(dec); err != nil { return err } } return nil } func (b *DurationBuilder) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) t, err := dec.Token() if err != nil { return err } if delim, ok := t.(json.Delim); !ok || delim != '[' { return fmt.Errorf("binary builder must unpack from json array, found %s", delim) } return b.unmarshal(dec) } var ( _ Builder = (*Int64Builder)(nil) _ Builder = (*Uint64Builder)(nil) _ Builder = (*Float64Builder)(nil) _ Builder = (*Int32Builder)(nil) _ Builder = (*Uint32Builder)(nil) _ Builder = (*Float32Builder)(nil) _ Builder = (*Int16Builder)(nil) _ Builder = (*Uint16Builder)(nil) _ Builder = (*Int8Builder)(nil) _ Builder = (*Uint8Builder)(nil) _ Builder = (*TimestampBuilder)(nil) _ Builder = (*Time32Builder)(nil) _ Builder = (*Time64Builder)(nil) _ Builder = (*Date32Builder)(nil) _ Builder = (*Date64Builder)(nil) _ Builder = (*DurationBuilder)(nil) )