lib/fluent/plugin/buf_arrow_memory.rb in fluent-plugin-arrow-0.0.3 vs lib/fluent/plugin/buf_arrow_memory.rb in fluent-plugin-arrow-0.0.4
- old
+ new
@@ -14,11 +14,10 @@
# limitations under the License.
require "arrow"
require 'fluent/plugin/buffer'
require 'fluent/plugin/buffer/arrow_memory_chunk'
-require 'fluent/plugin/arrow/field_wrapper'
module Fluent
module Plugin
class ArrowMemoryBuffer < Fluent::Plugin::Buffer
Plugin.register_buffer('arrow_memory', self)
@@ -31,22 +30,18 @@
def configure(conf)
super
# [{"name" => foo1, "type" => "uint64"}, {"name" => foo2, "type" => "struct", "fields" => [{"name" => bar1, "type" => "string"}]}
- @field_wrappers = @schema.each_with_object({}) do |field, h|
- h[field["name"]] = Fluent::Plugin::Arrow::FieldWrapper.build(field)
- end
-
- @arrow_schema = ::Arrow::Schema.new(@field_wrappers.values.map(&:arrow_field))
+ @arrow_schema = ::Arrow::Schema.new(@schema)
end
def resume
return {}, []
end
def generate_chunk(metadata)
- Fluent::Plugin::Buffer::ArrowMemoryChunk.new(metadata, @arrow_schema, @field_wrappers, chunk_size: @row_group_chunk_size, format: @arrow_format)
+ Fluent::Plugin::Buffer::ArrowMemoryChunk.new(metadata, @arrow_schema, chunk_size: @row_group_chunk_size, format: @arrow_format)
end
end
end
end