lib/baran/character_text_splitter.rb in baran-0.1.4 vs lib/baran/character_text_splitter.rb in baran-0.1.5
- old
+ new
@@ -1,15 +1,17 @@
require_relative './text_splitter'
module Baran
class CharacterTextSplitter < TextSplitter
+ attr_accessor :separator
+
def initialize(chunk_size: 1024, chunk_overlap: 64, separator: nil)
super(chunk_size: chunk_size, chunk_overlap: chunk_overlap)
@separator = separator || "\n\n"
end
def splitted(text)
- splits = @separator.empty? ? text.chars : text.split(@separator)
+ splits = separator.empty? ? text.chars : text.split(separator)
merged(splits, @separator)
end
end
end
\ No newline at end of file