lib/baran/text_splitter.rb in baran-0.1.6 vs lib/baran/text_splitter.rb in baran-0.1.7

@@ -1,5 +1,7 @@
+require 'logger'
+
 module Baran
   class TextSplitter
     attr_accessor :chunk_size, :chunk_overlap
 
     def initialize(chunk_size: 1024, chunk_overlap: 64)
@@ -44,9 +46,10 @@
           end
         end
 
         current_splits << split
         total += split.length
+        Logger.new(STDOUT).warn("Created a chunk of size #{total}, which is longer than the specified #{@chunk_size}") if total > @chunk_size
       end
 
       results << joined(current_splits, separator)
 
       results
\ No newline at end of file