class Eco::API::UseCases::Default::Utils::SplitJson < Eco::API::Common::Loaders::UseCase require_relative 'cli/split_json_cli' MAX_ITEMS = 15_000 name "split-json" type :other def main(*_args) if simulate? count = input_json.count log(:info) { "JSON '#{input_file}' has #{count} elements." } else split_json_into_files! msg = [] msg << "Total elements: #{total_count}" msg << "Generated files:" out_files.each do |file| msg << " * #{file}'" end log(:info) { msg.join("\n") } end end private def split_json_into_files! first = true while pending? swap_file! if first first = false curr_json << input_json.shift next_count! swap_file! if split? end generate_file! end def split? return false unless curr_count > max_items true end def max_items @max_items ||= max_items_options || self.class::MAX_ITEMS end # OPTIONS def max_items_options return unless (num = options.dig(:output, :file, :max_items)) num = num.to_i num = nil if num.zero? num end # OUTPUT JSON # Returns the last json array and blanks the variable def exporting_json! curr_json.tap do @curr_json = [] end end def curr_json @curr_json ||= [] end def pending? !input_json.empty? end def input_json @json ||= JSON.load(File.open(input_file, 'r')) end # COUNTS def next_count! @curr_count = curr_count + 1 @total_count = total_count + 1 end def total_count @total_count ||= 0 end def curr_count @curr_count ||= 0 end def new_count! @curr_count = 0 end # FILE GENERATION def swap_file! generate_file! next_file! end def generate_file! return unless @curr_file if curr_json.empty? log(:info) { msg = "No pending elements to be transferred " msg << "(skipping creation of file '#{curr_file}')" msg } out_files -= [curr_file] return end log(:info) { msg = "Generating file at elem #{total_count} " msg << "(file elements: #{curr_count})" msg } save_json(exporting_json!, curr_file) new_count! end def save_json(data, file) File.open(file, 'w') do |fd| first = true fd << '[' data.each do |elem| fd << "," unless first first = false fd << elem.to_json end fd << ']' end.tap do log(:info) { "Generated file '#{file}'" } end end # OUTPUT FILES attr_reader :curr_file def file_count @file_count ||= 0 end def next_file! @file_count = file_count + 1 @curr_file = generate_name(@file_count) @curr_file.tap do out_files << @curr_file end end def out_files @out_files ||= [] end def generate_name(fidx) File.join(input_dir, "#{input_name}_#{file_number(fidx)}#{input_ext}") end def file_number(num) "#{zeroed}#{num}"[-5..] end def zeroed "0" * 5 end # INPUT FILE def input_name @input_name ||= File.basename(input_basename, input_ext) end def input_ext @input_ext ||= input_basename.split('.')[1..].join('.').then do |name| ".#{name}" end end def input_basename @input_basename ||= File.basename(input_full_filename) end def input_dir @input_dir = File.dirname(input_full_filename) end def input_full_filename @input_full_filename ||= File.expand_path(input_file) end def input_file options.dig(:source, :file).tap do |file| next if file && File.exist?(file) unless file log(:warn) { 'No input file provided' } exit 1 end log(:warn) { "File '#{file}' doesn't exist" } end end end