# encoding: UTF-8 # Copyright 2012 Twitter, Inc # http://www.apache.org/licenses/LICENSE-2.0 module TwitterCldr module Resources class UnicodeDataImporter < Importer BLOCKS_FILE = 'ucd/Blocks.txt' UNICODE_DATA_FILE = 'ucd/UnicodeData.txt' CASEFOLDING_DATA_FILE = 'ucd/CaseFolding.txt' requirement :unicode, Versions.unicode_version, [BLOCKS_FILE, UNICODE_DATA_FILE, CASEFOLDING_DATA_FILE] output_path 'unicode_data' ruby_engine :mri def execute blocks = import_blocks unicode_data = import_unicode_data(blocks) casefolding_data = import_casefolding_data STDOUT.write('Writing data to disk... ') File.open(File.join(output_path, 'blocks.yml'), 'w') do |output| YAML.dump(blocks, output) end FileUtils.mkdir_p(File.join(output_path, 'blocks')) unicode_data.each do |block_name, code_points| File.open(File.join(output_path, 'blocks', "#{block_name}.yml"), 'w') do |output| YAML.dump(code_points, output) end end File.open(File.join(output_path, 'casefolding.yml'), 'w') do |output| YAML.dump(casefolding_data, output) end puts 'done' end private def output_path params.fetch(:output_path) end def import_blocks STDOUT.write('Importing blocks... ') blocks = {} File.open(blocks_file) do |input| input.each_line do |line| next unless line =~ /^([0-9A-F]+)\.\.([0-9A-F]+);(.+)$/ range = ($1.hex..$2.hex) name = block_name($3) blocks[name.to_sym] = range end end puts 'done' blocks end def parse_file(file, &block) UnicodeFileParser.parse_standard_file(file, &block) end def import_unicode_data(blocks) STDOUT.write('Importing Unicode data... ') unicode_data = Hash.new do |hash, key| hash[key] = Hash.new { |h, k| h[k] = {} } end parse_file(unicode_data_file) do |data| data[0] = data[0].hex unicode_data[find_block(blocks, data[0]).first][data[0]] = data end puts 'done' unicode_data end def import_casefolding_data STDOUT.write('Importing casefolding data... ') casefolding_data = parse_file(casefold_data_file).map do |data| { source: data[0].hex, target: data[2].split(" ").map(&:hex), status: data[1] } end puts 'done' casefolding_data end def casefold_data_file requirements[:unicode].source_path_for(CASEFOLDING_DATA_FILE) end def unicode_data_file requirements[:unicode].source_path_for(UNICODE_DATA_FILE) end def blocks_file requirements[:unicode].source_path_for(BLOCKS_FILE) end def find_block(blocks, code_point) blocks.detect { |_, range| range.include?(code_point) } end def block_name(string) string.strip.downcase.gsub(/[\s-]/, '_') end end end end