module Flydata module Command class Setup < Base include Helpers LOG_PATH_EXAMPLES= %w(/var/log/httpd/access_log /var/log/apache2/access.log /var/log/httpd-access.log /var/log/apache2/access_log /var/log/messages /var/log/maillog /var/log/mysql/error.log /home/*/deploy/shared/log/*.log) OTHER = '-- None of above --' # readline settings for asking log path Readline.completion_append_character = "/" Readline.completion_proc = Proc.new do |str| Dir[str+'*'].grep( /^#{Regexp.escape(str)}/ ) end def initial_run run do puts shown = show_registered_redshift_entries if shown true else _run end end puts print_usage puts "Completed setup of FlyData!" end def run(&block) Flydata::Command::Login.new.run unless flydata.credentials.authenticated? ret = block_given? ? yield : _run Flydata::Command::Sender.new.restart if ret end private def _run #ask redshift case ask_data_entry_type when :redshift start_redshift_mode when :s3backup start_s3_backup_mode when :restart_flydata true else false end end def ask_data_entry_type choose_one('Choose an option', nil, ["Setup Redshift and S3 Backup", "Setup S3 Backup only", "Restart FlyData", "Cancel"], [:redshift, :s3backup, :restart_flydata, :cancel]) end #### redshift backup mode def start_redshift_mode newline show_registered_redshift_entries newline ask_adding_new_redshift end def show_registered_redshift_entries show_registered_entries('RedshiftFileDataEntry') do |de| say(" - #{de['display_name']}: #{de['log_path']} -> #{de['redshift_table_name']} (#{de['redshift_table_name']}_dev)") end end def ask_adding_new_redshift puts "Start registration of a new entry:" newline # select table on redshift puts "[Select a Table from your Redshift cluster]" table_name = ask_redshift_table_name return unless table_name newline # enter the log path puts "[Enter a Local Log Path]" puts "Enter the absolute path of your local log that you want to upload to the '#{table_name}' table on Redshift." log_path = ask_log_path("Log path (tab:completion, return:cancel) >> ") return unless log_path and not log_path.empty? newline # select the log type puts "[Select a Log Format]" log_file_type = choose_log_file_type(log_path) # confirm and save newline puts "[Confirm]" separator puts " table(redshift) -> #{table_name} (#{table_name}_dev)" puts " local log path -> #{log_path}" puts " log file format -> #{log_file_type}" separator return unless ask_yes_no("Are you sure you want to register this new entry?", true) create_redshift_log_entry(log_path, log_file_type, table_name) end def choose_log_file_type(target_path) choose_one("Please select the log file format of (#{target_path})", nil, %w(csv tsv json)) end def ask_redshift_table_name ret = flydata.data_port.redshift_table_list all_tables = ret['table_list'].collect {|tn| tn.gsub(/_dev$/, '')}.uniq prod_tables = ret['table_list'].select {|tn| not tn.end_with?('_dev')} dev_tables = ret['table_list'].select {|tn| tn.end_with?('_dev')} if all_tables.size < 1 raise "Could not find tables on Redshift. Please create some tables on your Redshift cluster." end if development? menu_list = all_tables.collect do |tn| menu = ["#{tn}", "( #{tn}_dev )"] if dev_tables.index("#{tn}_dev").nil? menu << "!WARN The '#{tn}_dev' table does not exist on your Redshift cluster" else menu << "OK" end menu end menu_str_list = format_menu_list(menu_list) message = " !DEVELOPMENT MODE! FlyData will only upload to tables with '_dev' suffix in the name." else menu_list = all_tables.collect do |tn| menu = ["#{tn}", "( #{tn}_dev )"] if prod_tables.index("#{tn}").nil? menu << "!WARN '#{tn}' table does not exist" else menu << "OK" end menu end menu_str_list = format_menu_list(menu_list) message = " !PRODUCTION MODE!" end menu_str = choose_one("Please select the table on Redshift that you want to use to store your local data. \n#{message}", nil, menu_str_list) if menu_str menu_str.split.first end end def create_redshift_log_entry(log_path, log_file_type, redshift_table_name) data_port = flydata.data_port.get flydata.data_entry.create( data_port_id: data_port['id'], log_path: log_path, log_file_type: log_file_type, redshift_table_name: redshift_table_name) say("Process added successfuly!") return true end #### s3 backup mode def start_s3_backup_mode # choose entries unless (shown = show_registered_entries) list = build_recommended_entries print_recommended(list) register_all(list) if ask_yes_no("Register all of these common entries?") end unless (shown ||= show_registered_entries) and not more_entry? begin show_registered_entries unless shown shown = false choose_log_selection(list) newline end while more_entry? end return true end def show_registered_entries(type='FileDataEntry', &block) data_entries = retrieve_data_entries @last_fetched_entries = data_entries data_entries = data_entries.select {|de| de['type'] == type} if data_entries and data_entries.size > 0 puts('Registered entries on FlyData: ') separator data_entries.each { |data_entry| if block_given? yield data_entry else say(" - #{data_entry['display_name']}\t#{data_entry['log_path']}") end } separator true else false end end def choose_log_path_from_examples candidates = (`ls #{LOG_PATH_EXAMPLES.join(' ')} 2>/dev/null`).split(/\s+/) candidates = candidates.find_all{|path| File.readable?(path)} if @last_fetched_entries candidates = candidates - @last_fetched_entries.map{|v| v['log_path']} end return OTHER unless candidates.size > 0 candidates << OTHER choice = nil say('Please select your log path for sending FlyData') newline choose do |menu| menu.index = :letter menu.index_suffix = ") " menu.prompt = "Your log path: " menu.choices(*candidates) {|item| choice = item} end newline choice end def ask_and_create_data_entry path = ask_log_path create_log_entry(path) end def build_recommended_entries path_options=[] Dir['/var/log*/**/*log'].each{|f| if (FileTest.file?(f) and FileTest.readable?(f) and ( f =~ /apache2|httpd|syslog|mail|auth/)) and !(@last_fetched_entries and @last_fetched_entries.any?{|e| e['log_path'] == f}) then path_options << f end} path_options end def print_recommended(list, options=false) newline puts " Recommended list:" list.each_with_index { |value, index| puts " #{index+1}) #{value} " } if options puts(" #{list.length+1}) Enter your own path") end newline end def register_all(list) list.each{ |item| create_log_entry(item) } list.reject!{|x| x} end def choose_log_selection(list) path = nil list = build_recommended_entries unless list path_options = list loop do if path_options.empty? ask_and_create_data_entry return end print_recommended(path_options, true) choice = Readline.readline("Here are some common logs, enter the number next to the logs to add the log. Input nothing to cancel.") return if choice.empty? if choice.to_i==path_options.length+1 ask_and_create_data_entry return elsif (path_options[choice.to_i-1] != nil ) path = path_options[choice.to_i-1] path_options.delete_at(choice.to_i-1) break else puts("Not a valid entry, please try again"); end end create_log_entry(path) end def ask_log_deletion(path) unless File.writable?(path) say("Skip log deletion setting...") say(" This path is readonly for current user.") say(" Change user or permission, if you want to set log deletion option.") newline return end say("** Log deletion setting **") say("Flydata has a log deletion feature that flydata will delete old log archives uploaded by flydata automatically.") say("Flydata will delete logs whose last modified timestamp is 7 days ago.") say("For more details - http://docs.hapyrus.com/faq/how-log-deletion-works/") ask_yes_no("Set auto log deletion mode?") end def create_log_entry(path, log_deletion=false) data_port = flydata.data_port.get flydata.data_entry.create(data_port_id: data_port['id'], log_path: path, log_deletion: log_deletion) say("Process added successfuly!") if flydata.response.code == 200 end def more_entry? ask_yes_no("Do you want to add another log path?") end def ask_log_path(message = nil) path = nil message = "Enter the absolute path of your log (return to cancel): " unless message loop do path = Readline.readline(message) return if path.empty? if not (FileTest.file?(path) and FileTest.readable?(path)) say(" ! #{path} is not a readable file!") elsif @last_fetched_entries and @last_fetched_entries.any?{|e| e['log_path'] == path} say(" ! #{path} has been registered already.") else break end newline end path end end end end