#!/usr/bin/env ruby # Commandline client for controlling morph and running scrapers and things require "thor" require "rest_client" # TODO Do compression on the tar file #require 'zlib' require 'archive/tar/minitar' require 'pathname' require 'json' class MorphThor < Thor class_option :dev, default: false, type: :boolean, desc: "Run against a local dev of Morph running at http://localhost:3000" desc "[execute]", "execute morph scraper" option :directory, :default => Dir.getwd def execute api_key = retrieve_api_key if api_key.nil? api_key = ask("What is your key? (Go to #{base_url(options)}/settings)") save_api_key(api_key) end api_key_is_valid = false until api_key_is_valid begin puts "Uploading and running..." file = create_tar(options[:directory], all_paths(options[:directory])) result = RestClient.post("#{base_url(options)}/run", :api_key => api_key, :code => file) api_key_is_valid = true rescue RestClient::Unauthorized puts "Your key isn't working. Let's try again." api_key = ask("What is your key? (Go to #{base_url(options)}/settings)") save_api_key(api_key) end end # Interpret each line separately as json result.split("\n").each do |line| a = JSON.parse(line) if a["stream"] == "stdout" s = $stdout elsif a["stream"] == "stderr" s = $stderr else raise "Unknown stream" end s.puts a["text"] end end no_commands { def base_url(options) if options[:dev] "http://127.0.0.1:3000" else "https://morph.io" end end def config_path File.join(Dir.home, ".morph") end def save_api_key(api_key) configuration = {api_key: api_key} File.open(config_path, "w") {|f| f.write configuration.to_yaml} File.chmod(0600, config_path) end def retrieve_api_key if File.exists?(config_path) YAML.load(File.read(config_path))[:api_key] end end # TODO Temporary file should be named differently every time def create_dir_tar(directory) in_directory(directory) do tempfile = File.new('/tmp/out', 'wb') Archive::Tar::Minitar.pack('.', tempfile) File.new('/tmp/out', 'r') end end def in_directory(directory) cwd = FileUtils.pwd FileUtils.cd(directory) yield ensure FileUtils.cd(cwd) end def create_tar(directory, paths) tempfile = File.new('/tmp/out', 'wb') in_directory(directory) do begin tar = Archive::Tar::Minitar::Output.new("/tmp/out") paths.each do |entry| Archive::Tar::Minitar.pack_file(entry, tar) end ensure tar.close end end File.new('/tmp/out', 'r') end # Relative paths to all the files in the given directory (recursive) # (except for anything below a directory starting with ".") def all_paths(directory) result = [] Find.find(directory) do |path| if FileTest.directory?(path) if File.basename(path)[0] == ?. Find.prune end else result << Pathname.new(path).relative_path_from(Pathname.new(directory)).to_s end end result end # Relative path of database file (if it exists) def database_path(directory) path = "data.sqlite" path if File.exists?(File.join(directory, path)) end } end # If morph is run without any parameters it's the same as "morph execute" MorphThor.start(ARGV.empty? ? ["execute"] : ARGV)