Sha256: e4f23f2db5177cd1530739837fc9ceaecff7a0aa2edf83eed4ceaaac095e0ca4

Contents?: true

Size: 1.5 KB

Versions: 15

Compression:

Stored size: 1.5 KB

Contents

#!/usr/bin/env ruby
require 'optparse'
require 'ostruct'
require 'cgi'

require "rubygems"
require "mandy"

HadoopConfiguration.check_home_and_version

options = OpenStruct.new

OptionParser.new do |opts|
  opts.banner = "USAGE: mandy-local my_script.rb local_input_file local_output_folder [options]"
  
  opts.on("-v", '--variables name=value', "Pass additional parameters to jobs") do |config|
    options.cmdenv = config
  end
  
  opts.on("-j", '--json {"key":"1 value"}', "Pass JSON encoded parameters to jobs") do |config|
    options.cmdenv = "json=#{CGI.escape(config)}"
  end
  
  opts.on_tail("-h", "--help", "Show this message") do
    puts opts
    exit
  end
end.parse!

exec('mandy-local -h') unless ARGV.size >= 3

require "fileutils"

def absolute_path(path)
  path =~ /^\// ? path : File.join(Dir.pwd, path)
end

def set_env(opts_string)
  opts_string.split(' ').each do |pair|
    key, value = pair.split("=")
    ENV[key] = value
  end
end

set_env(options.cmdenv) if options.cmdenv 
file   = absolute_path(ARGV[0])
input  = ARGV[1].split(",").map {|f| absolute_path(f)}.join(" ")
output_folder = FileUtils.mkdir_p(absolute_path(ARGV[2]))
require file

out = nil
Mandy::Job.jobs.each_with_index do |job, i|
  out = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
  puts "Running #{job.name}..."
  reduce_phase = job.reducer_defined? ? %(| sort | mandy-reduce #{file} "#{job.name}") : ''
  `cat #{input} | mandy-map #{file} "#{job.name}" #{reduce_phase} > #{out}`
  input = out
end

puts out

Version data entries

15 entries across 15 versions & 1 rubygems

Version Path
mandy-0.5.28 bin/mandy-local
mandy-0.5.27 bin/mandy-local
mandy-0.5.26 bin/mandy-local
mandy-0.5.25 bin/mandy-local
mandy-0.5.24 bin/mandy-local
mandy-0.5.23 bin/mandy-local
mandy-0.5.22 bin/mandy-local
mandy-0.5.21 bin/mandy-local
mandy-0.5.20 bin/mandy-local
mandy-0.5.19 bin/mandy-local
mandy-0.5.17 bin/mandy-local
mandy-0.5.14 bin/mandy-local
mandy-0.5.13 bin/mandy-local
mandy-0.5.11 bin/mandy-local
mandy-0.5.10 bin/mandy-local