Contents

#!/usr/bin/env bash

input_file=${1} 			; shift
output_file=${1} 			; shift
map_script=${1-/bin/cat}		; shift
reduce_script=${1-/usr/bin/uniq}	; shift

if [ "$output_file" == "" ] ; then echo "$0 input_file output_file [mapper=/bin/cat] [reducer=/usr/bin/uniq] [extra_args]" ; exit ; fi

HADOOP_HOME=${HADOOP_HOME-/usr/lib/hadoop}

# Can add fun stuff like
# -jobconf mapred.map.tasks=3                                                       \
# -jobconf mapred.reduce.tasks=3                                                    \

exec ${HADOOP_HOME}/bin/hadoop \
     jar         ${HADOOP_HOME}/contrib/streaming/hadoop-*-streaming.jar		\
    "$@"                                                                                \
    -jobconf    "mapred.job.name=`basename $0`-$map_script-$input_file-$output_file"    \
    -mapper  	"$map_script"  								\
    -reducer	"$reduce_script"							\
    -input      "$input_file"								\
    -output  	"$output_file"

Version data entries

1 entries across 1 versions & 1 rubygems

Version	Path
wukong-1.4.9	bin/hdp-stream-flat