Sha256: e2f6d2c0d13029d8d9931c7c2c9a9a9bb8955e6533e57aa0064e4a5feed1656c
Contents?: true
Size: 986 Bytes
Versions: 8
Compression:
Stored size: 986 Bytes
Contents
#!/bin/bash
HADOOP_HOME=${HADOOP_HOME-/usr/lib/hadoop}
OUTPUT="$1" ; shift
INPUTS=''
for foo in $@; do
INPUTS="$INPUTS -input $foo\
"
done
echo "Removing output directory $OUTPUT"
hadoop fs -rmr $OUTPUT
cmd="${HADOOP_HOME}/bin/hadoop \
jar ${HADOOP_HOME}/contrib/streaming/hadoop-*streaming*.jar \
-partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner \
-jobconf mapred.output.compress=true \
-jobconf mapred.output.compression.codec=org.apache.hadoop.io.compress.BZip2Codec \
-jobconf mapred.reduce.tasks=1 \
-mapper \"/bin/cat\" \
-reducer \"/bin/cat\" \
$INPUTS
-output $OUTPUT \
"
echo $cmd
$cmd
Version data entries
8 entries across 8 versions & 1 rubygems