Sha256: cf09cc76fb8546ebdd2f6473b18e4bcf771572fe170c178b04c1e85a006d08a6
Contents?: true
Size: 1.09 KB
Versions: 6
Compression:
Stored size: 1.09 KB
Contents
/**
* <%= script_name %>
*
* Required parameters:
*
* -param INPUT_PATH Input path for script data (e.g. s3n://hawk-example-data/tutorial/excite.log.bz2)
* -param OUTPUT_PATH Output path for script data (e.g. s3n://my-output-bucket/<%= script_name %>)
*/
<% if not options[:skip_udf] %>
/**
* User-Defined Functions (UDFs)
*/
REGISTER '../udfs/python/<%= script_name %>.py' USING streaming_python AS <%= script_name %>;
<% end %>
-- This is an example of loading up input data
my_input_data = LOAD '$INPUT_PATH'
USING PigStorage('\t')
AS (field0:chararray, field1:chararray, field2:chararray);
-- This is an example pig operation
filtered = FILTER my_input_data
BY field0 IS NOT NULL;
-- This is an example call to a python user-defined function
with_udf_output = FOREACH filtered
GENERATE field0..field2,
<%= script_name %>.example_udf(field0) AS example_udf_field;
-- remove any existing data
rmf $OUTPUT_PATH;
-- store the results
STORE with_udf_output
INTO '$OUTPUT_PATH'
USING PigStorage('\t');
Version data entries
6 entries across 6 versions & 1 rubygems