Contents

# == Load/StoreFunctions ==
# BinaryDeserializer
# BinarySerializer
# BinStorage
# PigStorage
# PigDump
# TextLoader

module Wukong
  module AndPig
    class PigVar
      #===========================================================================
      #
      # The "LOAD" pig expression:
      #   MyRelation = LOAD 'my_relation.tsv' AS (attr_a: int, attr_b: chararray) ;
      #
      # The AS type spec is generated from klass
      #
      def self.pig_load rel, klass, options={ }
        filename = options[:filename] || default_filename(rel)
        self.set rel, self.new(klass, rel, "LOAD    '#{filename}' AS #{klass.typify(options[:has_rsrc])}")
        if options[:has_rsrc]
          lval = self[rel]
          lval.generate lval, *lval.fields
        end
        rel
      end

      #===========================================================================
      #
      #
      # The "STORE" pig imperative:
      #   STORE Relation INTO 'filename'
      # If no filename is given, the relation's name is used
      #
      def store filename=nil
        filename ||= default_filename
        self.class.emit "STORE %-19s INTO    '%s'" % [relation, filename]
        self
      end

      # Store the relation, removing the existing file
      def store! filename=nil
        filename ||= default_filename
        rmf!  filename
        mkdir File.dirname(filename)
        store filename
      end

      # Force a store to disk, then load (so all calculations proceed from there)
      def checkpoint! options={}
        options = options.reverse_merge :filename => default_filename
        store!   options[:filename]
        self.class.pig_load(self.name, klass, options)
      end

      def default_filename
        self.class.default_filename self.name
      end
      def self.default_filename name
        File.join(working_dir, name.to_s)
      end
    end
  end
end

Version data entries

3 entries across 3 versions & 2 rubygems

Version	Path
mrflip-wukong-0.1.0	lib/wukong/and_pig/operators/load_store.rb
wukong-0.1.4	lib/wukong/and_pig/operators/load_store.rb
wukong-0.1.1	lib/wukong/and_pig/operators/load_store.rb