Contents

hdfs_example: &hdfs_example
  config_files:
    - /etc/hadoop/conf/core-site.xml
    - /etc/hadoop/conf/hdfs-site.xml
  config:
    fs.defaultFS: 'hdfs://hadoop-nn1:8020'
    fs.hdfs.impl: 'org.apache.hadoop.hdfs.DistributedFileSystem'
    fs.file.impl: 'org.apache.hadoop.fs.LocalFileSystem'

local_fs_example: &local_fs_example
  config:
    fs.defaultFS: 'file:///'
    fs.hdfs.impl: 'org.apache.hadoop.fs.LocalFileSystem'
    fs.file.impl: 'org.apache.hadoop.fs.LocalFileSystem'
    io.compression.codecs: 'org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.BZip2Codec'

in:
  type: hdfs
  <<: *local_fs_example
  path: example/data*
  skip_header_lines: 1
  decompression: true
  parser:
    charset: UTF-8
    newline: CRLF
    type: csv
    delimiter: ','
    quote: '"'
    header_line: true
    columns:
    - {name: id, type: long}
    - {name: account, type: long}
    - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
    - {name: purchase, type: timestamp, format: '%Y%m%d'}
    - {name: comment, type: string}

out:
  type: stdout

Version data entries

4 entries across 4 versions & 1 rubygems

Version	Path
embulk-input-hdfs-0.3.2	example/config.yml
embulk-input-hdfs-0.3.1	example/config.yml
embulk-input-hdfs-0.3.0	example/config.yml
embulk-input-hdfs-0.2.1	example/config.yml