Contents

<source>
  @type http
  port 9880
  <parse>
    @type openlineage
  </parse>
</source>


# https://docs.fluentd.org/output/http
# tag should match fluentd input endpoint url http://localhost:9880/api/v1/lineage
<match api.v1.lineage> 
  @type copy
  <store>
    @type http
    endpoint_url "#{ENV['MARQUEZ_HTTP_ENDPOINT']}"
    content_type application/json
    bulk_request false # available since using https://github.com/fluent-plugins-nursery/fluent-plugin-out-http
    buffered true
    serializer json
    retryable_response_codes 408, 429, 500, 502, 503
    <buffer>
      @type file
      path /tmp/openlineage/buf/chunk-*
      flush_mode immediate
    </buffer>
  </store>
  <store>
    @type stdout # testing purpose to demonstrate that copy is working
  </store>
</match>

# count the number of incoming records per tag
<filter company.*>
  @type prometheus
  <metric>
    name fluentd_input_status_num_records_total
    type counter
    desc The total number of incoming records
    <labels>
      tag ${tag}
      hostname ${hostname}
    </labels>
  </metric>
</filter>

# expose metrics in prometheus format
<source>
  @type prometheus
  bind 0.0.0.0
  port 24231
  metrics_path /metrics
</source>

<source>
  @type prometheus_output_monitor
  interval 10
  <labels>
    hostname ${hostname}
  </labels>
</source>

# monitor fluentd itself
<source>
  @type monitor_agent
  bind 0.0.0.0
  port 24220
  @id in_monitor_agent
  include_config true
</source>

Version data entries

1 entries across 1 versions & 1 rubygems

Version	Path
fluent-plugin-openlineage-light-0.1.3	misc/fluent.conf