<source>
  @type forward
  port 24224
  bind 0.0.0.0
</source>

<source>
  @type dummy
  dummy {"json_field": {"foo": "val1", "bar": "val2", "hoge": 1}, "geography_field": {"type": "LineString", "coordinates": [[-118.4085, 33.9416], [-73.7781, 40.6413]]}, "timestamp_field": "2022-12-15T22:40:21+09:00", "date": "2022-12-15", "record_field": {"inner_field": "hoge", "inner_json": {"key1": "val1", "key2": "val2"}}, "repeated_string_field": ["a", "b", "c"]}
  auto_increment_key id

  tag insert_data
</source>

<match insert_data>
  @id bigquery-insert-integration
  @type bigquery_insert

  allow_retry_insert_errors true

  auth_method json_key
  json_key "#{ENV["GOOGLE_APPLICATION_CREDENTIALS"]}"

  <buffer>
    @type file

    chunk_limit_size 1m
    chunk_limit_records 1500
    total_limit_size 1g
    path ./log/bigquery-insert-integration

    flush_interval 15
    flush_thread_count 4
    flush_at_shutdown true

    retry_max_times 14
    retry_max_interval 30m
  </buffer>

  request_open_timeout_sec 2m

  slow_flush_log_threshold 30.0

  project "#{ENV["PROJECT_NAME"]}"
  dataset "#{ENV["DATASET_NAME"]}"
  table "#{ENV["TABLE_NAME"]}"
  auto_create_table false
  # schema_path integration/schema.json
  fetch_schema true
  fetch_schema_table "#{ENV["TABLE_NAME"]}"

  insert_id_field id

  <secondary>
    @type file
    path ./log/bigquery-insert-integration.errors
  </secondary>
</match>

<match load_data>
  @id bigquery-load-integration
  @type bigquery_load

  auth_method json_key
  json_key "#{ENV["GOOGLE_APPLICATION_CREDENTIALS"]}"

  <buffer>
    @type file

    chunk_limit_size 1m
    total_limit_size 1g
    path ./log/bigquery-load-integration

    flush_interval 120
    flush_thread_count 4
    flush_at_shutdown true

    retry_max_times 14
    retry_max_interval 30m
  </buffer>

  request_open_timeout_sec 2m

  slow_flush_log_threshold 300.0

  project "#{ENV["PROJECT_NAME"]}"
  dataset "#{ENV["DATASET_NAME"]}"
  table "#{ENV["TABLE_NAME"]}"
  auto_create_table false
  # schema_path integration/schema.json
  fetch_schema true
  fetch_schema_table "#{ENV["TABLE_NAME"]}"

  <secondary>
    @type file
    path ./log/bigquery-load-integration.errors
  </secondary>
</match>