test/plugin/base_test.rb in fluent-plugin-google-cloud-0.7.30 vs test/plugin/base_test.rb in fluent-plugin-google-cloud-0.8.0

- old
+ new

@@ -125,46 +125,10 @@ end end end end - def test_configure_partial_success - setup_gce_metadata_stubs - { - APPLICATION_DEFAULT_CONFIG => true, - PARTIAL_SUCCESS_DISABLED_CONFIG => false - }.each do |(config, partial_success)| - d = create_driver(config) - assert_equal partial_success, - d.instance.instance_variable_get(:@partial_success) - end - end - - def test_metadata_agent_url_customization - [ - # If @metadata_agent_url is set, use that even if the environment - # variable is set. - [CUSTOM_METADATA_AGENT_URL_CONFIG, true, CUSTOM_METADATA_AGENT_URL], - # If @metadata_agent_url is set and the environment variable is - # not set, use @metadata_agent_url. - [CUSTOM_METADATA_AGENT_URL_CONFIG, false, CUSTOM_METADATA_AGENT_URL], - # If @metadata_agent_url is not set and the environment variable is set, - # use the env. - [APPLICATION_DEFAULT_CONFIG, true, METADATA_AGENT_URL_FROM_ENV], - # If @metadata_agent_url is not set and the environment variable is - # not set, fall back to the default. - [APPLICATION_DEFAULT_CONFIG, false, DEFAULT_METADATA_AGENT_URL] - ].each do |(config, url_from_env, expected_url)| - ENV[METADATA_AGENT_URL_ENV_VAR] = METADATA_AGENT_URL_FROM_ENV if - url_from_env - setup_gce_metadata_stubs - d = create_driver(ENABLE_METADATA_AGENT_CONFIG + config) - assert_equal expected_url, d.instance.metadata_agent_url - ENV.delete(METADATA_AGENT_URL_ENV_VAR) - end - end - def test_configure_ignores_unknown_monitoring_type # Verify that driver creation succeeds when monitoring type is not # "prometheus" (in which case, we simply don't record metrics), # and that the counters are set to nil. setup_gce_metadata_stubs @@ -1756,180 +1720,52 @@ verify_log_entries(1, expected_params) end end end - # Metadata Agent related tests. - - # Test enable_metadata_agent not set or set to false. - def test_configure_enable_metadata_agent_default_and_false - setup_gce_metadata_stubs - [create_driver, create_driver(DISABLE_METADATA_AGENT_CONFIG)].each do |d| - assert_false d.instance.instance_variable_get(:@enable_metadata_agent) - end - end - - # Test enable_metadata_agent set to true. - def test_configure_enable_metadata_agent_true - new_stub_context do - setup_gce_metadata_stubs - setup_metadata_agent_stubs - d = create_driver(ENABLE_METADATA_AGENT_CONFIG) - assert_true d.instance.instance_variable_get(:@enable_metadata_agent) - end - end - - # Docker Container. - - # Test textPayload logs from Docker container stdout / stderr. - def test_docker_container_stdout_stderr_logs_text_payload - [1, 2, 3, 5, 11, 50].each do |n| - new_stub_context do - setup_gce_metadata_stubs - setup_metadata_agent_stubs - setup_logging_stubs do - d = create_driver(DOCKER_CONTAINER_CONFIG) - n.times do |i| - d.emit(docker_container_stdout_stderr_log_entry(log_entry(i))) - end - d.run - end - verify_log_entries(n, DOCKER_CONTAINER_PARAMS) - assert_requested_metadata_agent_stub("container.#{DOCKER_CONTAINER_ID}") - end - end - end - - # Test jsonPayload logs from Docker container stdout / stderr. - def test_docker_container_stdout_stderr_logs_json_payload - [1, 2, 3, 5, 11, 50].each do |n| - new_stub_context do - setup_gce_metadata_stubs - setup_metadata_agent_stubs - setup_logging_stubs do - d = create_driver(DOCKER_CONTAINER_CONFIG) - n.times do - d.emit(docker_container_stdout_stderr_log_entry( - '{"msg": "test log entry ' \ - "#{n}" \ - '", "tag2": "test", "data": ' \ - '5000, "severity": "WARNING"}')) - end - d.run - end - verify_log_entries(n, DOCKER_CONTAINER_PARAMS, 'jsonPayload') do |entry| - fields = entry['jsonPayload'] - assert_equal 3, fields.size, entry - assert_equal "test log entry #{n}", fields['msg'], entry - assert_equal 'test', fields['tag2'], entry - assert_equal 5000, fields['data'], entry - end - assert_requested_metadata_agent_stub("container.#{DOCKER_CONTAINER_ID}") - end - end - end - - # Test logs from applications running in Docker containers. These logs have - # the label "logging.googleapis.com/local_resource_id" set in the format of - # "container.<container_name>". - def test_docker_container_application_logs - new_stub_context do - setup_gce_metadata_stubs - setup_metadata_agent_stubs - setup_logging_stubs do - # Metadata Agent is not enabled. Will call Docker Remote API for - # container info. - d = create_driver(ENABLE_METADATA_AGENT_CONFIG) - d.emit(docker_container_application_log_entry(log_entry(0))) - d.run - end - verify_log_entries(1, DOCKER_CONTAINER_PARAMS_NO_STREAM) - assert_requested_metadata_agent_stub( - "#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{DOCKER_CONTAINER_NAME}") - end - end - - # Test k8s_container monitored resource including the fallback when Metadata - # Agent restarts. - def test_k8s_container_monitored_resource_fallback + # Test k8s_container monitored resource. + def test_k8s_container_monitored_resource [ - # When enable_metadata_agent is false. { config: APPLICATION_DEFAULT_CONFIG, - setup_metadata_agent_stub: false, setup_k8s_stub: false, log_entry: k8s_container_log_entry(log_entry(0)), expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK }, { config: APPLICATION_DEFAULT_CONFIG, - setup_metadata_agent_stub: true, setup_k8s_stub: false, log_entry: k8s_container_log_entry(log_entry(0)), expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK }, { config: APPLICATION_DEFAULT_CONFIG, - setup_metadata_agent_stub: true, setup_k8s_stub: true, log_entry: k8s_container_log_entry(log_entry(0)), expected_params: K8S_CONTAINER_PARAMS_FROM_LOCAL }, { config: APPLICATION_DEFAULT_CONFIG, - setup_metadata_agent_stub: false, setup_k8s_stub: true, log_entry: k8s_container_log_entry(log_entry(0)), expected_params: K8S_CONTAINER_PARAMS_FROM_LOCAL }, - # When enable_metadata_agent is true. { - config: ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: false, + config: CUSTOM_K8S_CONFIG, setup_k8s_stub: false, log_entry: k8s_container_log_entry(log_entry(0)), - expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK - }, - { - config: ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: false, - setup_k8s_stub: true, - log_entry: k8s_container_log_entry(log_entry(0)), - expected_params: K8S_CONTAINER_PARAMS_FROM_LOCAL - }, - { - config: CUSTOM_K8S_ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: false, - setup_k8s_stub: false, - log_entry: k8s_container_log_entry(log_entry(0)), expected_params: K8S_CONTAINER_PARAMS_CUSTOM }, { - config: EMPTY_K8S_ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: true, + config: EMPTY_K8S_CONFIG, setup_k8s_stub: true, log_entry: k8s_container_log_entry(log_entry(0)), - expected_params: K8S_CONTAINER_PARAMS - }, - { - config: ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: true, - setup_k8s_stub: false, - log_entry: k8s_container_log_entry(log_entry(0)), - expected_params: K8S_CONTAINER_PARAMS - }, - { - config: ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: true, - setup_k8s_stub: true, - log_entry: k8s_container_log_entry(log_entry(0)), - expected_params: K8S_CONTAINER_PARAMS + expected_params: K8S_CONTAINER_PARAMS_FROM_LOCAL } ].each do |test_params| new_stub_context do setup_gce_metadata_stubs - setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub]) setup_k8s_metadata_stubs(test_params[:setup_k8s_stub]) setup_logging_stubs do d = create_driver(test_params[:config], CONTAINER_TAG) d.emit(test_params[:log_entry]) d.run @@ -1947,30 +1783,27 @@ def test_k8s_container_monitored_resource_invalid_local_resource_id [ # When local_resource_id is not present or does not match k8s regexes. { - config: ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: true, + config: APPLICATION_DEFAULT_CONFIG, setup_k8s_stub: true, log_entry: k8s_container_log_entry( log_entry(0)).reject { |k, _| k == LOCAL_RESOURCE_ID_KEY }, expected_params: CONTAINER_FROM_TAG_PARAMS }, { - config: ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: true, + config: APPLICATION_DEFAULT_CONFIG, setup_k8s_stub: true, log_entry: k8s_container_log_entry( log_entry(0), local_resource_id: RANDOM_LOCAL_RESOURCE_ID), expected_params: CONTAINER_FROM_TAG_PARAMS } ].each do |test_params| new_stub_context do setup_gce_metadata_stubs - setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub]) setup_k8s_metadata_stubs(test_params[:setup_k8s_stub]) setup_logging_stubs do d = create_driver(test_params[:config], CONTAINER_TAG) d.emit(test_params[:log_entry]) d.run @@ -1980,53 +1813,34 @@ end end end end - # Test k8s_pod monitored resource including the fallback when Metadata Agent - # restarts. - def test_k8s_pod_monitored_resource_fallback + # Test k8s_pod monitored resource. + def test_k8s_pod_monitored_resource [ { config: APPLICATION_DEFAULT_CONFIG, - setup_metadata_agent_stub: true, setup_k8s_stub: true, log_entry: k8s_pod_log_entry(log_entry(0)), expected_params: K8S_POD_PARAMS_FROM_LOCAL }, { - config: ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: false, - setup_k8s_stub: true, - log_entry: k8s_pod_log_entry(log_entry(0)), - expected_params: K8S_POD_PARAMS_FROM_LOCAL - }, - { - config: CUSTOM_K8S_ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: false, + config: CUSTOM_K8S_CONFIG, setup_k8s_stub: false, log_entry: k8s_pod_log_entry(log_entry(0)), expected_params: K8S_POD_PARAMS_CUSTOM }, { - config: EMPTY_K8S_ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: true, + config: EMPTY_K8S_CONFIG, setup_k8s_stub: true, log_entry: k8s_pod_log_entry(log_entry(0)), - expected_params: K8S_POD_PARAMS - }, - { - config: ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: true, - setup_k8s_stub: true, - log_entry: k8s_pod_log_entry(log_entry(0)), - expected_params: K8S_POD_PARAMS + expected_params: K8S_POD_PARAMS_FROM_LOCAL } ].each do |test_params| new_stub_context do setup_gce_metadata_stubs - setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub]) setup_k8s_metadata_stubs(test_params[:setup_k8s_stub]) setup_logging_stubs do d = create_driver(test_params[:config]) d.emit(test_params[:log_entry]) d.run @@ -2040,53 +1854,34 @@ end end end end - # Test k8s_node monitored resource including the fallback when Metadata Agent - # restarts. - def test_k8s_node_monitored_resource_fallback + # Test k8s_node monitored resource. + def test_k8s_node_monitored_resource [ { config: APPLICATION_DEFAULT_CONFIG, - setup_metadata_agent_stub: true, setup_k8s_stub: true, log_entry: k8s_node_log_entry(log_entry(0)), expected_params: K8S_NODE_PARAMS_FROM_LOCAL }, { - config: ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: false, - setup_k8s_stub: true, - log_entry: k8s_node_log_entry(log_entry(0)), - expected_params: K8S_NODE_PARAMS_FROM_LOCAL - }, - { - config: CUSTOM_K8S_ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: false, + config: CUSTOM_K8S_CONFIG, setup_k8s_stub: false, log_entry: k8s_node_log_entry(log_entry(0)), expected_params: K8S_NODE_PARAMS_CUSTOM }, { - config: EMPTY_K8S_ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: true, + config: EMPTY_K8S_CONFIG, setup_k8s_stub: true, log_entry: k8s_node_log_entry(log_entry(0)), - expected_params: K8S_NODE_PARAMS - }, - { - config: ENABLE_METADATA_AGENT_CONFIG, - setup_metadata_agent_stub: true, - setup_k8s_stub: true, - log_entry: k8s_node_log_entry(log_entry(0)), - expected_params: K8S_NODE_PARAMS + expected_params: K8S_NODE_PARAMS_FROM_LOCAL } ].each do |test_params| new_stub_context do setup_gce_metadata_stubs - setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub]) setup_k8s_metadata_stubs(test_params[:setup_k8s_stub]) setup_logging_stubs do d = create_driver(test_params[:config]) d.emit(test_params[:log_entry]) d.run @@ -2100,87 +1895,10 @@ end end end end - # Test that the 'time' field from the json record is extracted and set to - # entry.timestamp for Docker container logs. - def test_time_field_extraction_for_docker_container_logs - new_stub_context do - setup_gce_metadata_stubs - setup_metadata_agent_stubs - setup_logging_stubs do - d = create_driver(ENABLE_METADATA_AGENT_CONFIG) - d.emit(docker_container_application_log_entry(log_entry(0))) - d.run - end - verify_log_entries(1, DOCKER_CONTAINER_PARAMS_NO_STREAM) do |entry, i| - verify_default_log_entry_text(entry['textPayload'], i, entry) - # Timestamp in 'time' field from log entry should be set properly. - actual_timestamp = timestamp_parse(entry['timestamp']) - assert_equal DOCKER_CONTAINER_SECONDS_EPOCH, - actual_timestamp['seconds'], entry - assert_equal DOCKER_CONTAINER_NANOS, actual_timestamp['nanos'], entry - end - assert_requested_metadata_agent_stub( - "#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{DOCKER_CONTAINER_NAME}") - end - end - - # Test that the 'source' field is properly extracted from the record json and - # set as a common label 'stream'. - def test_source_for_docker_container_logs - { - docker_container_stdout_stderr_log_entry( - log_entry(0), DOCKER_CONTAINER_STREAM_STDOUT) => - DOCKER_CONTAINER_PARAMS, - docker_container_stdout_stderr_log_entry( - log_entry(0), DOCKER_CONTAINER_STREAM_STDERR) => - DOCKER_CONTAINER_PARAMS_STREAM_STDERR, - docker_container_application_log_entry(log_entry(0)) => - DOCKER_CONTAINER_PARAMS_NO_STREAM, - docker_container_application_log_entry(log_entry(0)) \ - .merge('severity' => 'warning') => - DOCKER_CONTAINER_PARAMS_NO_STREAM - }.each do |log_entry, expected_params| - new_stub_context do - setup_gce_metadata_stubs - setup_metadata_agent_stubs - setup_logging_stubs do - d = create_driver(DOCKER_CONTAINER_CONFIG) - d.emit(log_entry) - d.run - end - verify_log_entries(1, expected_params) - end - end - end - - # Test GKE container logs. These logs have the label - # "logging.googleapis.com/local_resource_id" set in the format of - # "gke_container.<namespace_id>.<pod_name>.<container_name>". - def test_gke_container_logs - [1, 2, 3, 5, 11, 50].each do |n| - new_stub_context do - setup_gce_metadata_stubs - setup_k8s_metadata_stubs - setup_metadata_agent_stubs - setup_logging_stubs do - d = create_driver(ENABLE_METADATA_AGENT_CONFIG) - n.times do |i| - d.emit(gke_container_log_entry(log_entry(i))) - end - d.run - end - verify_log_entries(n, CONTAINER_FROM_APPLICATION_PARAMS) - assert_requested_metadata_agent_stub( - "#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \ - ".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}") - end - end - end - def test_uptime_metric setup_gce_metadata_stubs [ [ENABLE_PROMETHEUS_CONFIG, method(:assert_prometheus_metric_value)], [ENABLE_OPENCENSUS_CONFIG, method(:assert_opencensus_metric_value)] @@ -2313,44 +2031,18 @@ def clear_metrics Prometheus::Client.registry.instance_variable_set('@metrics', {}) OpenCensus::Stats.ensure_recorder.clear_stats end - # Metadata Agent. - - def metadata_request_url(local_resource_id) - "#{DEFAULT_METADATA_AGENT_URL}/monitoredResource/#{local_resource_id}" - end - # Provide a stub context that initializes @logs_sent, executes the block and # resets WebMock at the end. def new_stub_context @logs_sent = [] yield WebMock.reset! end - def setup_metadata_agent_stubs(should_respond = true) - if should_respond - MONITORED_RESOURCE_STUBS.each do |local_resource_id, resource| - stub_request(:get, metadata_request_url(local_resource_id)) - .to_return(status: 200, body: resource) - end - stub_request(:get, metadata_request_url(RANDOM_LOCAL_RESOURCE_ID)) - .to_return(status: 404, body: '') - else - # Simulate an environment with no metadata agent endpoint present. - stub_request(:get, - %r{#{DEFAULT_METADATA_AGENT_URL}\/monitoredResource/.*}) - .to_raise(Errno::EHOSTUNREACH) - end - end - - def assert_requested_metadata_agent_stub(local_resource_id) - assert_requested :get, metadata_request_url(local_resource_id) - end - # GKE Container. def container_tag_with_container_name(container_name) "kubernetes.#{K8S_POD_NAME}_#{K8S_NAMESPACE_NAME}_#{container_name}" end @@ -2386,35 +2078,9 @@ { log: log, LOCAL_RESOURCE_ID_KEY => "#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \ ".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}" - } - end - - # Docker Container. - - def docker_container_stdout_stderr_log_entry( - log, stream = DOCKER_CONTAINER_STREAM_STDOUT) - severity = if stream == DOCKER_CONTAINER_STREAM_STDOUT - 'INFO' - else - 'ERROR' - end - { - log: log, - source: stream, - severity: severity, - LOCAL_RESOURCE_ID_KEY => "container.#{DOCKER_CONTAINER_ID}" - } - end - - def docker_container_application_log_entry(log) - { - log: log, - time: DOCKER_CONTAINER_TIMESTAMP, - LOCAL_RESOURCE_ID_KEY => "#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}." \ - "#{DOCKER_CONTAINER_NAME}" } end # TODO(qingling128): Temporary fallback for metadata agent restarts. # k8s resources.