test/plugin/base_test.rb in fluent-plugin-google-cloud-0.6.24 vs test/plugin/base_test.rb in fluent-plugin-google-cloud-0.6.25.pre.1
- old
+ new
@@ -79,30 +79,31 @@
assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
assert_equal CUSTOM_ZONE, d.instance.zone
assert_equal CUSTOM_VM_ID, d.instance.vm_id
end
- def test_configure_invalid_metadata_missing_parts
+ def test_configure_metadata_missing_parts_on_other_platforms
setup_no_metadata_service_stubs
Fluent::GoogleCloudOutput::CredentialsInfo.stubs(:project_id).returns(nil)
- { CONFIG_MISSING_METADATA_PROJECT_ID => ['project_id'],
- CONFIG_MISSING_METADATA_ZONE => ['zone'],
- CONFIG_MISSING_METADATA_VM_ID => ['vm_id'],
- CONFIG_MISSING_METADATA_ALL => %w(project_id zone vm_id)
- }.each_with_index do |(config, parts), index|
- exception_count = 0
+ [[CONFIG_MISSING_METADATA_PROJECT_ID, ['project_id'], false],
+ [CONFIG_MISSING_METADATA_ZONE, [], true],
+ [CONFIG_MISSING_METADATA_VM_ID, [], true],
+ [CONFIG_MISSING_METADATA_ALL, ['project_id'], false]
+ ].each_with_index do |(config, missing_parts, is_valid_config), index|
begin
create_driver(config)
+ assert_true is_valid_config, "Invalid config at index #{index} should "\
+ 'have raised an error.'
rescue Fluent::ConfigError => error
+ assert_false is_valid_config, "Valid config at index #{index} should "\
+ "not have raised an error #{error}."
assert error.message.include?('Unable to obtain metadata parameters:'),
"Index #{index} failed."
- parts.each do |part|
+ missing_parts.each do |part|
assert error.message.include?(part), "Index #{index} failed."
end
- exception_count += 1
end
- assert_equal 1, exception_count, "Index #{index} failed."
end
end
def test_configure_partial_success
setup_gce_metadata_stubs
@@ -177,11 +178,11 @@
def test_gce_used_when_detect_subservice_is_false
setup_gce_metadata_stubs
# This would cause the resource type to be container.googleapis.com if not
# for the detect_subservice=false config.
- setup_container_metadata_stubs
+ setup_k8s_metadata_stubs
d = create_driver(NO_DETECT_SUBSERVICE_CONFIG)
d.run
assert_equal COMPUTE_CONSTANTS[:resource_type], d.instance.resource.type
end
@@ -465,11 +466,11 @@
end
end
def test_structured_payload_json_log_default_container_not_parsed
setup_gce_metadata_stubs
- setup_container_metadata_stubs
+ setup_k8s_metadata_stubs
json_string = '{"msg": "test log entry 0", "tag2": "test", ' \
'"data": 5000, "some_null_field": null}'
setup_logging_stubs do
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
d.emit(container_log_entry_with_metadata('notJSON' + json_string))
@@ -482,11 +483,11 @@
end
end
def test_structured_payload_json_log_detect_json_container_not_parsed
setup_gce_metadata_stubs
- setup_container_metadata_stubs
+ setup_k8s_metadata_stubs
json_string = '{"msg": "test log entry 0", "tag2": "test", ' \
'"data": 5000, "some_null_field": null}'
setup_logging_stubs do
d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
d.emit(container_log_entry_with_metadata('notJSON' + json_string))
@@ -497,11 +498,11 @@
end
end
def test_structured_payload_json_log_detect_json_container_parsed
setup_gce_metadata_stubs
- setup_container_metadata_stubs
+ setup_k8s_metadata_stubs
json_string = '{"msg": "test log entry 0", "tag2": "test", ' \
'"data": 5000, "some_null_field": null}'
setup_logging_stubs do
d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
d.emit(container_log_entry_with_metadata(json_string))
@@ -536,11 +537,11 @@
# Verify that empty string container name should fail the kubernetes regex
# match, thus the original tag is used as the log name.
def test_handle_empty_container_name
setup_gce_metadata_stubs
- setup_container_metadata_stubs
+ setup_k8s_metadata_stubs
container_name = ''
# This tag will not match the kubernetes regex because it requires a
# non-empty container name.
tag = container_tag_with_container_name(container_name)
setup_logging_stubs do
@@ -558,11 +559,11 @@
# Verify that container names with non-utf8 characters should be rejected when
# 'require_valid_tags' is true.
def test_reject_non_utf8_container_name_with_require_valid_tags_true
setup_gce_metadata_stubs
- setup_container_metadata_stubs
+ setup_k8s_metadata_stubs
non_utf8_tags = INVALID_TAGS.select do |tag, _|
tag.is_a?(String) && !tag.empty?
end
non_utf8_tags.each do |container_name, encoded_name|
setup_logging_stubs do
@@ -598,11 +599,11 @@
end
# Verify that tags extracted from container names are properly encoded.
def test_encode_tags_from_container_name_with_require_valid_tags_true
setup_gce_metadata_stubs
- setup_container_metadata_stubs
+ setup_k8s_metadata_stubs
VALID_TAGS.each do |tag, encoded_tag|
setup_logging_stubs do
@logs_sent = []
d = create_driver(REQUIRE_VALID_TAGS_CONFIG,
container_tag_with_container_name(tag))
@@ -637,11 +638,11 @@
# Verify that tags extracted from container names are properly encoded and
# sanitized.
def test_sanitize_tags_from_container_name_with_require_valid_tags_false
setup_gce_metadata_stubs
- setup_container_metadata_stubs
+ setup_k8s_metadata_stubs
# Log names are derived from container names for containers. And container
# names are extracted from the tag based on a regex match pattern. As a
# prerequisite, the tag should already be a string, thus we only test
# non-empty string cases here.
string_tags = ALL_TAGS.select { |tag, _| tag.is_a?(String) && !tag.empty? }
@@ -966,30 +967,30 @@
CONTAINER_FROM_TAG_PARAMS)
end
def test_one_container_log_from_tag_stderr
setup_gce_metadata_stubs
- setup_container_metadata_stubs
+ setup_k8s_metadata_stubs
setup_logging_stubs do
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
d.emit(container_log_entry(log_entry(0), 'stderr'))
d.run
end
expected_params = CONTAINER_FROM_TAG_PARAMS.merge(
labels: { "#{GKE_CONSTANTS[:service]}/stream" => 'stderr' }
) { |_, oldval, newval| oldval.merge(newval) }
verify_log_entries(1, expected_params) do |entry, i|
verify_default_log_entry_text(entry['textPayload'], i, entry)
- assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
- assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
+ assert_equal K8S_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+ assert_equal K8S_NANOS, entry['timestamp']['nanos'], entry
assert_equal 'ERROR', entry['severity'], entry
end
end
def test_json_container_log_metadata_from_plugin
setup_gce_metadata_stubs
- setup_container_metadata_stubs
+ setup_k8s_metadata_stubs
setup_logging_stubs do
d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
d.emit(container_log_entry_with_metadata('{"msg": "test log entry 0", ' \
'"tag2": "test", "data": ' \
'5000, "severity": "WARNING"}'))
@@ -1000,19 +1001,19 @@
fields = get_fields(entry['jsonPayload'])
assert_equal 3, fields.size, entry
assert_equal 'test log entry 0', get_string(fields['msg']), entry
assert_equal 'test', get_string(fields['tag2']), entry
assert_equal 5000, get_number(fields['data']), entry
- assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
- assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
+ assert_equal K8S_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+ assert_equal K8S_NANOS, entry['timestamp']['nanos'], entry
assert_equal 'WARNING', entry['severity'], entry
end
end
def test_json_container_log_metadata_from_tag
setup_gce_metadata_stubs
- setup_container_metadata_stubs
+ setup_k8s_metadata_stubs
setup_logging_stubs do
d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
d.emit(container_log_entry('{"msg": "test log entry 0", ' \
'"tag2": "test", "data": 5000, ' \
'"severity": "W"}'))
@@ -1023,12 +1024,12 @@
fields = get_fields(entry['jsonPayload'])
assert_equal 3, fields.size, entry
assert_equal 'test log entry 0', get_string(fields['msg']), entry
assert_equal 'test', get_string(fields['tag2']), entry
assert_equal 5000, get_number(fields['data']), entry
- assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
- assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
+ assert_equal K8S_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+ assert_equal K8S_NANOS, entry['timestamp']['nanos'], entry
assert_equal 'WARNING', entry['severity'], entry
end
end
def test_cloudfunctions_log
@@ -1337,29 +1338,28 @@
assert_requested_metadata_agent_stub(
"#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{DOCKER_CONTAINER_NAME}")
end
end
- # Test k8s monitored resource including the fallback when Metadata Agent
- # restarts.
- def test_k8s_monitored_resource_fallback
+ # Test k8s_container monitored resource including the fallback when Metadata
+ # Agent restarts.
+ def test_k8s_container_monitored_resource_fallback
[
- # k8s_container.
# When enable_metadata_agent is false.
{
config: APPLICATION_DEFAULT_CONFIG,
setup_metadata_agent_stub: false,
setup_k8s_stub: false,
log_entry: k8s_container_log_entry(log_entry(0)),
- expected_params: COMPUTE_PARAMS
+ expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK
},
{
config: APPLICATION_DEFAULT_CONFIG,
setup_metadata_agent_stub: true,
setup_k8s_stub: false,
log_entry: k8s_container_log_entry(log_entry(0)),
- expected_params: COMPUTE_PARAMS
+ expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK
},
{
config: APPLICATION_DEFAULT_CONFIG,
setup_metadata_agent_stub: true,
setup_k8s_stub: true,
@@ -1377,11 +1377,11 @@
{
config: ENABLE_METADATA_AGENT_CONFIG,
setup_metadata_agent_stub: false,
setup_k8s_stub: false,
log_entry: k8s_container_log_entry(log_entry(0)),
- expected_params: COMPUTE_PARAMS
+ expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK
},
{
config: ENABLE_METADATA_AGENT_CONFIG,
setup_metadata_agent_stub: false,
setup_k8s_stub: true,
@@ -1413,32 +1413,73 @@
config: ENABLE_METADATA_AGENT_CONFIG,
setup_metadata_agent_stub: true,
setup_k8s_stub: true,
log_entry: k8s_container_log_entry(log_entry(0)),
expected_params: K8S_CONTAINER_PARAMS
- },
+ }
+ ].each do |test_params|
+ new_stub_context do
+ setup_gce_metadata_stubs
+ setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
+ setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
+ setup_logging_stubs do
+ d = create_driver(test_params[:config], CONTAINER_TAG)
+ d.emit(test_params[:log_entry])
+ d.run
+ end
+ verify_log_entries(1, test_params[:expected_params],
+ 'jsonPayload') do |entry|
+ fields = get_fields(entry['jsonPayload'])
+ assert_equal 2, fields.size, entry
+ assert_equal 'test log entry 0', get_string(fields['log']), entry
+ assert_equal K8S_STREAM, get_string(fields['stream']), entry
+ end
+ end
+ end
+ end
+
+ def test_k8s_container_monitored_resource_invalid_local_resource_id
+ [
# When local_resource_id is not present or does not match k8s regexes.
{
config: ENABLE_METADATA_AGENT_CONFIG,
setup_metadata_agent_stub: true,
setup_k8s_stub: true,
log_entry: k8s_container_log_entry(
log_entry(0)).reject { |k, _| k == LOCAL_RESOURCE_ID_KEY },
- expected_params: COMPUTE_PARAMS
+ expected_params: CONTAINER_FROM_TAG_PARAMS
},
{
config: ENABLE_METADATA_AGENT_CONFIG,
setup_metadata_agent_stub: true,
setup_k8s_stub: true,
log_entry: k8s_container_log_entry(
log_entry(0),
local_resource_id: RANDOM_LOCAL_RESOURCE_ID),
- # When 'kube-env' is present, "compute.googleapis.com/resource_name" is
- # not added.
- expected_params: COMPUTE_PARAMS
- },
- # Specific cases for k8s_node.
+ expected_params: CONTAINER_FROM_TAG_PARAMS
+ }
+ ].each do |test_params|
+ new_stub_context do
+ setup_gce_metadata_stubs
+ setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
+ setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
+ setup_logging_stubs do
+ d = create_driver(test_params[:config], CONTAINER_TAG)
+ d.emit(test_params[:log_entry])
+ d.run
+ end
+ verify_log_entries(1, test_params[:expected_params]) do |entry|
+ assert_equal 'test log entry 0', entry['textPayload'], entry
+ end
+ end
+ end
+ end
+
+ # Test k8s_node monitored resource including the fallback when Metadata Agent
+ # restarts.
+ def test_k8s_node_monitored_resource_fallback
+ [
{
config: APPLICATION_DEFAULT_CONFIG,
setup_metadata_agent_stub: true,
setup_k8s_stub: true,
log_entry: k8s_node_log_entry(log_entry(0)),
@@ -1473,20 +1514,12 @@
expected_params: K8S_NODE_PARAMS
}
].each do |test_params|
new_stub_context do
setup_gce_metadata_stubs
- if test_params[:setup_metadata_agent_stub]
- setup_metadata_agent_stubs
- else
- setup_no_metadata_agent_stubs
- end
- if test_params[:setup_k8s_stub]
- setup_k8s_metadata_stubs
- else
- setup_no_k8s_metadata_stubs
- end
+ setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
+ setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
setup_logging_stubs do
d = create_driver(test_params[:config])
d.emit(test_params[:log_entry])
d.run
end
@@ -1559,11 +1592,11 @@
# "gke_container.<namespace_id>.<pod_name>.<container_name>".
def test_gke_container_logs
[1, 2, 3, 5, 11, 50].each do |n|
new_stub_context do
setup_gce_metadata_stubs
- setup_container_metadata_stubs
+ setup_k8s_metadata_stubs
setup_metadata_agent_stubs
setup_logging_stubs do
d = create_driver(ENABLE_METADATA_AGENT_CONFIG)
n.times do |i|
d.emit(gke_container_log_entry(log_entry(i)))
@@ -1571,11 +1604,11 @@
d.run
end
verify_log_entries(n, CONTAINER_FROM_APPLICATION_PARAMS)
assert_requested_metadata_agent_stub(
"#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
- ".#{CONTAINER_POD_NAME}.#{CONTAINER_CONTAINER_NAME}")
+ ".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}")
end
end
end
private
@@ -1651,48 +1684,38 @@
MANAGED_VM_BACKEND_NAME)
stub_metadata_request('instance/attributes/gae_backend_version',
MANAGED_VM_BACKEND_VERSION)
end
- def setup_container_metadata_stubs
- stub_metadata_request(
- 'instance/attributes/',
- "attribute1\nkube-env\nlast_attribute")
- stub_metadata_request('instance/attributes/kube-env',
- "ENABLE_NODE_LOGGING: \"true\"\n"\
- 'INSTANCE_PREFIX: '\
- "gke-#{CONTAINER_CLUSTER_NAME}-740fdafa\n"\
- 'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
- end
-
- def setup_k8s_metadata_stubs
- stub_metadata_request(
- 'instance/attributes/',
- "attribute1\ncluster-name\ncluster-location\nlast_attribute")
- stub_metadata_request('instance/attributes/cluster-location', K8S_LOCATION2)
- stub_metadata_request('instance/attributes/cluster-name', K8S_CLUSTER_NAME)
- end
-
- def setup_no_k8s_metadata_stubs
- ['cluster-location', 'cluster-name'].each do |metadata_name|
- stub_request(:get, %r{.*instance/attributes/#{metadata_name}.*})
- .to_return(status: 404,
- body: 'The requested URL /computeMetadata/v1/instance/' \
- "attributes/#{metadata_name} was not found on this" \
- ' server.')
+ def setup_k8s_metadata_stubs(should_respond = true)
+ if should_respond
+ stub_metadata_request(
+ 'instance/attributes/',
+ "attribute1\ncluster-location\ncluster-name\nlast_attribute")
+ stub_metadata_request('instance/attributes/cluster-location',
+ K8S_LOCATION2)
+ stub_metadata_request('instance/attributes/cluster-name',
+ K8S_CLUSTER_NAME)
+ else
+ ['cluster-location', 'cluster-name'].each do |metadata_name|
+ stub_request(:get, %r{.*instance/attributes/#{metadata_name}.*})
+ .to_return(status: 404,
+ body: 'The requested URL /computeMetadata/v1/instance/' \
+ "attributes/#{metadata_name} was not found on this" \
+ ' server.')
+ end
end
end
def setup_cloudfunctions_metadata_stubs
stub_metadata_request(
'instance/attributes/',
- "attribute1\nkube-env\ngcf_region\nlast_attribute")
- stub_metadata_request('instance/attributes/kube-env',
- "ENABLE_NODE_LOGGING: \"true\"\n"\
- 'INSTANCE_PREFIX: '\
- "gke-#{CLOUDFUNCTIONS_CLUSTER_NAME}-740fdafa\n"\
- 'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
+ "attribute1\ncluster-location\ncluster-name\ngcf_region\nlast_attribute")
+ stub_metadata_request('instance/attributes/cluster-location',
+ K8S_LOCATION2)
+ stub_metadata_request('instance/attributes/cluster-name',
+ K8S_CLUSTER_NAME)
stub_metadata_request('instance/attributes/gcf_region',
CLOUDFUNCTIONS_REGION)
end
def setup_dataproc_metadata_stubs
@@ -1723,69 +1746,69 @@
@logs_sent = []
yield
WebMock.reset!
end
- def setup_metadata_agent_stubs
- MONITORED_RESOURCE_STUBS.each do |local_resource_id, resource|
- stub_request(:get, metadata_request_url(local_resource_id))
- .to_return(status: 200, body: resource)
+ def setup_metadata_agent_stubs(should_respond = true)
+ if should_respond
+ MONITORED_RESOURCE_STUBS.each do |local_resource_id, resource|
+ stub_request(:get, metadata_request_url(local_resource_id))
+ .to_return(status: 200, body: resource)
+ end
+ stub_request(:get, metadata_request_url(RANDOM_LOCAL_RESOURCE_ID))
+ .to_return(status: 404, body: '')
+ else
+ # Simulate an environment with no metadata agent endpoint present.
+ stub_request(:get,
+ %r{#{DEFAULT_METADATA_AGENT_URL}\/monitoredResource/.*})
+ .to_raise(Errno::EHOSTUNREACH)
end
- stub_request(:get, metadata_request_url(RANDOM_LOCAL_RESOURCE_ID))
- .to_return(status: 404, body: '')
end
- def setup_no_metadata_agent_stubs
- # Simulate an environment with no metadata agent endpoint present.
- stub_request(:get, %r{#{DEFAULT_METADATA_AGENT_URL}\/monitoredResource/.*})
- .to_raise(Errno::EHOSTUNREACH)
- end
-
def assert_requested_metadata_agent_stub(local_resource_id)
assert_requested :get, metadata_request_url(local_resource_id)
end
# GKE Container.
def container_tag_with_container_name(container_name)
- "kubernetes.#{CONTAINER_POD_NAME}_#{CONTAINER_NAMESPACE_NAME}_" \
- "#{container_name}"
+ "kubernetes.#{K8S_POD_NAME}_#{K8S_NAMESPACE_NAME}_#{container_name}"
end
def container_log_entry_with_metadata(
- log, container_name = CONTAINER_CONTAINER_NAME)
+ log, container_name = K8S_CONTAINER_NAME)
{
log: log,
- stream: CONTAINER_STREAM,
- time: CONTAINER_TIMESTAMP,
+ stream: K8S_STREAM,
+ time: K8S_TIMESTAMP,
kubernetes: {
namespace_id: CONTAINER_NAMESPACE_ID,
- namespace_name: CONTAINER_NAMESPACE_NAME,
+ namespace_name: K8S_NAMESPACE_NAME,
pod_id: CONTAINER_POD_ID,
- pod_name: CONTAINER_POD_NAME,
+ pod_name: K8S_POD_NAME,
container_name: container_name,
labels: {
CONTAINER_LABEL_KEY => CONTAINER_LABEL_VALUE
}
}
}
end
- def container_log_entry(log, stream = CONTAINER_STREAM)
+ def container_log_entry(log, stream = K8S_STREAM)
{
log: log,
stream: stream,
- time: CONTAINER_TIMESTAMP
+ time: K8S_TIMESTAMP
}
end
def gke_container_log_entry(log)
{
log: log,
LOCAL_RESOURCE_ID_KEY =>
"#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
- ".#{CONTAINER_POD_NAME}.#{CONTAINER_CONTAINER_NAME}"
+ ".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}"
}
end
# Docker Container.
@@ -1947,22 +1970,21 @@
assert_equal n, entry_count
end
def verify_container_logs(log_entry_factory, expected_params)
setup_gce_metadata_stubs
- setup_container_metadata_stubs
+ setup_k8s_metadata_stubs
[1, 2, 3, 5, 11, 50].each do |n|
@logs_sent = []
setup_logging_stubs do
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
n.times { |i| d.emit(log_entry_factory.call(log_entry(i))) }
d.run
end
verify_log_entries(n, expected_params) do |entry, i|
verify_default_log_entry_text(entry['textPayload'], i, entry)
- assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'],
- entry
- assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
+ assert_equal K8S_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+ assert_equal K8S_NANOS, entry['timestamp']['nanos'], entry
assert_equal CONTAINER_SEVERITY, entry['severity'], entry
end
end
end