test/test_out_s3.rb in fluent-plugin-s3-1.6.1 vs test/test_out_s3.rb in fluent-plugin-s3-1.7.0

- old
+ new

@@ -50,106 +50,151 @@ def check_apikeys end end.configure(conf) end - def test_configure - d = create_driver - assert_equal 'test_key_id', d.instance.aws_key_id - assert_equal 'test_sec_key', d.instance.aws_sec_key - assert_equal 'test_bucket', d.instance.s3_bucket - assert_equal 'log', d.instance.path - assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext - assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type - assert_equal false, d.instance.force_path_style - assert_equal nil, d.instance.compute_checksums - assert_equal nil, d.instance.signature_version - assert_equal true, d.instance.check_bucket - assert_equal true, d.instance.check_object - end + sub_test_case "configure" do + def test_configure + d = create_driver + assert_equal 'test_key_id', d.instance.aws_key_id + assert_equal 'test_sec_key', d.instance.aws_sec_key + assert_equal 'test_bucket', d.instance.s3_bucket + assert_equal 'log', d.instance.path + assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext + assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type + assert_equal false, d.instance.force_path_style + assert_equal nil, d.instance.compute_checksums + assert_equal nil, d.instance.signature_version + assert_equal true, d.instance.check_bucket + assert_equal true, d.instance.check_object + end - def test_s3_endpoint_with_valid_endpoint - d = create_driver(CONFIG + 's3_endpoint riak-cs.example.com') - assert_equal 'riak-cs.example.com', d.instance.s3_endpoint - end + def test_s3_endpoint_with_valid_endpoint + d = create_driver(CONFIG + 's3_endpoint riak-cs.example.com') + assert_equal 'riak-cs.example.com', d.instance.s3_endpoint + end - data('US West (Oregon)' => 's3-us-west-2.amazonaws.com', - 'EU (Frankfurt)' => 's3.eu-central-1.amazonaws.com', - 'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com') - def test_s3_endpoint_with_invalid_endpoint(endpoint) - assert_raise(Fluent::ConfigError, "s3_endpoint parameter is not supported, use s3_region instead. This parameter is for S3 compatible services") { - create_driver(CONFIG + "s3_endpoint #{endpoint}") - } - end + data('US West (Oregon)' => 's3-us-west-2.amazonaws.com', + 'EU (Frankfurt)' => 's3.eu-central-1.amazonaws.com', + 'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com') + def test_s3_endpoint_with_invalid_endpoint(endpoint) + assert_raise(Fluent::ConfigError, "s3_endpoint parameter is not supported, use s3_region instead. This parameter is for S3 compatible services") { + create_driver(CONFIG + "s3_endpoint #{endpoint}") + } + end - def test_configure_with_mime_type_json - conf = CONFIG.clone - conf << "\nstore_as json\n" - d = create_driver(conf) - assert_equal 'json', d.instance.instance_variable_get(:@compressor).ext - assert_equal 'application/json', d.instance.instance_variable_get(:@compressor).content_type - end + def test_configure_with_mime_type_json + conf = CONFIG.clone + conf << "\nstore_as json\n" + d = create_driver(conf) + assert_equal 'json', d.instance.instance_variable_get(:@compressor).ext + assert_equal 'application/json', d.instance.instance_variable_get(:@compressor).content_type + end - def test_configure_with_mime_type_text - conf = CONFIG.clone - conf << "\nstore_as text\n" - d = create_driver(conf) - assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext - assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type - end + def test_configure_with_mime_type_text + conf = CONFIG.clone + conf << "\nstore_as text\n" + d = create_driver(conf) + assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext + assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type + end - def test_configure_with_mime_type_lzo - conf = CONFIG.clone - conf << "\nstore_as lzo\n" - d = create_driver(conf) - assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext - assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type - rescue => e - # TODO: replace code with disable lzop command - assert(e.is_a?(Fluent::ConfigError)) - end + def test_configure_with_mime_type_lzo + conf = CONFIG.clone + conf << "\nstore_as lzo\n" + d = create_driver(conf) + assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext + assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type + rescue => e + # TODO: replace code with disable lzop command + assert(e.is_a?(Fluent::ConfigError)) + end - def test_configure_with_path_style - conf = CONFIG.clone - conf << "\nforce_path_style true\n" - d = create_driver(conf) - assert d.instance.force_path_style - end + def test_configure_with_path_style + conf = CONFIG.clone + conf << "\nforce_path_style true\n" + d = create_driver(conf) + assert d.instance.force_path_style + end - def test_configure_with_compute_checksums - conf = CONFIG.clone - conf << "\ncompute_checksums false\n" - d = create_driver(conf) - assert_equal false, d.instance.compute_checksums - end + def test_configure_with_compute_checksums + conf = CONFIG.clone + conf << "\ncompute_checksums false\n" + d = create_driver(conf) + assert_equal false, d.instance.compute_checksums + end - def test_configure_with_hex_random_length - conf = CONFIG.clone - assert_raise Fluent::ConfigError do - create_driver(conf + "\nhex_random_length 17\n") + def test_configure_with_hex_random_length + conf = CONFIG.clone + assert_raise Fluent::ConfigError do + create_driver(conf + "\nhex_random_length 17\n") + end + assert_nothing_raised do + create_driver(conf + "\nhex_random_length 16\n") + end end - assert_nothing_raised do - create_driver(conf + "\nhex_random_length 16\n") + + def test_configure_with_no_check_on_s3 + conf = CONFIG.clone + conf << "\ncheck_bucket false\ncheck_object false\n" + d = create_driver(conf) + assert_equal false, d.instance.check_bucket + assert_equal false, d.instance.check_object end - end - def test_configure_with_no_check_on_s3 - conf = CONFIG.clone - conf << "\ncheck_bucket false\ncheck_object false\n" - d = create_driver(conf) - assert_equal false, d.instance.check_bucket - assert_equal false, d.instance.check_object - end + def test_configure_with_grant + conf = CONFIG.clone + conf << "\grant_full_control id='0123456789'\ngrant_read id='1234567890'\ngrant_read_acp id='2345678901'\ngrant_write_acp id='3456789012'\n" + d = create_driver(conf) + assert_equal "id='0123456789'", d.instance.grant_full_control + assert_equal "id='1234567890'", d.instance.grant_read + assert_equal "id='2345678901'", d.instance.grant_read_acp + assert_equal "id='3456789012'", d.instance.grant_write_acp + end - def test_configure_with_grant - conf = CONFIG.clone - conf << "\grant_full_control id='0123456789'\ngrant_read id='1234567890'\ngrant_read_acp id='2345678901'\ngrant_write_acp id='3456789012'\n" - d = create_driver(conf) - assert_equal "id='0123456789'", d.instance.grant_full_control - assert_equal "id='1234567890'", d.instance.grant_read - assert_equal "id='2345678901'", d.instance.grant_read_acp - assert_equal "id='3456789012'", d.instance.grant_write_acp + CONFIG_WITH_OBJECTKEY_DEFAULT = %[ + s3_object_key_format "%{path}%{time_slice}_%{index}.%{file_extension}" + aws_key_id test_key_id + aws_sec_key test_sec_key + s3_bucket test_bucket + path log + utc + buffer_type memory + time_slice_format %Y%m%d-%H + ] + + CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD = %[ + s3_object_key_format "%{path}%{time_slice}_${chunk_id}.%{file_extension}" + aws_key_id test_key_id + aws_sec_key test_sec_key + s3_bucket test_bucket + path log + utc + buffer_type memory + time_slice_format %Y%m%d-%H + ] + + data("non_objectkey", {"expected_warning_num" => 1, "conf" => CONFIG, "workers" => 1, "with_multi_buffers" => false}) + data("non_objectkey-multi_buffer", {"expected_warning_num" => 2, "conf" => CONFIG, "workers" => 1, "with_multi_buffers" => true}) + data("non_objectkey-multi_worker", {"expected_warning_num" => 2, "conf" => CONFIG, "workers" => 2, "with_multi_buffers" => false}) + data("default_objectkey", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_DEFAULT, "workers" => 1, "with_multi_buffers" => false}) + data("default_objectkey-multi_buffer", {"expected_warning_num" => 1, "conf" => CONFIG_WITH_OBJECTKEY_DEFAULT, "workers" => 1, "with_multi_buffers" => true}) + data("default_objectkey-multi_worker", {"expected_warning_num" => 1, "conf" => CONFIG_WITH_OBJECTKEY_DEFAULT, "workers" => 2, "with_multi_buffers" => false}) + data("fixed_objectkey", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD, "workers" => 1, "with_multi_buffers" => false}) + data("fixed_objectkey-multi_buffer", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD, "workers" => 1, "with_multi_buffers" => true}) + data("fixed_objectkey-multi_worker", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD, "workers" => 2, "with_multi_buffers" => false}) + def test_configure_warning_on_parallel(data) + conf = data["conf"].clone + if data["with_multi_buffers"] + conf << "\n<buffer>\n@type memory\nflush_thread_count 2\n</buffer>\n" + end + assert_rr do + d = Fluent::Test::Driver::Output.new(Fluent::Plugin::S3Output, opts: {"workers": data["workers"]}) + mock(d.instance.log).warn(anything).times(data["expected_warning_num"]) + d.configure(conf) + end + end end def test_format d = create_driver @@ -515,13 +560,13 @@ assert_instance_of(Aws::Credentials, credentials) end def test_assume_role_credentials expected_credentials = Aws::Credentials.new("test_key", "test_secret") - mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn", - role_session_name: "test_session", - client: anything){ + mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn", + role_session_name: "test_session", + client: anything }){ expected_credentials } config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n") config += %[ <assume_role_credentials> @@ -538,13 +583,13 @@ def test_assume_role_credentials_with_region expected_credentials = Aws::Credentials.new("test_key", "test_secret") sts_client = Aws::STS::Client.new(region: 'ap-northeast-1') mock(Aws::STS::Client).new(region: 'ap-northeast-1'){ sts_client } - mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn", - role_session_name: "test_session", - client: sts_client){ + mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn", + role_session_name: "test_session", + client: sts_client }){ expected_credentials } config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n") config += %[ s3_region ap-northeast-1 @@ -563,13 +608,13 @@ def test_assume_role_with_iam_credentials expected_credentials = Aws::Credentials.new("test_key_id", "test_sec_key") sts_client = Aws::STS::Client.new(region: 'ap-northeast-1', credentials: expected_credentials) mock(Aws::Credentials).new("test_key_id", "test_sec_key") { expected_credentials } mock(Aws::STS::Client).new(region: 'ap-northeast-1', credentials: expected_credentials){ sts_client } - mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn", - role_session_name: "test_session", - client: sts_client){ + mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn", + role_session_name: "test_session", + client: sts_client } ){ expected_credentials } config = CONFIG_TIME_SLICE config += %[ s3_region ap-northeast-1 @@ -590,14 +635,14 @@ expected_credentials = Aws::Credentials.new("test_key", "test_secret") expected_region = "ap-northeast-1" expected_sts_http_proxy = 'http://example.com' sts_client = Aws::STS::Client.new(region: expected_region, http_proxy: expected_sts_http_proxy) mock(Aws::STS::Client).new(region:expected_region, http_proxy: expected_sts_http_proxy){ sts_client } - mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn", - role_session_name: "test_session", - client: sts_client, - sts_http_proxy: expected_sts_http_proxy){ + mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn", + role_session_name: "test_session", + client: sts_client, + sts_http_proxy: expected_sts_http_proxy }){ expected_credentials } config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n") config += %[ s3_region #{expected_region} @@ -617,14 +662,14 @@ def test_assume_role_credentials_with_sts_http_proxy expected_credentials = Aws::Credentials.new("test_key", "test_secret") expected_sts_http_proxy = 'http://example.com' sts_client = Aws::STS::Client.new(region: "us-east-1", http_proxy: expected_sts_http_proxy) mock(Aws::STS::Client).new(region: "us-east-1", http_proxy: expected_sts_http_proxy){ sts_client } - mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn", - role_session_name: "test_session", - client: sts_client, - sts_http_proxy: expected_sts_http_proxy){ + mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn", + role_session_name: "test_session", + client: sts_client, + sts_http_proxy: expected_sts_http_proxy }){ expected_credentials } config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n") config += %[ <assume_role_credentials> @@ -643,14 +688,14 @@ def test_assume_role_credentials_with_sts_endpoint_url expected_credentials = Aws::Credentials.new("test_key", "test_secret") expected_sts_endpoint_url = 'http://example.com' sts_client = Aws::STS::Client.new(region: "us-east-1", endpoint: expected_sts_endpoint_url) mock(Aws::STS::Client).new(region: "us-east-1", endpoint: expected_sts_endpoint_url){ sts_client } - mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn", - role_session_name: "test_session", - client: sts_client, - sts_endpoint_url: expected_sts_endpoint_url){ + mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn", + role_session_name: "test_session", + client: sts_client, + sts_endpoint_url: expected_sts_endpoint_url }){ expected_credentials } config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n") config += %[ <assume_role_credentials> @@ -669,13 +714,13 @@ def test_assume_role_credentials_with_sts_region expected_credentials = Aws::Credentials.new("test_key", "test_secret") expected_sts_region = 'ap-south-1' sts_client = Aws::STS::Client.new(region: expected_sts_region) mock(Aws::STS::Client).new(region: expected_sts_region){ sts_client } - mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn", - role_session_name: "test_session", - client: sts_client){ + mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn", + role_session_name: "test_session", + client: sts_client }){ expected_credentials } config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n") config += %[ <assume_role_credentials> @@ -692,14 +737,16 @@ end def test_web_identity_credentials expected_credentials = Aws::Credentials.new("test_key", "test_secret") mock(Aws::AssumeRoleWebIdentityCredentials).new( - role_arn: "test_arn", - role_session_name: "test_session", - web_identity_token_file: "test_file", - client: anything + { + role_arn: "test_arn", + role_session_name: "test_session", + web_identity_token_file: "test_file", + client: anything + } ){ expected_credentials } config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n") @@ -720,13 +767,15 @@ def test_web_identity_credentials_with_sts_region expected_credentials = Aws::Credentials.new("test_key", "test_secret") sts_client = Aws::STS::Client.new(region: 'us-east-1') mock(Aws::STS::Client).new(region: 'us-east-1'){ sts_client } mock(Aws::AssumeRoleWebIdentityCredentials).new( - role_arn: "test_arn", - role_session_name: "test_session", - web_identity_token_file: "test_file", - client: sts_client + { + role_arn: "test_arn", + role_session_name: "test_session", + web_identity_token_file: "test_file", + client: sts_client + } ){ expected_credentials } config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")