test/plugin/base_test.rb in fluent-plugin-google-cloud-0.6.5.pre.1 vs test/plugin/base_test.rb in fluent-plugin-google-cloud-0.6.5

- old
+ new

@@ -514,10 +514,15 @@ emit_index += 1 d.emit('message' => log_entry(emit_index), 'timestampSeconds' => ts.tv_sec, 'timestampNanos' => ts.tv_nsec) expected_ts.push(ts) emit_index += 1 + d.emit('message' => log_entry(emit_index), + 'timestampSeconds' => "#{ts.tv_sec}", + 'timestampNanos' => "#{ts.tv_nsec}") + expected_ts.push(ts) + emit_index += 1 d.run end end verify_index = 0 verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry| @@ -777,138 +782,10 @@ assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry assert_equal 'WARNING', entry['severity'], entry end end - # Docker Container. - - # Test textPayload logs from Docker container stderr / stdout. - def test_text_payload_docker_container_logs - setup_gce_metadata_stubs - [1, 2, 3, 5, 11, 50].each do |n| - @logs_sent = [] - setup_logging_stubs do - d = create_driver(APPLICATION_DEFAULT_CONFIG, DOCKER_CONTAINER_TAG) - # Generate log entries with 'log' (simply a string) and 'time' fields. - n.times { |i| d.emit(docker_container_log_entry(log_entry(i))) } - d.run - end - # 'container_id' and 'container_name' should have been extracted from tag - # and properly set in resource.labels and common metadata labels as in - # DOCKER_CONTAINER_PARAMS. - verify_log_entries(n, DOCKER_CONTAINER_PARAMS) do |entry| - # Timestamp in 'time' field from log entry should be set properly. - assert_equal DOCKER_CONTAINER_SECONDS_EPOCH, - entry['timestamp']['seconds'], entry - assert_equal DOCKER_CONTAINER_NANOS, entry['timestamp']['nanos'], - entry - # Severity is 'DEFAULT' because 'stream' info is absent from log entry. - assert_equal_with_default entry['severity'], 'DEFAULT', 'DEFAULT', entry - end - end - end - - # Test jsonPayload logs from Docker container stderr / stdout. - def test_json_payload_docker_container_logs - setup_gce_metadata_stubs - [1, 2, 3, 5, 11, 50].each do |n| - @logs_sent = [] - setup_logging_stubs do - d = create_driver(APPLICATION_DEFAULT_CONFIG, DOCKER_CONTAINER_TAG) - n.times do - # Generate log entries with 'log' (json) and 'time' fields. - d.emit(docker_container_log_entry('{"msg": "test log entry ' \ - "#{n}" \ - '", "tag2": "test", "data": ' \ - '5000, "severity": "WARNING"}')) - end - d.run - end - # 'container_id' and 'container_name' should have been extracted from tag - # and properly set in resource.labels and common metadata labels as in - # DOCKER_CONTAINER_PARAMS. - verify_log_entries(n, DOCKER_CONTAINER_PARAMS, 'jsonPayload') do |entry| - # 'log' field should be detected as json and parsed properly. - fields = get_fields(entry['jsonPayload']) - assert_equal 3, fields.size, entry - assert_equal "test log entry #{n}", get_string(fields['msg']), entry - assert_equal 'test', get_string(fields['tag2']), entry - assert_equal 5000, get_number(fields['data']), entry - # Timestamp in 'time' field from log entry should be set properly. - assert_equal DOCKER_CONTAINER_SECONDS_EPOCH, - entry['timestamp']['seconds'], entry - assert_equal DOCKER_CONTAINER_NANOS, entry['timestamp']['nanos'], entry - # Severity in 'severity' field from log entry should be set properly. - assert_equal 'WARNING', entry['severity'], entry - end - end - end - - # Test the following metadata is properly extracted from json record: - # * container_id - # * container_name - # * source (this field maps to the 'stream' metadata label. - def test_docker_container_logs_metadata_from_json_record - setup_gce_metadata_stubs - { - # 'stream' label should be extracted from json record when present. - docker_container_log_entry_with_metadata( - log_entry(0) - ) => { - # When the 'source' field from the json record has value 'stdout', - # 'severity' should be 'INFO'. - 'params' => DOCKER_CONTAINER_PARAMS_WITH_STREAM_STDOUT, - 'severity' => 'INFO' - }, - # 'container_id' and 'container_name' can be extracted from tag or - # json record. If present in both cases, values in the json record will - # overwrite values in tags. For example: - # DOCKER_CONTAINER_PARAMS_WITH_METADATA_OVERWRITTEN has different - # 'container_id' and 'container_name' values from DOCKER_CONTAINER_TAG, - # and we need to verify these two fields get overwritten as expected. - docker_container_log_entry_with_metadata( - log_entry(0), DOCKER_CONTAINER_ID2, DOCKER_CONTAINER_NAME2, - DOCKER_CONTAINER_STREAM_STDERR - ) => { - # When the 'source' field from the json record has value 'stderr', - # 'severity' should be 'ERROR'. - 'params' => DOCKER_CONTAINER_PARAMS_WITH_METADATA_OVERWRITTEN, - 'severity' => 'ERROR' - } - }.each do |log_entry, expected| - @logs_sent = [] - setup_logging_stubs do - d = create_driver(APPLICATION_DEFAULT_CONFIG, DOCKER_CONTAINER_TAG) - d.emit(log_entry) - d.run - end - verify_log_entries(1, expected['params']) do |entry| - assert_equal DOCKER_CONTAINER_SECONDS_EPOCH, - entry['timestamp']['seconds'], entry - assert_equal DOCKER_CONTAINER_NANOS, entry['timestamp']['nanos'], - entry - assert_equal_with_default entry['severity'], expected['severity'], - 'DEFAULT', entry - end - end - end - - # Test logs from applications running in Docker containers. - def test_docker_container_application_logs - setup_gce_metadata_stubs - setup_docker_remote_api_stubs - setup_logging_stubs do - # Metadata Agent is not enabled. Will call Docker Remote API for - # container info. - d = create_driver(APPLICATION_DEFAULT_CONFIG, - DOCKER_CONTAINER_TAG_WITH_APPLICATION) - d.emit('message' => log_entry(0)) - d.run - end - verify_log_entries(1, DOCKER_CONTAINER_WITH_APPLICATION_PARAMS) - end - def test_cloudfunctions_log setup_gce_metadata_stubs setup_cloudfunctions_metadata_stubs [1, 2, 3, 5, 11, 50].each do |n| setup_logging_stubs do @@ -980,53 +857,46 @@ d.run end verify_log_entries(1, DATAPROC_PARAMS, 'jsonPayload') end - def test_http_request_from_record - setup_gce_metadata_stubs - setup_logging_stubs do - d = create_driver - d.emit('httpRequest' => HTTP_REQUEST_MESSAGE) - d.run - end - verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry| - assert_equal HTTP_REQUEST_MESSAGE, entry['httpRequest'], entry - assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry - end + def test_log_entry_http_request_field_from_record + verify_subfields_from_record(DEFAULT_HTTP_REQUEST_KEY) end - def test_http_request_partial_from_record - setup_gce_metadata_stubs - setup_logging_stubs do - d = create_driver - d.emit('httpRequest' => HTTP_REQUEST_MESSAGE.merge( - 'otherKey' => 'value')) - d.run - end - verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry| - assert_equal HTTP_REQUEST_MESSAGE, entry['httpRequest'], entry - fields = get_fields(entry['jsonPayload']) - request = get_fields(get_struct(fields['httpRequest'])) - assert_equal 'value', get_string(request['otherKey']), entry - end + def test_log_entry_source_location_field_from_record + verify_subfields_from_record(DEFAULT_SOURCE_LOCATION_KEY) end - def test_http_request_when_not_hash - setup_gce_metadata_stubs - setup_logging_stubs do - d = create_driver - d.emit('httpRequest' => 'a_string') - d.run - end - verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry| - fields = get_fields(entry['jsonPayload']) - assert_equal 'a_string', get_string(fields['httpRequest']), entry - assert_nil entry['httpRequest'], entry - end + def test_log_entry_operation_field_from_record + verify_subfields_from_record(DEFAULT_OPERATION_KEY) end + def test_log_entry_http_request_field_partial_from_record + verify_subfields_partial_from_record(DEFAULT_HTTP_REQUEST_KEY) + end + + def test_log_entry_source_location_field_partial_from_record + verify_subfields_partial_from_record(DEFAULT_SOURCE_LOCATION_KEY) + end + + def test_log_entry_operation_field_partial_from_record + verify_subfields_partial_from_record(DEFAULT_OPERATION_KEY) + end + + def test_log_entry_http_request_field_when_not_hash + verify_subfields_when_not_hash(DEFAULT_HTTP_REQUEST_KEY) + end + + def test_log_entry_source_location_field_when_not_hash + verify_subfields_when_not_hash(DEFAULT_SOURCE_LOCATION_KEY) + end + + def test_log_entry_operation_field_when_not_hash + verify_subfields_when_not_hash(DEFAULT_OPERATION_KEY) + end + def test_http_request_from_record_with_referer_nil_or_absent setup_gce_metadata_stubs [ http_request_message_with_nil_referer, http_request_message_with_absent_referer @@ -1080,37 +950,82 @@ assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry end end end - def test_trace_field_assignment + def test_labels_from_record setup_gce_metadata_stubs + setup_logging_stubs do + d = create_driver + d.emit(DEFAULT_LABELS_KEY => CUSTOM_LABELS_MESSAGE) + d.run + end + labels = COMPUTE_PARAMS[:labels].merge(CUSTOM_LABELS_MESSAGE) + params = COMPUTE_PARAMS.merge(labels: labels) + verify_log_entries(1, params, 'labels') do |entry| + assert_nil get_fields(entry['jsonPayload'])[DEFAULT_LABELS_KEY], entry + end + end + + def test_labels_from_record_conflict + setup_gce_metadata_stubs + setup_logging_stubs do + d = create_driver + d.emit(DEFAULT_LABELS_KEY => { CONFLICTING_LABEL_KEY => 'a_string' }) + d.run + end + verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry| + fields = get_fields(entry['jsonPayload']) + labels = get_fields(get_struct(fields[DEFAULT_LABELS_KEY])) + assert_equal('a_string', get_string(labels[CONFLICTING_LABEL_KEY]), entry) + end + end + + def test_labels_from_record_when_not_hash + setup_gce_metadata_stubs + setup_logging_stubs do + d = create_driver + d.emit(DEFAULT_LABELS_KEY => 'a_string') + d.run + end + verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry| + fields = get_fields(entry['jsonPayload']) + assert_equal 'a_string', get_string(fields[DEFAULT_LABELS_KEY]), entry + end + end + + def test_log_entry_trace_field + setup_gce_metadata_stubs message = log_entry(0) trace = 'projects/project-1/traces/1234567890abcdef1234567890abcdef' [ { # It leaves trace entry field nil if no trace value sent. driver_config: APPLICATION_DEFAULT_CONFIG, emitted_log: { 'msg' => message }, + expected_fields: { 'msg' => message }, expected_trace_value: nil }, { # By default, it sets trace via Google-specific key. driver_config: APPLICATION_DEFAULT_CONFIG, emitted_log: { 'msg' => message, DEFAULT_TRACE_KEY => trace }, + expected_fields: { 'msg' => message }, expected_trace_value: trace }, { # It allows setting the trace via a custom configured key. driver_config: CONFIG_CUSTOM_TRACE_KEY_SPECIFIED, emitted_log: { 'msg' => message, 'custom_trace_key' => trace }, + expected_fields: { 'msg' => message }, expected_trace_value: trace }, { # It no longer sets trace by the default key if custom key specified. driver_config: CONFIG_CUSTOM_TRACE_KEY_SPECIFIED, emitted_log: { 'msg' => message, DEFAULT_TRACE_KEY => trace }, + expected_fields: { 'msg' => message, DEFAULT_TRACE_KEY => trace }, expected_trace_value: nil } ].each do |input| setup_logging_stubs do @logs_sent = [] @@ -1118,133 +1033,20 @@ d.emit(input[:emitted_log]) d.run end verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry| assert_equal input[:expected_trace_value], entry['trace'], input - end - end - end - def test_trace_removal_from_json_payload - setup_gce_metadata_stubs - message = log_entry(0) - trace = 'projects/project-1/traces/1234567890abcdef1234567890abcdef' - [ - { - # By default, it removes trace value from jsonPayload. - driver_config: APPLICATION_DEFAULT_CONFIG, - emitted_log: { 'msg' => message, DEFAULT_TRACE_KEY => trace }, - expected_json_payload: { 'msg' => message } - }, - { - # It keeps the trace value in jsonPayload if keep_trace_key set to true. - driver_config: CONFIG_KEEP_TRACE_KEY_TRUE, - emitted_log: { 'msg' => message, DEFAULT_TRACE_KEY => trace }, - expected_json_payload: { 'msg' => message, DEFAULT_TRACE_KEY => trace } - } - ].each do |input| - setup_logging_stubs do - @logs_sent = [] - d = create_driver(input[:driver_config]) - d.emit(input[:emitted_log]) - d.run - end - verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry| fields = get_fields(entry['jsonPayload']) - assert_equal input[:expected_json_payload].size, fields.size, input + assert_equal input[:expected_fields].size, fields.size, input fields.each do |key, value| - assert_equal(input[:expected_json_payload][key], - get_string(value), input) + assert_equal input[:expected_fields][key], get_string(value), input end end end end - # Metadata Agent related tests. - def test_configure_enable_metadata_agent_default_and_false - setup_gce_metadata_stubs - [create_driver, create_driver(DISABLE_METADATA_AGENT_CONFIG)].each do |d| - assert_false d.instance.instance_variable_get(:@enable_metadata_agent) - end - end - - def test_configure_enable_metadata_agent_true - setup_gce_metadata_stubs - setup_metadata_agent_stubs(IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY) - d = create_driver(ENABLE_METADATA_AGENT_CONFIG) - assert_true d.instance.instance_variable_get(:@enable_metadata_agent) - end - - # Test an implicit monitored resource can be retrieved from Metadata Agent - # with an empty string as the locally-unique id. - def test_retrieve_implicit_monitored_resource - # Minimum set up for gce metadata stubs. Stubs for 'vm_id' and 'location' - # info are deliberately not set up. - setup_gce_metadata_stubs_minimum - setup_metadata_agent_stubs(IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY) - setup_logging_stubs do - d = create_driver(ENABLE_METADATA_AGENT_CONFIG) - d.emit('message' => log_entry(0)) - d.run - end - verify_log_entries(1, COMPUTE_PARAMS) - end - - # Test a docker container monitored resource can be retrieved from Metadata - # Agent with "container.<container_id>" as the locally-unique id. - def test_retrieve_docker_container_monitored_resource - setup_gce_metadata_stubs_minimum - setup_metadata_agent_stubs(IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY) - setup_metadata_agent_stubs(DOCKER_CONSTANTS[:resource_type]) - setup_logging_stubs do - # Tag format here: "container.<container_id>.<container_name>" - d = create_driver(ENABLE_METADATA_AGENT_CONFIG, DOCKER_CONTAINER_TAG) - d.emit('message' => log_entry(0)) - d.run - end - # gce_metadata_stubs has ZONE1, while metadata agent stub has ZONE2. Here we - # need to verify ZONE2 overwrites ZONE1 as expected. - verify_log_entries(1, DOCKER_CONTAINER_PARAMS_WITH_ZONE2) - end - - # Test a docker container monitored resource can be retrieved from Metadata - # Agent with "container.<container_name>" as the locally-unique id when the - # log entry comes from an application running in the docker container. - def test_retrieve_docker_container_monitored_resource_with_application - setup_gce_metadata_stubs_minimum - setup_metadata_agent_stubs(IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY) - setup_metadata_agent_stubs( - "#{DOCKER_CONSTANTS[:resource_type]}_application") - setup_logging_stubs do - # Tag format here: "application-container.<container_name>. - # <additional_tag>" - d = create_driver(ENABLE_METADATA_AGENT_CONFIG, - DOCKER_CONTAINER_TAG_WITH_APPLICATION) - d.emit('message' => log_entry(0)) - d.run - end - verify_log_entries(1, DOCKER_CONTAINER_WITH_APPLICATION_PARAMS) - end - - # Test a GKE container monitored resource can be retrieved from Metadata - # Agent with "gke_containerName.<namespace_id>.<pod_name>.<container_name>" as - # the locally-unique id when the log entry comes from an application running - # in Kubenetes. - def test_retrieve_gke_application_container_monitored_resource - setup_gce_metadata_stubs_minimum - setup_container_metadata_stubs - setup_metadata_agent_stubs(IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY) - setup_metadata_agent_stubs( - "#{CONTAINER_CONSTANTS[:resource_type]}_application") - setup_logging_stubs do - d = create_driver(ENABLE_METADATA_AGENT_CONFIG) - d.emit(gke_application_container_log_entry(log_entry(0))) - d.run - end - verify_log_entries(1, CONTAINER_FROM_APPLICATION_PARAMS) - end - private def stub_metadata_request(metadata_path, response_body) stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' + metadata_path) @@ -1277,30 +1079,10 @@ status: 200, headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length, 'Content-Type' => 'application/json' }) end - # The minimum stubs needed for infomation that Metadata Agent can not provide. - def setup_gce_metadata_stubs_minimum - # Stub the root, used for platform detection by the plugin and 'googleauth'. - stub_request(:get, 'http://169.254.169.254') - .to_return(status: 200, headers: { 'Metadata-Flavor' => 'Google' }) - - # Create stubs for all the GCE metadata lookups the agent needs to make. - stub_metadata_request('project/project-id', PROJECT_ID) - stub_metadata_request('instance/attributes/', - "attribute1\nattribute2\nattribute3") - - # Used by 'googleauth' to fetch the default service account credentials. - stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' \ - 'instance/service-accounts/default/token') - .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}), - status: 200, - headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length, - 'Content-Type' => 'application/json' }) - end - def setup_ec2_metadata_stubs # Stub the root, used for platform detection. stub_request(:get, 'http://169.254.169.254') .to_return(status: 200, headers: { 'Server' => 'EC2ws' }) @@ -1347,15 +1129,10 @@ 'INSTANCE_PREFIX: '\ "gke-#{CONTAINER_CLUSTER_NAME}-740fdafa\n"\ 'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S') end - def setup_docker_remote_api_stubs - stub_request(:get, "http://unix/containers/#{DOCKER_CONTAINER_NAME}/json") - .to_return(status: 200, body: "{\"Id\":\"#{DOCKER_CONTAINER_ID}\"}") - end - def setup_cloudfunctions_metadata_stubs stub_metadata_request( 'instance/attributes/', "attribute1\nkube-env\ngcf_region\nlast_attribute") stub_metadata_request('instance/attributes/kube-env', @@ -1381,24 +1158,15 @@ def setup_prometheus Prometheus::Client.registry.instance_variable_set('@metrics', {}) end - def setup_metadata_agent_stubs(resource_type) - unique_id = MONITORED_RESOURCE_STUBS[resource_type]['unique_id'] - resource = MONITORED_RESOURCE_STUBS[resource_type]['monitored_resource'] - stub_request(:get, 'http://local-metadata-agent.stackdriver.com:8000' \ - "/monitoredResource/#{unique_id}") - .to_return(status: 200, body: resource) - end - def container_tag_with_container_name(container_name) "kubernetes.#{CONTAINER_POD_NAME}_#{CONTAINER_NAMESPACE_NAME}_" \ "#{container_name}" end - # GKE Container def container_log_entry_with_metadata( log, container_name = CONTAINER_CONTAINER_NAME) { log: log, stream: CONTAINER_STREAM, @@ -1422,41 +1190,10 @@ stream: stream, time: CONTAINER_TIMESTAMP } end - def gke_application_container_log_entry(log) - { - log: log, - LOCALLY_UNIQUE_ID_LABEL_NAME => 'gke_containerName' \ - ".#{CONTAINER_NAMESPACE_ID}" \ - ".#{CONTAINER_POD_NAME}" \ - ".#{CONTAINER_CONTAINER_NAME}" - } - end - - # Docker Container - def docker_container_log_entry_with_metadata( - log, container_id = DOCKER_CONTAINER_ID, - container_name = DOCKER_CONTAINER_NAME, - stream = DOCKER_CONTAINER_STREAM_STDOUT) - { - log: log, - container_id: container_id, - container_name: container_name, - time: DOCKER_CONTAINER_TIMESTAMP, - source: stream - } - end - - def docker_container_log_entry(log) - { - log: log, - time: CONTAINER_TIMESTAMP - } - end - def cloudfunctions_log_entry(i) { stream: 'stdout', log: '[D][2015-09-25T12:34:56.789Z][123-0] ' + log_entry(i) } @@ -1543,25 +1280,74 @@ end end assert i == n, "Number of entries #{i} does not match expected number #{n}" end - def verify_container_logs(log_entry_method, expected_params) + def verify_container_logs(log_entry_factory, expected_params) setup_gce_metadata_stubs setup_container_metadata_stubs [1, 2, 3, 5, 11, 50].each do |n| @logs_sent = [] setup_logging_stubs do d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG) - n.times { |i| d.emit(log_entry_method.call(log_entry(i))) } + n.times { |i| d.emit(log_entry_factory.call(log_entry(i))) } d.run end verify_log_entries(n, expected_params) do |entry| assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry assert_equal CONTAINER_SEVERITY, entry['severity'], entry end + end + end + + def verify_subfields_from_record(payload_key) + destination_key, payload_value = LOG_ENTRY_SUBFIELDS_PARAMS[payload_key] + @logs_sent = [] + setup_gce_metadata_stubs + setup_logging_stubs do + d = create_driver + d.emit(payload_key => payload_value) + d.run + end + verify_log_entries(1, COMPUTE_PARAMS, destination_key) do |entry| + assert_equal payload_value, entry[destination_key], entry + fields = get_fields(entry['jsonPayload']) + assert_nil fields[payload_key], entry + end + end + + def verify_subfields_partial_from_record(payload_key) + destination_key, payload_value = LOG_ENTRY_SUBFIELDS_PARAMS[payload_key] + @logs_sent = [] + setup_gce_metadata_stubs + setup_logging_stubs do + d = create_driver + d.emit(payload_key => payload_value.merge('otherKey' => 'value')) + d.run + end + verify_log_entries(1, COMPUTE_PARAMS, destination_key) do |entry| + assert_equal payload_value, entry[destination_key], entry + fields = get_fields(entry['jsonPayload']) + request = get_fields(get_struct(fields[payload_key])) + assert_equal 'value', get_string(request['otherKey']), entry + end + end + + def verify_subfields_when_not_hash(payload_key) + destination_key = LOG_ENTRY_SUBFIELDS_PARAMS[payload_key][0] + @logs_sent = [] + setup_gce_metadata_stubs + setup_logging_stubs do + d = create_driver + d.emit(payload_key => 'a_string') + d.run + end + verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry| + field = get_fields(entry['jsonPayload'])[payload_key] + assert_equal 'a_string', get_string(field), entry + assert_nil entry[destination_key], entry end end # Replace the 'referer' field with nil. def http_request_message_with_nil_referer