Contents

#
# Copyright 2023- Mykola Panin
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

require 'fluent/plugin/filter'

module Fluent
  module Plugin
    class JsonSizeLimitFilter < Filter
      Fluent::Plugin.register_filter('jsonsizelimit', self)

      config_param :max_size, :size, default: 250 * 1024

      def configure(conf)
        super
        log.debug "Configuring JsonSizeLimitFilter with max_size: #{@max_size}"
      end

      def filter(tag, time, record)
        original_size = record.to_json.bytesize
        json_str = record.to_json

        if json_str.bytesize > @max_size
          sorted_fields = record.sort_by { |k, v| v.to_s.length }.reverse

          sorted_fields.each do |key, value|
            next unless value.is_a?(String) && value.length > 10

            current_size = json_str.bytesize
            excess_size = current_size - @max_size

            new_length = [value.length - excess_size, 0].max
            record[key] = value[0...new_length]
            json_str = record.to_json

            break if json_str.bytesize <= @max_size
          end

          final_size = json_str.bytesize
          log.debug "Reduced record size from #{original_size} to #{final_size} bytes"
        end

        record
      end
    end
  end
end

Version data entries

1 entries across 1 versions & 1 rubygems

Version	Path
fluent-plugin-fluent-plugin-json-size-limit-0.1.15	lib/fluent/plugin/filter_jsonsizelimit.rb