chain.rb in dynamoid-3.2.0

- old
+ new

@@ -1,14 +1,19 @@
 # frozen_string_literal: true
 
-module Dynamoid #:nodoc:
+require_relative 'key_fields_detector'
+require_relative 'ignored_conditions_detector'
+require_relative 'overwritten_conditions_detector'
+require_relative 'nonexistent_fields_detector'
+
+module Dynamoid
   module Criteria
     # The criteria chain is equivalent to an ActiveRecord relation (and realistically I should change the name from
     # chain to relation). It is a chainable object that builds up a query and eventually executes it by a Query or Scan.
     class Chain
-      attr_accessor :query, :source, :values, :consistent_read
-      attr_reader :hash_key, :range_key, :index_name
+      attr_reader :query, :source, :consistent_read, :key_fields_detector
+
       include Enumerable
       # Create a new criteria chain.
       #
       # @param [Class] source the class upon which the ultimate query will be performed.
       def initialize(source)
@@ -20,10 +25,13 @@
         # Honor STI and :type field if it presents
         type = @source.inheritance_field
         if @source.attributes.key?(type)
           @query[:"#{type}.in"] = @source.deep_subclasses.map(&:name) << @source.name
         end
+
+        # we should re-initialize keys detector every time we change query
+        @key_fields_detector = KeyFieldsDetector.new(@query, @source)
       end
 
       # The workhorse method of the criteria chain. Each key in the passed in hash will become another criteria that the
       # ultimate query must match. A key can either be a symbol or a string, and should be an attribute name or
       # an attribute name with a range operator.
@@ -34,11 +42,30 @@
       # @example A more complicated criteria
       #   where(:name => 'Josh', 'created_at.gt' => DateTime.now - 1.day)
       #
       # @since 0.2.0
       def where(args)
-        query.update(args.dup.symbolize_keys)
+        detector = IgnoredConditionsDetector.new(args)
+        if detector.found?
+          Dynamoid.logger.warn(detector.warning_message)
+        end
+
+        detector = OverwrittenConditionsDetector.new(@query, args)
+        if detector.found?
+          Dynamoid.logger.warn(detector.warning_message)
+        end
+
+        detector = NonexistentFieldsDetector.new(args, @source)
+        if detector.found?
+          Dynamoid.logger.warn(detector.warning_message)
+        end
+
+        query.update(args.symbolize_keys)
+
+        # we should re-initialize keys detector every time we change query
+        @key_fields_detector = KeyFieldsDetector.new(@query, @source)
+
         self
       end
 
       def consistent
         @consistent_read = true
@@ -51,11 +78,11 @@
       def all
         records
       end
 
       def count
-        if key_present?
+        if @key_fields_detector.key_present?
           count_via_query
         else
           count_via_scan
         end
       end
@@ -72,17 +99,17 @@
       #
       def delete_all
         ids = []
         ranges = []
 
-        if key_present?
-          Dynamoid.adapter.query(source.table_name, range_query).collect do |hash|
+        if @key_fields_detector.key_present?
+          Dynamoid.adapter.query(source.table_name, range_query).flat_map{ |i| i }.collect do |hash|
             ids << hash[source.hash_key.to_sym]
             ranges << hash[source.range_key.to_sym] if source.range_key
           end
         else
-          Dynamoid.adapter.scan(source.table_name, scan_query, scan_opts).collect do |hash|
+          Dynamoid.adapter.scan(source.table_name, scan_query, scan_opts).flat_map{ |i| i }.collect do |hash|
             ids << hash[source.hash_key.to_sym]
             ranges << hash[source.range_key.to_sym] if source.range_key
           end
         end
 
@@ -126,54 +153,73 @@
       # @since 0.2.0
       def each(&block)
         records.each(&block)
       end
 
+      def find_by_pages(&block)
+        pages.each(&block)
+      end
+
       private
 
       # The actual records referenced by the association.
       #
       # @return [Enumerator] an iterator of the found records.
       #
       # @since 0.2.0
       def records
-        if key_present?
-          records_via_query
+        pages.lazy.flat_map { |i| i }
+      end
+
+      # Arrays of records, sized based on the actual pages produced by DynamoDB
+      #
+      # @return [Enumerator] an iterator of the found records.
+      #
+      # @since 3.1.0
+      def pages
+        if @key_fields_detector.key_present?
+          pages_via_query
         else
-          records_via_scan
+          issue_scan_warning if Dynamoid::Config.warn_on_scan && query.present?
+          pages_via_scan
         end
       end
 
-      def records_via_query
+      # If the query matches an index, we'll query the associated table to find results.
+      #
+      # @return [Enumerator] an iterator of the found pages. An array of records
+      #
+      # @since 3.1.0
+      def pages_via_query
         Enumerator.new do |yielder|
-          Dynamoid.adapter.query(source.table_name, range_query).each do |hash|
-            yielder.yield source.from_database(hash)
+          Dynamoid.adapter.query(source.table_name, range_query).each do |items, metadata|
+            yielder.yield items.map { |hash| source.from_database(hash) }, metadata.slice(:last_evaluated_key)
           end
         end
       end
 
       # If the query does not match an index, we'll manually scan the associated table to find results.
       #
-      # @return [Enumerator] an iterator of the found records.
+      # @return [Enumerator] an iterator of the found pages. An array of records
       #
-      # @since 0.2.0
-      def records_via_scan
-        if Dynamoid::Config.warn_on_scan && query.present?
-          Dynamoid.logger.warn 'Queries without an index are forced to use scan and are generally much slower than indexed queries!'
-          Dynamoid.logger.warn "You can index this query by adding index declaration to #{source.to_s.downcase}.rb:"
-          Dynamoid.logger.warn "* global_secondary_index hash_key: 'some-name', range_key: 'some-another-name'"
-          Dynamoid.logger.warn "* local_secondary_indexe range_key: 'some-name'"
-          Dynamoid.logger.warn "Not indexed attributes: #{query.keys.sort.collect { |name| ":#{name}" }.join(', ')}"
-        end
-
+      # @since 3.1.0
+      def pages_via_scan
         Enumerator.new do |yielder|
-          Dynamoid.adapter.scan(source.table_name, scan_query, scan_opts).each do |hash|
-            yielder.yield source.from_database(hash)
+          Dynamoid.adapter.scan(source.table_name, scan_query, scan_opts).each do |items, metadata|
+            yielder.yield(items.map { |hash| source.from_database(hash) }, metadata.slice(:last_evaluated_key))
           end
         end
       end
 
+      def issue_scan_warning
+        Dynamoid.logger.warn 'Queries without an index are forced to use scan and are generally much slower than indexed queries!'
+        Dynamoid.logger.warn "You can index this query by adding index declaration to #{source.to_s.downcase}.rb:"
+        Dynamoid.logger.warn "* global_secondary_index hash_key: 'some-name', range_key: 'some-another-name'"
+        Dynamoid.logger.warn "* local_secondary_index range_key: 'some-name'"
+        Dynamoid.logger.warn "Not indexed attributes: #{query.keys.sort.collect { |name| ":#{name}" }.join(', ')}"
+      end
+
       def count_via_query
         Dynamoid.adapter.query_count(source.table_name, range_query)
       end
 
       def count_via_scan
@@ -236,28 +282,28 @@
 
       def range_query
         opts = {}
 
         # Add hash key
-        opts[:hash_key] = @hash_key
-        opts[:hash_value] = type_cast_condition_parameter(@hash_key, query[@hash_key])
+        opts[:hash_key] = @key_fields_detector.hash_key
+        opts[:hash_value] = type_cast_condition_parameter(@key_fields_detector.hash_key, query[@key_fields_detector.hash_key])
 
         # Add range key
-        if @range_key
-          opts[:range_key] = @range_key
-          if query[@range_key].present?
-            value = type_cast_condition_parameter(@range_key, query[@range_key])
+        if @key_fields_detector.range_key
+          opts[:range_key] = @key_fields_detector.range_key
+          if query[@key_fields_detector.range_key].present?
+            value = type_cast_condition_parameter(@key_fields_detector.range_key, query[@key_fields_detector.range_key])
             opts.update(range_eq: value)
           end
 
-          query.keys.select { |k| k.to_s =~ /^#{@range_key}\./ }.each do |key|
+          query.keys.select { |k| k.to_s =~ /^#{@key_fields_detector.range_key}\./ }.each do |key|
             opts.merge!(range_hash(key))
           end
         end
 
-        (query.keys.map(&:to_sym) - [@hash_key.to_sym, @range_key.try(:to_sym)])
-          .reject { |k, _| k.to_s =~ /^#{@range_key}\./ }
+        (query.keys.map(&:to_sym) - [@key_fields_detector.hash_key.to_sym, @key_fields_detector.range_key.try(:to_sym)])
+          .reject { |k, _| k.to_s =~ /^#{@key_fields_detector.range_key}\./ }
           .each do |key|
           if key.to_s.include?('.')
             opts.update(field_hash(key))
           else
             value = type_cast_condition_parameter(key, query[key])
@@ -282,58 +328,19 @@
             Dumping.dump_field(value_casted, options)
           end
         end
       end
 
-      def key_present?
-        query_keys = query.keys.collect { |k| k.to_s.split('.').first }
-
-        # See if querying based on table hash key
-        if query.keys.map(&:to_s).include?(source.hash_key.to_s)
-          @hash_key = source.hash_key
-
-          # Use table's default range key
-          if query_keys.include?(source.range_key.to_s)
-            @range_key = source.range_key
-            return true
-          end
-
-          # See if can use any local secondary index range key
-          # Chooses the first LSI found that can be utilized for the query
-          source.local_secondary_indexes.each do |_, lsi|
-            next unless query_keys.include?(lsi.range_key.to_s)
-            @range_key = lsi.range_key
-            @index_name = lsi.name
-          end
-
-          return true
-        end
-
-        # See if can use any global secondary index
-        # Chooses the first GSI found that can be utilized for the query
-        # But only do so if projects ALL attributes otherwise we won't
-        # get back full data
-        source.global_secondary_indexes.each do |_, gsi|
-          next unless query.keys.map(&:to_s).include?(gsi.hash_key.to_s) && gsi.projected_attributes == :all
-          @hash_key = gsi.hash_key
-          @range_key = gsi.range_key
-          @index_name = gsi.name
-          return true
-        end
-
-        # Could not utilize any indices so we'll have to scan
-        false
-      end
-
       # Start key needs to be set up based on the index utilized
       # If using a secondary index then we must include the index's composite key
       # as well as the tables composite key.
       def start_key
         return @start if @start.is_a?(Hash)
-        hash_key = @hash_key || source.hash_key
-        range_key = @range_key || source.range_key
 
+        hash_key = @key_fields_detector.hash_key || source.hash_key
+        range_key = @key_fields_detector.range_key || source.range_key
+
         key = {}
         key[hash_key] = type_cast_condition_parameter(hash_key, @start.send(hash_key))
         if range_key
           key[range_key] = type_cast_condition_parameter(range_key, @start.send(range_key))
         end
@@ -347,10 +354,10 @@
         key
       end
 
       def query_opts
         opts = {}
-        opts[:index_name] = @index_name if @index_name
+        opts[:index_name] = @key_fields_detector.index_name if @key_fields_detector.index_name
         opts[:select] = 'ALL_ATTRIBUTES'
         opts[:record_limit] = @record_limit if @record_limit
         opts[:scan_limit] = @scan_limit if @scan_limit
         opts[:batch_size] = @batch_size if @batch_size
         opts[:exclusive_start_key] = start_key if @start