test_mixpanel.rb in embulk-input-mixpanel-0.3.3

- old
+ new

@@ -48,21 +48,25 @@
       def setup_logger
         stub(Embulk).logger { ::Logger.new(IO::NULL) }
       end
 
       class GuessTest < self
+        def setup
+          # Do nothing from parent
+          mute_warn
+        end
+
         def test_from_date_old_date
           config = {
             type: "mixpanel",
             api_key: API_KEY,
             api_secret: API_SECRET,
             from_date: FROM_DATE,
           }
 
-          from_date = config[:from_date]
-          to_date = Date.parse(from_date) + Mixpanel::SLICE_DAYS_COUNT
-          stub_export(from_date, to_date)
+          stub_export_all
+          mock(Embulk.logger).info(/^Guessing.*#{Regexp.escape FROM_DATE}\.\./)
 
           actual = Mixpanel.guess(embulk_config(config))
           assert_equal(expected, actual)
         end
 
@@ -72,63 +76,56 @@
             api_key: API_KEY,
             api_secret: API_SECRET,
             from_date: Date.today.to_s,
           }
 
-          assert_raise(ConfigError) do
-            Mixpanel.guess(embulk_config(config))
-          end
+          stub_export_all
+          mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date.to_s}/)
+
+          Mixpanel.guess(embulk_config(config))
         end
 
         def test_from_date_yesterday
+          from_date = (Date.today - 1).to_s
           config = {
             type: "mixpanel",
             api_key: API_KEY,
             api_secret: API_SECRET,
-            from_date: (Date.today - 1).to_s,
+            from_date: from_date,
           }
 
-          from_date = config[:from_date]
-          to_date = from_date
-          stub_export(from_date, to_date)
+          stub_export_all
+          mock(Embulk.logger).info(/Guessing.*#{Regexp.escape from_date}/)
 
-          actual = Mixpanel.guess(embulk_config(config))
-          assert_equal(expected, actual)
+          Mixpanel.guess(embulk_config(config))
         end
 
         def test_no_from_date
           config = {
             type: "mixpanel",
             api_key: API_KEY,
             api_secret: API_SECRET,
           }
 
-          from_date = Date.today - 1 - Mixpanel::SLICE_DAYS_COUNT
-          to_date = Date.today - 1
-          stub_export(from_date, to_date)
+          stub_export_all
+          mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date.to_s}/)
 
-          actual = Mixpanel.guess(embulk_config(config))
-          assert_equal(expected, actual)
+          Mixpanel.guess(embulk_config(config))
         end
 
         private
 
-        def stub_export(from_date, to_date)
-          params = {
-            api_key: API_KEY,
-            event: nil,
-            where: nil,
-            bucket: nil,
-            from_date: from_date.to_s,
-            to_date: to_date.to_s,
-          }
-
+        def stub_export_all
           any_instance_of(MixpanelApi::Client) do |klass|
-            stub(klass).export(params) { records }
+            stub(klass).export(anything) { records }
           end
         end
 
+        def mute_warn
+          stub(Embulk.logger).warn(anything) {}
+        end
+
         def embulk_config(config)
           DataSource[*config.to_a.flatten(1)]
         end
 
         def expected
@@ -334,13 +331,16 @@
             schema: schema
           )
         end
 
         def columns
-          schema.map do |col|
+          unknown_columns = [Column.new(nil, "unknown_columns", :string)]
+          configured_columns = schema.map do |col|
             Column.new(nil, col["name"], col["type"].to_sym)
           end
+
+          configured_columns + unknown_columns
         end
       end
 
       def test_export_params
         config_params = [
@@ -369,11 +369,11 @@
 
       class RunTest < self
         def setup_client
 
           any_instance_of(MixpanelApi::Client) do |klass|
-            stub(klass).export(anything) { records }
+            stub(klass).request { records_raw_response }
           end
         end
 
         def setup
           super
@@ -413,10 +413,56 @@
           mock(@page_builder).finish
 
           @plugin.run
         end
 
+        class UnknownColumnsTest < self
+          def setup
+            super
+            @page_builder = Object.new
+            @plugin = Mixpanel.new(task, nil, nil, @page_builder)
+          end
+
+          def test_run
+            Embulk.logger.warn(anything)
+            stub(@plugin).preview? { false }
+
+            # NOTE: Expect records are contained same record
+            record = records.first
+            properties = record["properties"]
+
+            time = properties["time"]
+            tz = TZInfo::Timezone.get(TIMEZONE)
+            offset = tz.period_for_local(time, true).offset.utc_offset
+            adjusted_time = time - offset
+
+            added = [
+              properties["foo"],
+              adjusted_time,
+              {"int" => properties["int"], "event" => record["event"]}.to_json
+            ]
+
+            mock(@page_builder).add(added).times(records.length * 2)
+            mock(@page_builder).finish
+
+            @plugin.run
+          end
+
+          private
+
+          def task
+            super.merge(schema: schema, fetch_unknown_columns: true)
+          end
+
+          def schema
+            [
+              {"name" => "foo", "type" => "long"},
+              {"name" => "time", "type" => "long"},
+            ]
+          end
+        end
+
         private
 
         def timezone_offset_seconds
           60 * 60 * 9 # Asia/Tokyo
         end
@@ -438,10 +484,13 @@
           api_secret: API_SECRET,
           timezone: TIMEZONE,
           schema: schema,
           dates: DATES.to_a.map(&:to_s),
           params: Mixpanel.export_params(embulk_config),
+          fetch_unknown_columns: false,
+          retry_initial_wait_sec: 2,
+          retry_limit: 3,
         }
       end
 
       def records
         [
@@ -454,10 +503,14 @@
             }
           },
         ] * 30
       end
 
+      def records_raw_response
+        records.map(&:to_json).join("\n")
+      end
+
       def record_epoch
         1234567890
       end
 
       def config
@@ -465,9 +518,12 @@
           type: "mixpanel",
           api_key: API_KEY,
           api_secret: API_SECRET,
           from_date: FROM_DATE,
           fetch_days: DAYS,
+          fetch_unknown_columns: false,
+          retry_initial_wait_sec: 2,
+          retry_limit: 3,
         }
       end
 
       def embulk_config
         DataSource[*config.to_a.flatten(1)]