sync_spec.rb in flydata-0.6.4

- old
+ new

@@ -3,14 +3,14 @@
 require 'flydata/command/sync'
 
 module Flydata
   module Command
     describe Sync do
-      let(:subject_object){ described_class.new }
       subject { subject_object }
+      let(:subject_object){ described_class.new }
 
-      let(:default_mysqldump_dir) do
+      let(:default_dump_dir) do
         File.join('/tmp', "sync_dump_#{Time.now.to_i}")
       end
       let(:default_data_entry) do
         {"id"=>93,
         "name"=>"flydata_sync_mysql",
@@ -34,30 +34,26 @@
         "mysql_data_entry_preference" =>
           { "host"=>"localhost", "port"=>3306, "username"=>"masashi",
             "password"=>"welcome", "database"=>"sync_test", "tables"=>["table1", "table2", "table4"],
             "invalid_tables"=>["table3"],
             "new_tables"=>["table4"],
-            "mysqldump_dir"=>default_mysqldump_dir, "forwarder" => "tcpforwarder",
+            "dump_dir"=>default_dump_dir, "forwarder" => "tcpforwarder",
             "data_servers"=>"localhost:9905" }
          }
       end
       let(:col4_type) { "binary" }
       let(:col4_width) { 34 }
       let(:col4_type_str) { "#{col4_type}(#{col4_width})" }
       let(:mysql_table_columns) {
         {"id"=>{:column_name=>"id", :format_type_str=>"int(11)", :format_type=>"int", :format_size=>11}, "name"=>{:column_name=>"name", :format_type_str=>"varchar(40)", :format_type=>"varchar", :format_size=>40, :default=>nil}, "created_at"=>{:column_name=>"created_at", :format_type_str=>"timestamp", :format_type=>"timestamp", :default=>"CURRENT_TIMESTAMP"}, "bin"=>{:column_name=>"bin", :format_type_str=>col4_type_str, :format_type=>col4_type, :format_size=>col4_width, :default=>nil}, "bin2"=>{:column_name=>"bin2", :format_type_str=>"blob", :format_type=>"blob"}, "varbin"=>{:column_name=>"varbin", :format_type_str=>"varchar(34)", :format_type=>"varchar", :format_size=>34, :default=>nil}}
       }
-      let(:mysql_table) do
-        Flydata::Parser::Mysql::MysqlTable.new("test_table", mysql_table_columns, ['id'])
-      end
-
       after :each do
-        if Dir.exists?(default_mysqldump_dir)
-          Dir.delete(default_mysqldump_dir) rescue nil
+        if Dir.exists?(default_dump_dir)
+          Dir.delete(default_dump_dir) rescue nil
         end
-        if File.exists?(default_mysqldump_dir)
-          File.delete(default_mysqldump_dir) rescue nil
+        if File.exists?(default_dump_dir)
+          File.delete(default_dump_dir) rescue nil
         end
       end
 
       describe '#cleanup_sync_server' do
         let(:rest_client) { double('rest_client') }
@@ -76,11 +72,11 @@
             expect(rest_client).to receive(:post).and_return("{}")
             subject.send(:cleanup_sync_server, default_data_entry)
           end
         end
       end
-      describe '#generate_mysqldump' do
+      describe '#generate_source_dump' do
         let (:flydata) { double('flydata') }
         let (:dp) { double('dp') }
         let (:default_data_port) { double('default_data_port') }
         let (:default_sync_fm) { double('default_sync_fm') }
         let (:default_dump_pos) do {
@@ -91,176 +87,77 @@
         let (:default_backup) { double('default_backup') }
         let (:target_tables) { ["test_table_1"] }
         let (:db_byte) { 1 }
         let (:disk_byte) { 100 }
         before do
+          require 'flydata/source_mysql/generate_source_dump'
+          allow_any_instance_of(Flydata::SourceMysql::GenerateSourceDump).to receive(:dump_size).and_return(db_byte)
+          allow_any_instance_of(Flydata::SourceMysql::GenerateSourceDump).to receive(:run_compatibility_check)
+        end
+        before do
           expect(subject).to receive(:flydata).and_return(flydata).at_least(:once)
+          allow(subject).to receive(:data_entry).and_return(default_data_entry)
           expect(flydata).to receive(:data_port).and_return(dp)
-          allow(flydata).to receive(:data_entry).and_return(default_data_entry)
           expect(dp).to receive(:get).and_return(default_data_port)
           allow(File).to receive(:exists?).and_return(false)
           expect(default_sync_fm).to receive(:load_dump_pos).and_return(default_dump_pos)
           expect(default_sync_fm).to receive(:dump_file_path).and_return(default_fp)
           expect(default_sync_fm).to receive(:backup_dir).and_return(default_backup)
           expect(subject).to receive(:target_tables).and_return(target_tables).at_least(:once)
           expect(subject).to receive(:print)
           expect(subject).to receive(:log_info)
           expect(subject).to receive(:log_info_stdout).at_least(:once)
           expect(subject).to receive(:ask_yes_no).and_return(true).at_least(:once)
-          Flydata::Parser::Mysql::DatabaseSizeCheck.any_instance.should_receive(:get_db_bytesize).and_return(db_byte)
-          Flydata::MysqlCompatibilityCheck.any_instance.should_receive(:check)
           expect_any_instance_of(FlydataCore::Event::ApiEventSender).to receive(:send_event).once
         end
         context 'with no stream option' do
           before do
             expect(default_sync_fm).to receive(:save_sync_info).once
             expect(subject).to receive(:free_disk_space).and_return(disk_byte)
             expect(File).to receive(:dirname)
           end
           it 'will export to dump file' do
             expect(subject).to receive(:call_block_or_return_io)
-            Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.any_instance.should_receive(:dump).with(default_fp)
+            expect_any_instance_of(Flydata::SourceMysql::GenerateSourceDump).
+              to receive(:dump).with(target_tables, default_fp)
 
-            subject.send(:generate_mysqldump, default_data_entry, default_sync_fm)
+            subject.send(:generate_source_dump, default_data_entry, default_sync_fm)
           end
           it 'will remove dump file on interrupt' do
             expect(default_sync_fm).to receive(:delete_dump_file)
+            expect_any_instance_of(Flydata::SourceMysql::Parser::MysqlDumpGeneratorNoMasterData).to receive(:dump).and_raise(Interrupt)
 
             expect {
-              Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.any_instance.should_receive(:dump).and_raise(Interrupt)
-              subject.send(:generate_mysqldump, default_data_entry, default_sync_fm)
+              subject.send(:generate_source_dump, default_data_entry, default_sync_fm)
             }.to raise_error
           end
         end
         context 'with stream option' do
           it 'will export to io' do
             expect(default_sync_fm).to receive(:save_sync_info).once
-            Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.any_instance.should_receive(:dump)
+            expect_any_instance_of(Flydata::SourceMysql::Parser::MysqlDumpGeneratorNoMasterData).to receive(:dump)
 
-            subject.send(:generate_mysqldump, default_data_entry, default_sync_fm, false)
+            subject.send(:generate_source_dump, default_data_entry, default_sync_fm, false)
           end
         end
       end
-      describe '#do_generate_table_ddl' do
-        before do
-          allow(subject).to receive(:data_entry).and_return(default_data_entry)
-          allow_any_instance_of(Flydata::Api::DataEntry).to receive(:update_table_validity).and_return(true)
-          subject.send(:set_current_tables, nil, include_all_tables: true)
-        end
-        shared_examples 'throws an error' do
-          it "throws an error" do
-            expect {
-              subject.send(:do_generate_table_ddl, default_data_entry)
-            }.to raise_error
-          end
-        end
-        context 'with full options' do
-          it 'issues mysqldump command with expected parameters' do
-            expect(Open3).to receive(:popen3).with(
-             'mysqldump -h localhost -P 3306 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
-            subject.send(:do_generate_table_ddl, default_data_entry)
-          end
-        end
-        context 'without_host' do
-          before do
-            default_data_entry['mysql_data_entry_preference'].delete('host')
-          end
-          include_examples 'throws an error'
-        end
-        context 'with empty host' do
-          before do
-            default_data_entry['mysql_data_entry_preference']['host'] = ""
-          end
-          include_examples 'throws an error'
-        end
-        context 'without_port' do
-          before do
-            default_data_entry['mysql_data_entry_preference'].delete('port')
-          end
-          it "uses the default port" do
-            expect(Open3).to receive(:popen3).with(
-             'mysqldump -h localhost -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
-            subject.send(:do_generate_table_ddl, default_data_entry)
-          end
-        end
-        context 'with_port_override' do
-          before do
-            default_data_entry['mysql_data_entry_preference']['port'] = 1234
-          end
-          it "uses the specified port" do
-            expect(Open3).to receive(:popen3).with(
-             'mysqldump -h localhost -P 1234 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
-            subject.send(:do_generate_table_ddl, default_data_entry)
-          end
-        end
-        context 'without_username' do
-          before do
-            default_data_entry['mysql_data_entry_preference'].delete('username')
-          end
-          include_examples 'throws an error'
-        end
-        context 'with empty username' do
-          before do
-            default_data_entry['mysql_data_entry_preference']['username'] = ""
-          end
-          include_examples 'throws an error'
-        end
-        context 'without_password' do
-          before do
-            default_data_entry['mysql_data_entry_preference'].delete('password')
-          end
-          it "call mysqldump without MYSQL_PW set" do
-            expect(Open3).to receive(:popen3).with(
-             'mysqldump -h localhost -P 3306 -umasashi --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
-            subject.send(:do_generate_table_ddl, default_data_entry)
-          end
-        end
-        context 'with password containing symbols' do
-          before do
-            default_data_entry['mysql_data_entry_preference'].delete('password')
-            default_data_entry['mysql_data_entry_preference']['password']="welcome&!@^@#^"
-          end
-          it "call mysqldump with MYSQL_PW set with correct symbols" do
-            expect(Open3).to receive(:popen3).with(
-             'mysqldump -h localhost -P 3306 -umasashi -pwelcome\\&\\!@\\^@\\#\\^ --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
-            subject.send(:do_generate_table_ddl, default_data_entry)
-          end
-        end
-        context 'without_database' do
-          before do
-            default_data_entry['mysql_data_entry_preference'].delete('database')
-          end
-          include_examples 'throws an error'
-        end
-        context 'with empty database' do
-          before do
-            default_data_entry['mysql_data_entry_preference']['database'] = ""
-          end
-          include_examples 'throws an error'
-        end
-        context 'with empty tables' do
-          let(:sync_cmd) { described_class.new }
-          before do
-            default_data_entry['mysql_data_entry_preference']['tables'] = []
-            default_data_entry['mysql_data_entry_preference']['invalid_tables'] = []
-            default_data_entry['mysql_data_entry_preference']['new_tables'] = []
-            allow(sync_cmd).to receive(:data_entry).and_return(default_data_entry)
-            sync_cmd.send(:set_current_tables, nil, include_all_tables: true)
-          end
-          it 'should raise error' do
-            expect{sync_cmd.send(:do_generate_table_ddl, default_data_entry)}.to raise_error
-          end
-        end
-      end
       describe '#convert_to_flydata_values' do
-        subject { subject_object.send(:convert_to_flydata_values, mysql_table, values) }
+        subject { subject_object.send(:convert_to_flydata_values, source_table, values) }
         let(:values) { [4, 'John', nil, col4_value, nil, nil] }
 
+        let(:source_table) do
+          Flydata::Parser::SourceTable.new("test_table", mysql_table_columns, ['id'])
+        end
+
         before do
-          mysql_table.set_value_converters(FlydataCore::TableDef::MysqlTableDef::VALUE_CONVERTERS)
+          require 'flydata/parser/source_table'
         end
 
+        before do
+          source_table.set_value_converters(FlydataCore::TableDef::MysqlTableDef::VALUE_CONVERTERS)
+        end
+
         context 'with binary column' do
           let(:col4_type) { "binary" }
           let(:col4_width) { 5 }
           let(:truncated_binary) { "0xC04482" }
           let(:col4_value) { "#{truncated_binary}0000" }
@@ -268,9 +165,75 @@
           it 'truncates trailing "0" if the type is binary' do
             expected_values = values.dup
             expected_values[3] = truncated_binary
             subject
             expect(values).to eq expected_values
+          end
+        end
+      end
+
+      describe '#data_entry' do
+        subject { subject_object.send(:data_entry) }
+
+        let(:de) { { 'mysql_data_entry_preference' => mp } }
+        let(:mp) { { 'tables' => 'Users,Addresses' } }
+
+        let(:sfm) { double('sfm') }
+        let(:ssl_ca_content) { double('ssl_ca_content') }
+        let(:ssl_ca_path) { double('ssl_ca_path') }
+
+        before do
+          allow(subject_object).to receive(:retrieve_data_entries).
+            and_return([de])
+        end
+
+        context 'type RedshiftMysqlDataEntry' do
+          before { de['type'] = 'RedshiftMysqlDataEntry' }
+          context 'called once' do
+            before do
+              expect(subject_object).to receive(:retrieve_data_entries).
+                and_return([de])
+            end
+            context 'without tables_append_only' do
+              it "expands a table list string to an array of tables" do
+                subject
+                expect(mp['tables']).to eq %w(Users Addresses)
+              end
+            end
+            context 'with tables_append_only' do
+              before { mp['tables_append_only'] = 'Invoices,Sessions,Addresses' }
+              it "creates an array of tables from 'tables' and 'tables_append_only' combined" do
+                subject
+                expect(mp['tables']).to eq %w(Users Addresses Invoices Sessions)
+              end
+            end
+            context 'with ssl_ca_content' do
+              before { mp["ssl_ca_content"] = ssl_ca_content }
+              it "saves the content to a local file via SyncFileManager" do
+                expect(SyncFileManager).to receive(:new).with(de).
+                  and_return(sfm)
+                expect(sfm).to receive(:save_ssl_ca).with(ssl_ca_content)
+                expect(sfm).to receive(:ssl_ca_path).and_return(ssl_ca_path)
+
+                subject
+                expect(mp['ssl_ca']).to eq ssl_ca_path
+                expect(mp['sslca']).to eq ssl_ca_path
+              end
+            end
+          end
+          context 'called twice' do
+            before { subject }
+            it "repurposes the saved de" do
+              expect(subject_object).to receive(:retrieve_data_entries).never
+              subject
+              expect(mp['tables']).to eq %w(Users Addresses)
+            end
+          end
+        end
+        context 'type RedshiftFileDataEntry' do
+          before { de['type'] = 'RedshiftFileDataEntry' }
+          it "raises an error about unsupported data entry" do
+            expect { subject }.to raise_error /(supported data entry|data entry.*support)/
           end
         end
       end
     end
   end