spec/flydata/command/sync_spec.rb in flydata-0.7.6 vs spec/flydata/command/sync_spec.rb in flydata-0.7.7
- old
+ new
@@ -72,10 +72,11 @@
expect(rest_client).to receive(:post).and_return("{}")
subject.send(:cleanup_sync_server, default_data_entry)
end
end
end
+
describe '#generate_source_dump' do
let (:flydata) { double('flydata') }
let (:dp) { double('dp') }
let (:default_data_port) { double('default_data_port') }
let (:default_sync_fm) { double('default_sync_fm') }
@@ -86,10 +87,11 @@
let (:default_fp) { double('default_fp') }
let (:default_backup) { double('default_backup') }
let (:target_tables) { ["test_table_1"] }
let (:db_byte) { 1 }
let (:disk_byte) { 100 }
+
before do
require 'flydata/source_mysql/generate_source_dump'
allow_any_instance_of(Flydata::SourceMysql::GenerateSourceDump).to receive(:dump_size).and_return(db_byte)
allow_any_instance_of(Flydata::SourceMysql::GenerateSourceDump).to receive(:run_compatibility_check)
end
@@ -107,10 +109,11 @@
expect(subject).to receive(:log_info)
expect(subject).to receive(:log_info_stdout).at_least(:once)
expect(subject).to receive(:ask_yes_no).and_return(true).at_least(:once)
expect_any_instance_of(FlydataCore::Event::ApiEventSender).to receive(:send_event).once
end
+
context 'with no stream option' do
before do
expect(default_sync_fm).to receive(:save_sync_info).once
expect(subject).to receive(:free_disk_space).and_return(disk_byte)
expect(File).to receive(:dirname)
@@ -129,19 +132,21 @@
expect {
subject.send(:generate_source_dump, default_data_entry, default_sync_fm)
}.to raise_error
end
end
+
context 'with stream option' do
it 'will export to io' do
expect(default_sync_fm).to receive(:save_sync_info).once
expect_any_instance_of(Flydata::SourceMysql::Parser::MysqlDumpGeneratorNoMasterData).to receive(:dump)
subject.send(:generate_source_dump, default_data_entry, default_sync_fm, false)
end
end
end
+
describe '#convert_to_flydata_values' do
subject { subject_object.send(:convert_to_flydata_values, source_table, values) }
let(:values) { [4, 'John', nil, col4_value, nil, nil] }
let(:source_table) do
@@ -173,18 +178,38 @@
describe '#data_entry' do
subject { subject_object.send(:data_entry) }
let(:de) { { 'mysql_data_entry_preference' => mp } }
- let(:mp) { { 'tables' => 'Users,Addresses',
- 'table_attributes' => [
- {"table_name"=>"Users", "status"=>"init_sync_pending"},
- {"table_name"=>"Addresses", "status"=>"init_sync_pending"}
- ],
- 'pk_override' => {"Users"=>["id"]}
+ let(:mp) { { 'tables' => default_tables_str,
+ 'table_attributes' => default_table_attributes,
+ 'pk_override' => pk_override_hash
} }
+ let(:default_tables_str) { 'Users,Addresses' }
+ let(:default_table_attributes) {[
+ {"table_name"=>"Users", "status"=>"init_sync_pending"},
+ {"table_name"=>"Addresses", "status"=>"init_sync_pending"}
+ ]}
+ let(:append_only_tables_list) { %w|Invoices Sessions Addresses| }
+ let(:append_only_tables_str) { 'Invoices,Sessions,Addresses' }
+ let(:tbl_attrs_for_append_only_tables) { [
+ {"table_name"=>"Invoices", "omit_events"=>["delete"], "status"=>"init_sync_pending"},
+ {"table_name"=>"Sessions", "omit_events"=>["delete"], "status"=>"init_sync_pending"},
+ {"table_name"=>"Addresses", "omit_events"=>["delete"], "status"=>"init_sync_pending"}
+ ] }
+
+ let(:invalid_tables_list) { %w|error_fullsync_1 error_append_2| }
+ let(:invalid_tables_str) { 'error_fullsync_1,error_append_2' }
+ let(:tbl_attrs_for_invalid_tables) {[
+ {"table_name"=>"error_fullsync_1", "status"=>"init_sync_pending",
+ "invalid_table_reason"=>"no primary key defined"},
+ {"table_name"=>"error_append_2", "omit_events"=>["delete"], "status"=>"init_sync_pending",
+ "invalid_table_reason"=>"table does not exist in the MySQL database"},
+ ]}
+
+ let(:pk_override_hash) { {"Users"=>["id"]} }
let(:sfm) { double('sfm') }
let(:ssl_ca_content) { double('ssl_ca_content') }
let(:ssl_ca_path) { double('ssl_ca_path') }
before do
@@ -197,30 +222,78 @@
context 'called once' do
before do
expect(subject_object).to receive(:retrieve_data_entries).
and_return([de])
end
+
context 'without tables_append_only' do
it "expands a table list string to an array of tables" do
subject
expect(mp['tables']).to eq %w(Users Addresses)
end
end
+
context 'with tables_append_only' do
before do
- mp['tables_append_only'] = 'Invoices,Sessions,Addresses'
- mp['table_attributes'] += [
- {"table_name"=>"Invoices", "omit_events"=>["delete"], "status"=>"init_sync_pending"},
- {"table_name"=>"Sessions", "omit_events"=>["delete"], "status"=>"init_sync_pending"},
- {"table_name"=>"Addresses", "omit_events"=>["delete"], "status"=>"init_sync_pending"}
- ]
+ mp['tables_append_only'] = append_only_tables_str
+ mp['table_attributes'] = default_table_attributes + tbl_attrs_for_append_only_tables
end
it "creates an array of tables from 'tables' and 'tables_append_only' combined" do
subject
expect(mp['tables']).to eq %w(Users Addresses Invoices Sessions)
end
+
+ it 'creates an array of append-only tables' do
+ subject
+ expect(mp['tables_append_only']).to eq append_only_tables_list
+ end
end
+
+ context 'with invalid tables and invalid append-only tables' do
+ before do
+ mp['tables_append_only'] = append_only_tables_str
+ mp['table_attributes'] = default_table_attributes + tbl_attrs_for_append_only_tables
+
+ mp['invalid_tables'] = invalid_tables_str
+ mp['table_attributes'] = default_table_attributes +
+ tbl_attrs_for_append_only_tables +
+ tbl_attrs_for_invalid_tables
+ end
+
+ it 'does not change value for `tables`' do
+ subject
+ expect(mp['tables']).to eq %w(Users Addresses Invoices Sessions)
+ end
+
+ it 'does not change value for `tables_append_only`' do
+ subject
+ expect(mp['tables_append_only']).to eq append_only_tables_list
+ end
+
+ it 'creates an array of invalid tables' do
+ subject
+ expect(mp['invalid_tables']).to eq invalid_tables_list
+ end
+
+ it 'categorize invalid tables by sync type' do
+ subject
+ expect(mp['invalid_tables_append_only']).to eq %w(error_append_2)
+ expect(mp['invalid_tables_full_sync']).to eq %w(error_fullsync_1)
+ end
+ end
+
+ context 'with an invalid table which do not have table_attributes entry' do
+ before do
+ mp['invalid_tables'] = invalid_tables_str + ',table_from_conf'
+ mp['table_attributes'] += default_table_attributes + tbl_attrs_for_invalid_tables
+ end
+
+ it 'cannot determine sync type for the table and raise an error' do
+ expect { subject }.to raise_error /Sync type for invalid table `table_from_conf` not known/
+ end
+ end
+
context 'with ssl_ca_content' do
before { mp["ssl_ca_content"] = ssl_ca_content }
it "saves the content to a local file via SyncFileManager" do
expect(SyncFileManager).to receive(:new).with(de).
and_return(sfm)
@@ -231,22 +304,119 @@
expect(mp['ssl_ca']).to eq ssl_ca_path
expect(mp['sslca']).to eq ssl_ca_path
end
end
end
+
context 'called twice' do
before { subject }
it "repurposes the saved de" do
expect(subject_object).to receive(:retrieve_data_entries).never
subject
expect(mp['tables']).to eq %w(Users Addresses)
end
end
end
+
context 'type RedshiftFileDataEntry' do
before { de['type'] = 'RedshiftFileDataEntry' }
it "raises an error about unsupported data entry" do
expect { subject }.to raise_error /(supported data entry|data entry.*support)/
+ end
+ end
+ end
+
+ describe '#set_current_tables' do
+ subject { subject_object.send(:set_current_tables, input_tables, options) }
+ let(:input_tables) { nil }
+ let(:table_lists) {{
+ "tables" => tables,
+ "invalid_tables" => invalid_tables,
+ "new_tables" => new_tables,
+ "tables_append_only" => tables_append_only,
+ "invalid_tables_append_only" => invalid_tables_append_only,
+ }}
+ let(:real_new_tables) { [] }
+ let(:tables) { ["table1", "table2", "table4"] }
+ let(:invalid_tables) { ["table3_invalid","append3_invalid"] }
+ let(:invalid_tables_append_only) { ["append3_invalid"] }
+ let(:new_tables) { ["table4"] }
+
+ let(:data_entry_with_table_lists) do
+ new_de = default_data_entry.dup
+ new_de['mysql_data_entry_preference'].merge!(table_lists)
+ new_de
+ end
+ before do
+ allow(subject_object).to receive(:data_entry).and_return(data_entry_with_table_lists)
+ end
+
+ context 'when include_all_tables option is true' do
+ let(:options) { { include_all_tables: true } }
+
+ context 'when there are append-only tables (valid & invalid)' do
+ let(:tables_append_only) { ["append1","append2"] }
+ let(:invalid_tables_append_only) { ["append3_invalid"] }
+ it do
+ subject
+ expect(subject_object.instance_variable_get(:@full_tables)).to eq(
+ tables + invalid_tables)
+ expect(subject_object.instance_variable_get(:@append_only_tables)).to eq(
+ tables_append_only + invalid_tables_append_only)
+ end
+ end
+
+ context "when table_lists['tables_append_only'] is nil" do
+ let(:tables_append_only) { nil }
+ it do
+ subject
+ expect(subject_object.instance_variable_get(:@full_tables)).to eq(
+ tables + invalid_tables)
+ expect(subject_object.instance_variable_get(:@append_only_tables)).to eq(
+ invalid_tables_append_only)
+ end
+ end
+
+ context 'when there is no append-only tables' do
+ let(:tables_append_only) { [] }
+ let(:invalid_tables_append_only) { [] }
+ it do
+ subject
+ expect(subject_object.instance_variable_get(:@full_tables)).to eq(
+ tables + invalid_tables)
+ expect(subject_object.instance_variable_get(:@append_only_tables)).to eq([])
+ end
+ end
+ end
+
+ context 'when include_all_tables option is false' do
+ let(:options) { { include_all_tables: false } }
+ context 'when there is no real_new_tables' do
+ let(:real_new_tables) { [] }
+ let(:tables_append_only) { ["append1","append2"] }
+ let(:invalid_tables_append_only) { ["append3_invalid"] }
+ before do
+ allow_any_instance_of(SyncFileManager).to receive(:get_new_table_list).and_return(real_new_tables)
+ end
+ it do
+ subject
+ expect(subject_object.instance_variable_get(:@full_tables)).to eq(tables - real_new_tables)
+ expect(subject_object.instance_variable_get(:@append_only_tables)).to eq(tables_append_only)
+ end
+ end
+
+ context 'there is real_new_tables' do
+ let(:real_new_tables) { ['table1'] }
+ let(:tables_append_only) { ["append1","append2"] }
+ let(:invalid_tables_append_only) { ["append3_invalid"] }
+ before do
+ allow_any_instance_of(SyncFileManager).to receive(:get_new_table_list).and_return(real_new_tables)
+ end
+ it do
+ subject
+ expect(subject_object.instance_variable_get(:@full_tables)).to eq(tables - real_new_tables)
+ expect(subject_object.instance_variable_get(:@append_only_tables)).to eq(tables_append_only)
+ end
end
end
end
end
end