lib/gooddata/models/model.rb in gooddata-0.6.0.pre11 vs lib/gooddata/models/model.rb in gooddata-0.6.0

- old
+ new

@@ -1,15 +1,18 @@ +# encoding: UTF-8 + +require_relative '../helpers' + require 'open-uri' require 'active_support/all' + ## # Module containing classes that counter-part GoodData server-side meta-data # elements, including the server-side data model. # module GoodData - module Model - # GoodData REST API categories LDM_CTG = 'ldm' LDM_MANAGE_CTG = 'ldm-manage' # Model naming conventions @@ -22,11 +25,11 @@ ATTRIBUTE_FOLDER_PREFIX = 'dim' ATTRIBUTE_PREFIX = 'attr' LABEL_PREFIX = 'label' FACT_PREFIX = 'fact' DATE_FACT_PREFIX = 'dt' - DATE_ATTRIBUTE = "date" + DATE_ATTRIBUTE = 'date' DATE_ATTRIBUTE_DEFAULT_DISPLAY_FORM = 'mdyy' TIME_FACT_PREFIX = 'tm.dt' TIME_ATTRIBUTE_PREFIX = 'attr.time' FACT_FOLDER_PREFIX = 'ffld' @@ -81,41 +84,39 @@ # kick the load pull = {'pullIntegration' => File.basename(dir)} link = project.md.links('etl')['pull'] task = GoodData.post link, pull - while (GoodData.get(task["pullTask"]["uri"])["taskStatus"] === "RUNNING" || GoodData.get(task["pullTask"]["uri"])["taskStatus"] === "PREPARED") do + while GoodData.get(task['pullTask']['uri'])['taskStatus'] === 'RUNNING' || GoodData.get(task['pullTask']['uri'])['taskStatus'] === 'PREPARED' sleep 30 end - if (GoodData.get(task["pullTask"]["uri"])["taskStatus"] == "ERROR") + if GoodData.get(task['pullTask']['uri'])['taskStatus'] == 'ERROR' s = StringIO.new GoodData.download_from_user_webdav(File.basename(dir) + '/upload_status.json', s) - js = JSON.parse(s.string) + js = MultiJson.load(s.string) fail "Load Failed with error #{JSON.pretty_generate(js)}" end end def merge_dataset_columns(a_schema_blueprint, b_schema_blueprint) a_schema_blueprint = a_schema_blueprint.to_hash b_schema_blueprint = b_schema_blueprint.to_hash d = Marshal.load(Marshal.dump(a_schema_blueprint)) d[:columns] = d[:columns] + b_schema_blueprint[:columns] d[:columns].uniq! - columns_that_failed_to_merge = d[:columns].group_by {|x| x[:name]}.map {|k, v| [k, v.count]}.find_all {|x| x[1] > 1} + columns_that_failed_to_merge = d[:columns].group_by { |x| x[:name] }.map { |k, v| [k, v.count] }.find_all { |x| x[1] > 1 } fail "Columns #{columns_that_failed_to_merge} failed to merge. When merging columns with the same name they have to be identical." unless columns_that_failed_to_merge.empty? d end - end class ProjectBlueprint - attr_accessor :data def self.from_json(spec) if spec.is_a?(String) - ProjectBlueprint.new(JSON.parse(File.read(spec), :symbolize_names => true)) + ProjectBlueprint.new(MultiJson.load(File.read(spec), :symbolize_keys => true)) else ProjectBlueprint.new(spec) end end @@ -138,11 +139,11 @@ data[:datasets].insert(index, a_dataset.to_hash) end end def remove_dataset(dataset_name) - x = data[:datasets].find {|d| d[:name] == dataset_name} + x = data[:datasets].find { |d| d[:name] == dataset_name } index = data[:datasets].index(x) data[:datasets].delete_at(index) end def date_dimensions @@ -203,32 +204,30 @@ data[:title] end def to_wire_model { - "diffRequest" => { - "targetModel" => { - "projectModel" => { - "datasets" => datasets.map {|d| d.to_wire_model}, - "dateDimensions" => date_dimensions.map {|d| - { - "dateDimension" => { - "name" => d[:name], - "title" => d[:title] || d[:name].humanize - } - }} - }}}} + 'diffRequest' => { + 'targetModel' => { + 'projectModel' => { + 'datasets' => datasets.map { |d| d.to_wire_model }, + 'dateDimensions' => date_dimensions.map { |d| + { + 'dateDimension' => { + 'name' => d[:name], + 'title' => d[:title] || d[:name].humanize + } + } } + }}}} end def to_hash @data end - end class SchemaBlueprint - attr_accessor :data def change(&block) builder = SchemaBuilder.create_from_data(self) block.call(builder) @@ -241,12 +240,12 @@ @data = init_data end def upload(source, options={}) project = options[:project] || GoodData.project - fail "You have to specify a project into which you want to load." if project.nil? - mode = options[:load] || "FULL" + fail 'You have to specify a project into which you want to load.' if project.nil? + mode = options[:load] || 'FULL' project.upload(source, to_schema, mode) end def merge!(a_blueprint) new_blueprint = GoodData::Model.merge_dataset_columns(self, a_blueprint) @@ -269,11 +268,11 @@ def columns data[:columns] end def has_anchor? - columns.any? { |c| c[:type].to_s == "anchor" } + columns.any? { |c| c[:type].to_s == 'anchor' } end def anchor find_column_by_type(:anchor, :first) end @@ -333,31 +332,27 @@ end def ==(other) to_hash == other.to_hash end - end class ProjectBuilder - attr_reader :title, :datasets, :reports, :metrics, :uploads, :users, :assert_report, :date_dimensions class << self - - def create_from_data(blueprint) - pb = ProjectBuilder.new + def create_from_data(blueprint, title = 'Title') + pb = ProjectBuilder.new(title) pb.data = blueprint.to_hash pb end def create(title, options={}, &block) pb = ProjectBuilder.new(title) block.call(pb) pb end - end def initialize(title) @title = title @datasets = [] @@ -368,12 +363,18 @@ @users = [] @dashboards = [] @date_dimensions = [] end - def add_date_dimension(name, options={}) - @date_dimensions << {:urn => options[:urn], :name => name, :title => options[:title]} + def add_date_dimension(name, options = {}) + dimension = { + urn: options[:urn], + name: name, + title: options[:title] + } + + @date_dimensions << dimension end def add_dataset(name, &block) builder = GoodData::Model::SchemaBuilder.new(name) block.call(builder) @@ -401,30 +402,30 @@ block.call(db) @dashboards << db.to_hash end def load_metrics(file) - new_metrics = JSON.parse(open(file).read, :symbolize_names => true) + new_metrics = MultiJson.load(open(file).read, :symbolize_keys => true) @metrics = @metrics + new_metrics end def load_datasets(file) - new_metrics = JSON.parse(open(file).read, :symbolize_names => true) + new_metrics = MultiJson.load(open(file).read, :symbolize_keys => true) @datasets = @datasets + new_metrics end def assert_report(report, result) @assert_tests << {:report => report, :result => result} end def upload(data, options={}) - mode = options[:mode] || "FULL" + mode = options[:mode] || 'FULL' dataset = options[:dataset] @uploads << { - :source => data, - :mode => mode, - :dataset => dataset + :source => data, + :mode => mode, + :dataset => dataset } end def add_users(users) @users << users @@ -440,30 +441,28 @@ end end def to_hash { - :title => @title, - :datasets => @datasets, - :uploads => @uploads, - :dashboards => @dashboards, - :metrics => @metrics, - :reports => @reports, - :users => @users, - :assert_tests => @assert_tests, - :date_dimensions => @date_dimensions + :title => @title, + :datasets => @datasets, + :uploads => @uploads, + :dashboards => @dashboards, + :metrics => @metrics, + :reports => @reports, + :users => @users, + :assert_tests => @assert_tests, + :date_dimensions => @date_dimensions } end def get_dataset(name) datasets.find { |d| d.name == name } end - end class DashboardBuilder - def initialize(title) @title = title @tabs = [] end @@ -474,18 +473,17 @@ tb end def to_hash { - :name => @name, - :tabs => @tabs.map { |tab| tab.to_hash } + :name => @name, + :tabs => @tabs.map { |tab| tab.to_hash } } end end class TabBuilder - def initialize(title) @title = title @stuff = [] end @@ -493,35 +491,31 @@ @stuff << {:type => :report}.merge(options) end def to_hash { - :title => @title, - :items => @stuff + :title => @title, + :items => @stuff } end - end class SchemaBuilder - attr_accessor :data class << self - def create_from_data(blueprint) sc = SchemaBuilder.new sc.data = blueprint.to_hash sc end - end def initialize(name=nil) @data = { - :name => name, - :columns => [] + :name => name, + :columns => [] } end def name data[:name] @@ -573,24 +567,21 @@ end def to_schema Schema.new(to_hash) end - end class ProjectCreator - class << self def migrate(options={}) - - spec = options[:spec] || fail("You need to provide spec for migration") + spec = options[:spec] || fail('You need to provide spec for migration') spec = spec.to_hash token = options[:token] project = options[:project] || GoodData::Project.create(:title => spec[:title], :auth_token => token) - fail("You need to specify token for project creation") if token.nil? && project.nil? + fail('You need to specify token for project creation') if token.nil? && project.nil? begin GoodData.with_project(project) do |p| # migrate_date_dimensions(p, spec[:date_dimensions] || []) migrate_datasets(p, spec) @@ -613,12 +604,13 @@ def migrate_datasets(project, spec) bp = ProjectBlueprint.new(spec) # schema = Schema.load(schema) unless schema.respond_to?(:to_maql_create) # project = GoodData.project unless project - result = GoodData.post("/gdc/projects/#{GoodData.project.pid}/model/diff", bp.to_wire_model) - link = result["asyncTask"]["link"]["poll"] + uri = "/gdc/projects/#{GoodData.project.pid}/model/diff" + result = GoodData.post(uri, bp.to_wire_model) + link = result['asyncTask']['link']['poll'] response = GoodData.get(link, :process => false) # pp response while response.code != 200 sleep 1 GoodData.connection.retryable(:tries => 3, :on => RestClient::InternalServerError) do @@ -628,13 +620,13 @@ end end response = GoodData.get(link) ldm_links = GoodData.get project.md[LDM_CTG] ldm_uri = Links.new(ldm_links)[LDM_MANAGE_CTG] - chunks = response["projectModelDiff"]["updateScripts"].find_all {|script| script["updateScript"]["preserveData"] == true && script["updateScript"]["cascadeDrops"] == false}.map {|x| x["updateScript"]["maqlDdlChunks"]}.flatten + chunks = response['projectModelDiff']['updateScripts'].find_all { |script| script['updateScript']['preserveData'] == true && script['updateScript']['cascadeDrops'] == false }.map { |x| x['updateScript']['maqlDdlChunks'] }.flatten chunks.each do |chunk| - GoodData.post ldm_uri, { 'manage' => { 'maql' => chunk } } + GoodData.post ldm_uri, {'manage' => {'maql' => chunk}} end bp.datasets.each do |ds| schema = ds.to_schema GoodData::ProjectMetadata["manifest_#{schema.name}"] = schema.to_manifest.to_json @@ -667,11 +659,11 @@ end def load(project, spec) if spec.has_key?(:uploads) spec[:uploads].each do |load| - schema = GoodData::Model::Schema.new(spec[:datasets].detect {|d| d[:name] == load[:dataset]}) + schema = GoodData::Model::Schema.new(spec[:datasets].detect { |d| d[:name] == load[:dataset] }) project.upload(load[:source], schema, load[:mode]) end end end @@ -714,44 +706,47 @@ def self.load(file) Schema.new JSON.load(open(file)) end - def initialize(config, name = nil) + def initialize(config, name = 'Default Name', title = 'Default Title') super() @fields = [] @attributes = [] @facts = [] @folders = { - :facts => {}, - :attributes => {} + :facts => {}, + :attributes => {} } @references = [] @labels = [] config[:name] = name unless config[:name] - config[:title] = config[:title] || config[:name].humanize + config[:title] = config[:name] unless config[:title] + config[:title] = title unless config[:title] + config[:title] = config[:title].humanize + fail 'Schema name not specified' unless config[:name] self.name = config[:name] self.title = config[:title] self.config = config end def config=(config) config[:columns].each do |c| case c[:type].to_s - when "attribute" + when 'attribute' add_attribute c - when "fact" + when 'fact' add_fact c - when "date" + when 'date' add_date c - when "anchor" + when 'anchor' set_anchor c - when "label" + when 'label' add_label c - when "reference" + when 'reference' add_reference c else fail "Unexpected type #{c[:type]} in #{c.inspect}" end end @@ -771,24 +766,25 @@ ## # Generates MAQL DDL script to drop this data set and included pieces # def to_maql_drop - maql = "" + maql = '' [attributes, facts].each do |obj| maql += obj.to_maql_drop end maql += "DROP {#{self.identifier}};\n" end ## # Generates MAQL DDL script to create this data set and included pieces # def to_maql_create + # TODO: Use template (.erb) maql = "# Create the '#{self.title}' data set\n" maql += "CREATE DATASET {#{self.identifier}} VISUAL (TITLE \"#{self.title}\");\n\n" - [ attributes, facts, { 1 => @anchor } ].each do |objects| + [attributes, facts, {1 => @anchor}].each do |objects| objects.values.each do |obj| maql += "# Create '#{obj.title}' and add it to the '#{self.title}' data set.\n" maql += obj.to_maql_create maql += "ALTER DATASET {#{self.identifier}} ADD {#{obj.identifier}};\n\n" end @@ -807,11 +803,11 @@ folders_maql = "# Create folders\n" (folders[:attributes].values + folders[:facts].values).each { |folder| folders_maql += folder.to_maql_create } folders_maql + "\n" + maql + "SYNCHRONIZE {#{identifier}};\n" end - def upload(path, project = nil, mode = "FULL") + def upload(path, project = nil, mode = 'FULL') if path =~ URI::regexp Tempfile.open('remote_file') do |temp| temp << open(path).read temp.flush upload_data(temp, mode) @@ -825,35 +821,36 @@ GoodData::Model.upload_data(path, to_manifest(mode)) end # Generates the SLI manifest describing the data loading # - def to_manifest(mode="FULL") + def to_manifest(mode = 'FULL') { - 'dataSetSLIManifest' => { - 'parts' => fields.reduce([]) { |memo, f| val = f.to_manifest_part(mode); memo << val unless val.nil?; memo }, - 'dataSet' => self.identifier, - 'file' => 'data.csv', # should be configurable - 'csvParams' => { - 'quoteChar' => '"', - 'escapeChar' => '"', - 'separatorChar' => ',', - 'endOfLine' => "\n" - } + 'dataSetSLIManifest' => { + 'parts' => fields.reduce([]) { |memo, f| val = f.to_manifest_part(mode); memo << val unless val.nil?; memo }, + 'dataSet' => self.identifier, + 'file' => 'data.csv', # should be configurable + 'csvParams' => { + 'quoteChar' => '"', + 'escapeChar' => '"', + 'separatorChar' => ',', + 'endOfLine' => "\n" } + } } end def to_wire_model { - "dataset" => { - "identifier" => identifier, - "title" => title, - "anchor" => @anchor.to_wire_model, - "facts" => facts.map {|f| f.to_wire_model}, - "attributes" => attributes.map {|a| a.to_wire_model}, - "references" => references.map {|r| r.is_a?(DateReference) ? r.schema_ref : type_prefix + "." + r.schema_ref }}} + 'dataset' => { + 'identifier' => identifier, + 'title' => title, + 'anchor' => @anchor.to_wire_model, + 'facts' => facts.map { |f| f.to_wire_model }, + 'attributes' => attributes.map { |a| a.to_wire_model }, + 'references' => references.map { |r| r.is_a?(DateReference) ? r.schema_ref : type_prefix + '.' + r.schema_ref }} + } end private def add_attribute(column) @@ -900,18 +897,17 @@ date = DateColumn.new column, self @fields << date date.parts.values.each { |p| @fields << p } date.facts.each { |f| facts << f } date.attributes.each { |a| attributes << a } - date.references.each {|r| references << r} + date.references.each { |r| references << r } end def set_anchor(column) @anchor = Anchor.new column, self @fields << @anchor end - end ## # This is a base class for server-side LDM elements such as attributes, labels and # facts @@ -920,11 +916,13 @@ attr_accessor :folder, :name, :title, :schema def initialize(hash, schema) super() raise ArgumentError.new("Schema must be provided, got #{schema.class}") unless schema.is_a? Schema - @name = hash[:name] || raise("Data set fields must have their names defined") + raise('Data set fields must have their names defined') if hash[:name].nil? + + @name = hash[:name] @title = hash[:title] || hash[:name].humanize @folder = hash[:folder] @schema = schema end @@ -983,11 +981,11 @@ @labels = [] @primary_label = Label.new hash, self, schema end def table - @table ||= "d_" + @schema.name + "_" + name + @table ||= 'd_' + @schema.name + '_' + name end def key; "#{@name}#{FK_SUFFIX}"; end @@ -999,78 +997,78 @@ maql end def to_manifest_part(mode) { - 'referenceKey' => 1, - 'populates' => [@primary_label.identifier], - 'mode' => mode, - 'columnName' => name + 'referenceKey' => 1, + 'populates' => [@primary_label.identifier], + 'mode' => mode, + 'columnName' => name } end def to_wire_model { - "attribute" => { - "identifier" => identifier, - "title" => title, - "labels" => labels.map do |l| - { - "label" => { - "identifier" => l.identifier, - "title" => l.title, - "type" => "GDC.text" - } - } - end - } + 'attribute' => { + 'identifier' => identifier, + 'title' => title, + 'labels' => labels.map do |l| + { + 'label' => { + 'identifier' => l.identifier, + 'title' => l.title, + 'type' => 'GDC.text' + } + } + end + } } end - end ## # GoodData display form abstraction. Represents a default representation # of an attribute column or an additional representation defined in a LABEL # field # class Label < Column - attr_accessor :attribute - def type_prefix ; 'label' ; end + def type_prefix; + 'label'; + end # def initialize(hash, schema) def initialize(hash, attribute, schema) super hash, schema - attribute = attribute.nil? ? schema.fields.find {|field| field.name === hash[:reference]} : attribute + attribute = attribute.nil? ? schema.fields.find { |field| field.name === hash[:reference] } : attribute @attribute = attribute attribute.labels << self end def to_maql_create - "# LABEL FROM LABEL" + '# LABEL FROM LABEL' "ALTER ATTRIBUTE {#{@attribute.identifier}} ADD LABELS {#{identifier}}" \ + " VISUAL (TITLE #{title.inspect}) AS {#{column}};\n" end def to_manifest_part(mode) { - 'populates' => [identifier], - 'mode' => mode, - 'columnName' => name + 'populates' => [identifier], + 'mode' => mode, + 'columnName' => name } end def column "#{@attribute.table}.#{LABEL_COLUMN_PREFIX}#{name}" end alias :inspect_orig :inspect def inspect - inspect_orig.sub(/>$/, " @attribute=" + @attribute.to_s.sub(/>$/, " @name=#{@attribute.name}") + '>') + inspect_orig.sub(/>$/, " @attribute=#{@attribute.to_s.sub(/>$/, " @name=#{@attribute.name}")}>") end end ## # A GoodData attribute that represents a data set's connection point or a data set @@ -1079,18 +1077,18 @@ class Anchor < Attribute def initialize(column, schema) if column then super else - super({:type => "anchor", :name => 'id'}, schema) + super({:type => 'anchor', :name => 'id'}, schema) @labels = [] @primary_label = nil end end def table - @table ||= "f_" + @schema.name + @table ||= 'f_' + @schema.name end def to_maql_create maql = super maql += "\n# Connect '#{self.title}' to all attributes of this data set\n" @@ -1098,11 +1096,10 @@ maql += "ALTER ATTRIBUTE {#{c.identifier}} ADD KEYS " \ + "{#{table}.#{c.key}};\n" end maql end - end ## # GoodData fact abstraction # @@ -1132,31 +1129,30 @@ + " AS {#{column}};\n" end def to_manifest_part(mode) { - 'populates' => [identifier], - 'mode' => mode, - 'columnName' => name + 'populates' => [identifier], + 'mode' => mode, + 'columnName' => name } end def to_wire_model { - "fact" => { - "identifier" => identifier, - "title" => title + 'fact' => { + 'identifier' => identifier, + 'title' => title } } end end ## # Reference to another data set # class Reference < Column - attr_accessor :reference, :schema_ref def initialize(column, schema) super column, schema # pp column @@ -1191,43 +1187,42 @@ "ALTER ATTRIBUTE {#{self.identifier} DROP KEYS {#{@schema.table}.#{key}};\n" end def to_manifest_part(mode) { - 'populates' => [label_column], - 'mode' => mode, - 'columnName' => name, - 'referenceKey' => 1 + 'populates' => [label_column], + 'mode' => mode, + 'columnName' => name, + 'referenceKey' => 1 } end end ## # Date as a reference to a date dimension # class DateReference < Reference - attr_accessor :format, :output_format, :urn def initialize(column, schema) super column, schema - @output_format = column["format"] || 'dd/MM/yyyy' + @output_format = column['format'] || 'dd/MM/yyyy' @format = @output_format.gsub('yyyy', '%Y').gsub('MM', '%m').gsub('dd', '%d') - @urn = column[:urn] || "URN:GOODDATA:DATE" + @urn = column[:urn] || 'URN:GOODDATA:DATE' end def identifier @identifier ||= "#{@schema_ref}.#{DATE_ATTRIBUTE}" end def to_manifest_part(mode) { - 'populates' => ["#{identifier}.#{DATE_ATTRIBUTE_DEFAULT_DISPLAY_FORM}"], - 'mode' => mode, - 'constraints' => {"date" => output_format}, - 'columnName' => name, - 'referenceKey' => 1 + 'populates' => ["#{identifier}.#{DATE_ATTRIBUTE_DEFAULT_DISPLAY_FORM}"], + 'mode' => mode, + 'constraints' => {'date' => output_format}, + 'columnName' => name, + 'referenceKey' => 1 } end # def to_maql_create # # urn:chefs_warehouse_fiscal:date @@ -1247,14 +1242,14 @@ "#{DATE_COLUMN_PREFIX}#{super}"; end def to_manifest_part(mode) { - 'populates' => ['label.stuff.mmddyy'], - "format" => "unknown", - "mode" => mode, - "referenceKey" => 1 + 'populates' => ['label.stuff.mmddyy'], + 'format' => 'unknown', + 'mode' => mode, + 'referenceKey' => 1 } end end ## @@ -1272,11 +1267,10 @@ ## # Time as a reference to a time-of-a-day dimension # class TimeReference < Reference - end ## # Time field that's not connected to a time-of-a-day dimension # @@ -1341,20 +1335,21 @@ end def to_manifest_part(mode) nil end - end ## # Base class for GoodData attribute and fact folder abstractions # class Folder < MdObject def initialize(title) + # TODO: should a super be here? + # how to deal with name vs title? @title = title - @name = title + @name = GoodData::Helpers.sanitize_string(title) end def to_maql_create "CREATE FOLDER {#{type_prefix}.#{name}}" \ + " VISUAL (#{visual}) TYPE #{type};\n" @@ -1364,49 +1359,46 @@ ## # GoodData attribute folder abstraction # class AttributeFolder < Folder def type; - "ATTRIBUTE"; + 'ATTRIBUTE' end def type_prefix; - "dim"; + 'dim' end end ## # GoodData fact folder abstraction # class FactFolder < Folder def type; - "FACT"; + 'FACT' end def type_prefix; - "ffld"; + 'ffld' end end class DateDimension < MdObject - def initialize(spec={}) super() @name = spec[:name] @title = spec[:title] || @name - @urn = spec[:urn] || "URN:GOODDATA:DATE" + @urn = spec[:urn] || 'URN:GOODDATA:DATE' end def to_maql_create # urn = "urn:chefs_warehouse_fiscal:date" # title = "title" # name = "name" - maql = "" + maql = '' maql += "INCLUDE TEMPLATE \"#{@urn}\" MODIFY (IDENTIFIER \"#{@name}\", TITLE \"#{@title}\");" maql end - end - end end \ No newline at end of file