require 'rest_client' module RailsConnector class ContentServiceObjQueries def initialize(queries) @queries = queries @open_queries = queries.dup @results = {} end def open_queries @open_queries[0..99] end def handle_response(response) objs = {} response["objs"].each do |obj| objs[obj["_id"].first] = obj end queries_to_delete = [] response["results"].each_with_index do |response, i| query = @open_queries[i] if response["continuation_handle"] query[:continuation_handle] = response["continuation_handle"] else queries_to_delete << i end result = (@results[query.__id__] ||= []) response["refs"].each do |obj_ref| id = obj_ref["id"] # TODO fetch missing ObjData from Service result << (objs[id] or raise "Data for Obj with id #{id} missing!") end end queries_to_delete.reverse_each {|i| @open_queries.delete_at(i) } end def results @queries.map {|query| @results[query.__id__] || [] } end def finished? open_queries.empty? end end # connects the cloud connector to the connector service class ServiceCmsBackend < CmsBackend VERSION = "v1".freeze BLOB_DATA_CACHE_PREFIX = 'blob_data'.freeze def initialize @query_counter = 0 end def begin_caching @editable_cache = Configuration.cache_editable_workspaces ? persistent_cache : Cache.new @read_only_cache = persistent_cache @blob_data_cache = persistent_cache end def end_caching @editable_cache = @read_only_cache = @blob_data_cache = nil end def caching? @read_only_cache && @editable_cache && @blob_data_cache end def query_counter @query_counter end def find_workspace_data_by_id(id) raw_data = ContentService.query( "workspaces/query", :workspace_id => id ) if raw_workspace_data = raw_data['workspace'] WorkspaceDataFromService.new raw_workspace_data end end def find_obj_data_by(workspace_data, index, keys) raw_data = if caching? find_raw_data_from_cache_or_database_by(workspace_data, index, keys) else find_raw_data_from_database_by(workspace_data, index, keys) end raw_data.map do |raw_list| raw_list.map do |raw_obj_data| ObjDataFromService.new(raw_obj_data) end end end def find_blob_data_by_id(id) if caching? find_blob_data_from_cache_or_database_by_id(id) else find_blob_data_from_database_by(id) end end private def encoding_aware_cache_prefix prefix = VERSION.dup prefix.concat('-utf8') if String.new.encoding_aware? prefix end def persistent_cache Cache.new(:fallback_backend => Rails.cache, :cache_prefix => encoding_aware_cache_prefix) end def find_raw_data_from_cache_or_database_by(workspace_data, index, keys) keys_from_database = [] # load results from cache results_from_cache = keys.map do |key| cache_for(workspace_data).read(cache_key_for(workspace_data, index, key)).tap do |objs| keys_from_database << key unless objs end end # load cache misses from database and store them in cache results_from_database = find_raw_data_from_database_by(workspace_data, index, keys_from_database) keys_from_database.each_with_index do |key, key_number| store_raw_data_list_in_cache(workspace_data, index, key, results_from_database[key_number]) end # combine the results results_from_cache.map do |objs_from_cache| objs_from_cache || results_from_database.shift end end def find_raw_data_from_database_by(workspace_data, index, keys) return [] if keys.blank? instrumenter = ActiveSupport::Notifications.instrumenter instrumenter.instrument( "cms_load.rails_connector", :name => "Obj Load", :index => index, :keys => keys ) do @query_counter += 1 queries = ContentServiceObjQueries.new(keys.map {|key| {:type => index, :param => key} }) until queries.finished? queries.handle_response(ContentService.query( "objs/query", :queries => queries.open_queries, :workspace_id => workspace_data.id, :revision_id => workspace_data.revision_id )) end queries.results end end UNIQUE_INDICES = [:id, :path, :permalink].freeze def store_raw_data_list_in_cache(workspace_data, index, key, raw_data_list) raw_data_list.each do |values| UNIQUE_INDICES.each do |unique_index| unique_index_values = values["_#{unique_index}"] if unique_index_values.present? store_item_in_cache(workspace_data, unique_index, unique_index_values.first, [values]) end end end unless UNIQUE_INDICES.include?(index) store_item_in_cache(workspace_data, index, key, raw_data_list) end end def store_item_in_cache(workspace_data, index, key, item) cache_for(workspace_data).write(cache_key_for(workspace_data, index, key), item) if caching? end def cache_key_for(workspace_data, index, key) "rev/#{workspace_data.revision_id}/obj/#{index}/#{key}" end def cache_for(workspace_data) workspace_data.cachable? ? @read_only_cache : @editable_cache end def find_blob_data_from_cache_or_database_by_id(id) cache_key = "#{BLOB_DATA_CACHE_PREFIX}/#{id}" if data_from_cache = @blob_data_cache.read(cache_key) data_from_cache else data_from_database = find_blob_data_from_database_by(id) if maxage = data_from_database['maxage'] @blob_data_cache.write(cache_key, data_from_database, :expires_in => maxage) end data_from_database end end def find_blob_data_from_database_by(id) @query_counter += 1 ContentService.query('blobs/query', :blob_ids => [id])['blobs'][id] end end end