require 'rest_client' module Scrivito class ContentServiceObjQueries def initialize(queries) @queries = queries @open_queries = queries.dup @results = {} end def open_queries @open_queries[0..99] end def handle_response(response) objs = {} response["objs"].each do |obj| objs[obj["_id"].first] = obj end queries_to_delete = [] response["results"].each_with_index do |response, i| query = @open_queries[i] if response["continuation_handle"] query[:continuation_handle] = response["continuation_handle"] else queries_to_delete << i end result = (@results[query.__id__] ||= []) response["refs"].each do |obj_ref| id = obj_ref["id"] # TODO fetch missing ObjData from Service result << (objs[id] or raise "Data for Obj with id #{id} missing!") end end queries_to_delete.reverse_each {|i| @open_queries.delete_at(i) } end def results @queries.map {|query| @results[query.__id__] || [] } end def finished? open_queries.empty? end end class CmsBackend BLOB_DATA_CACHE_PREFIX = 'blob_data'.freeze VALID_INDEX_NAMES = %w[id path ppath permalink].freeze class << self def instance @instance ||= new end end def initialize @query_counter = 0 end def begin_caching @caching = true end def end_caching CmsCacheStorage.cache.clear @caching = false end def caching? !!@caching end def query_counter @query_counter end def find_workspace_data_by_id(id) workspace_data_from_cache = WorkspaceDataFromService.find_from_cache(id) from_content_state_id = workspace_data_from_cache.try(:content_state_id) request_params = {:workspace_id => id} request_params[:content_state_id] = from_content_state_id if from_content_state_id raw_data = if id == 'published' && workspace_data_from_cache begin ContentService.query('workspaces/query', request_params, timeout: 1) rescue CommunicationError => e warn_backend_not_available(id, from_content_state_id, e.message) return workspace_data_from_cache end else ContentService.query('workspaces/query', request_params) end if raw_workspace_data = raw_data['workspace'] workspace_data = WorkspaceDataFromService.new(raw_workspace_data) workspace_data.store_in_cache if from_content_state_id != workspace_data.content_state_id workspace_data end end def find_obj_data_by(revision, index, keys) find_obj_data_filtering_deleted_by(revision, index, keys, false) end def find_obj_data_including_deleted_by(revision, index, keys) find_obj_data_filtering_deleted_by(revision, index, keys, true) end def find_blob_data_by_id(id) cache_key = "#{BLOB_DATA_CACHE_PREFIX}/#{id}" if data_from_cache = CmsCacheStorage.cache.read(cache_key) data_from_cache else data_from_database = find_blob_data_from_database_by(id) if maxage = data_from_database['maxage'] CmsCacheStorage.cache.write(cache_key, data_from_database, :expires_in => maxage) end data_from_database end end def search_objs(workspace, params) content_state = workspace.revision.content_state cache_index = 'search' cache_key = params.to_param if result = fetch_search_result_from_cache(content_state, cache_index, cache_key) result else request_search_result_from_backend(workspace, params).tap do |result| store_search_result_in_cache(content_state, cache_index, cache_key, result) end end end def find_obj_class_data_by_name(revision, name) find_all_obj_class_data(revision).find { |obj_class_data| obj_class_data.name == name } end def find_all_obj_class_data(revision) content_state = revision.content_state if obj_classes_data = fetch_obj_classes_data_from_cache(content_state) obj_classes_data else request_obj_classes_data_from_backend(revision).tap do |obj_classes_data| store_obj_classes_data_in_cache(content_state, obj_classes_data) end end end private def fetch_obj_classes_data_from_cache(content_state) ContentStateCaching.find_obj_classes_data(content_state) if caching? end def request_obj_classes_data_from_backend(revision) response = CmsRestApi.get("revisions/#{revision.id}/obj_classes", include_inactive: true) response['results'].map { |raw_data| ObjClassData.new(raw_data) } end def store_obj_classes_data_in_cache(content_state, obj_classes_data) ContentStateCaching.store_obj_classes_data(content_state, obj_classes_data) if caching? end def fetch_search_result_from_cache(content_state, cache_index, cache_key) content_state.find_obj_data(cache_index, cache_key) if caching? end def request_search_result_from_backend(workspace, params) CmsRestApi.get("workspaces/#{workspace.id}/objs/search", params) end def store_search_result_in_cache(content_state, cache_index, cache_key, result) content_state.save_obj_data(cache_index, cache_key, result) if caching? end def find_obj_data_filtering_deleted_by(revision, index, keys, include_deleted) index = index.to_s assert_valid_index_name(index) raw_data = find_raw_data_from_cache_or_database_by(revision, index, keys) raw_data.map do |raw_result| raw_result.each_with_object([]) do |raw_data, result| next if raw_data['_modification'] == ['deleted'] && !include_deleted result << ObjDataFromService.new(raw_data) end end end def find_raw_data_from_cache_or_database_by(revision, index, keys) keys_from_database = [] # load results from cache results_from_cache = keys.map do |key| find_raw_data_from_cache_by(revision, index, key).tap do |objs| keys_from_database << key unless objs end end # load cache misses from database and store them in cache results_from_database = find_raw_data_from_database_by(revision, index, keys_from_database) keys_from_database.each_with_index do |key, key_number| store_raw_data_list_in_cache(revision, index, key, results_from_database[key_number]) end # combine the results results_from_cache.map do |objs_from_cache| objs_from_cache || results_from_database.shift end end def find_raw_data_from_cache_by(revision, index, key) ContentStateCaching.find_obj_data(revision.content_state, index, key) if caching? end def find_raw_data_from_database_by(revision, index, keys) return [] if keys.blank? instrumenter = ActiveSupport::Notifications.instrumenter instrumenter.instrument( "cms_load.scrivito", :name => "Obj Load", :index => index, :keys => keys ) do @query_counter += 1 queries = ContentServiceObjQueries.new(keys.map {|key| {:type => index, :param => key} }) queries.handle_response(request_content_service(queries, revision)) until queries.finished? queries.results end end def request_content_service(queries, revision) ContentService.query('objs/query', content_service_request_params(queries, revision)) end def content_service_request_params(queries, revision) params = { queries: queries.open_queries, revision_id: revision.id, include_deleted: true } # Not every revision has a workspace, # e.g. when requesting with the base revision of the current workspace. params[:workspace_id] = revision.workspace.id if revision.workspace params end UNIQUE_INDICES = [:id, :path, :permalink].freeze def store_raw_data_list_in_cache(revision, index, key, raw_data_list) raw_data_list.each do |values| UNIQUE_INDICES.each do |unique_index| unique_index_values = values["_#{unique_index}"] if unique_index_values.present? store_item_in_cache(revision, unique_index, unique_index_values.first, [values]) end end end unless UNIQUE_INDICES.include?(index) store_item_in_cache(revision, index, key, raw_data_list) end end def store_item_in_cache(revision, index, key, item) ContentStateCaching.store_obj_data(revision.content_state, index, key, item) end def find_blob_data_from_database_by(id) @query_counter += 1 ContentService.query('blobs/query', :blob_ids => [id])['blobs'][id] end def assert_valid_index_name(index) raise ArgumentError, "invalid index name '#{index}'" unless VALID_INDEX_NAMES.include?(index) end def warn_backend_not_available(workspace_id, content_state_id, error_message) message = <<-EOS Couldn't connect to content service for workspace with id=#{workspace_id} and content_state_id=#{content_state_id}. #{error_message} Serving from cache. EOS Rails.logger.warn(message) end end end