require "jiralicious" require "parallel" require "limiter" require "embulk/input/jira_api/issue" require "timeout" module Embulk module Input module JiraApi class Client MAX_RATE_LIMIT = 50 MIN_RATE_LIMIT = 2 # Normal http request timeout is 300s SEARCH_ISSUES_TIMEOUT_SECONDS = 300 DEFAULT_SEARCH_RETRY_TIMES = 10 def initialize @rate_limiter = Limiter::RateQueue.new(MAX_RATE_LIMIT, interval: 2) end def self.setup(&block) Jiralicious.configure(&block) new end def search_issues(jql, options={}) issues_raw = search(jql, options).issues_raw # Maximum number of issues to retrieve is 50 rate_limit = MAX_RATE_LIMIT success_items = [] fail_items = [] error_object = nil timeout_and_retry(SEARCH_ISSUES_TIMEOUT_SECONDS * MAX_RATE_LIMIT ) do retry_count = 0 semaphore = Mutex.new @rate_limiter = Limiter::RateQueue.new(rate_limit, interval: 2) error_object = nil while issues_raw.length > 0 && retry_count <= DEFAULT_SEARCH_RETRY_TIMES do Parallel.each(issues_raw, in_threads: rate_limit) do |issue_raw| # https://github.com/dorack/jiralicious/blob/v0.4.0/lib/jiralicious/search_result.rb#L32-34 begin issue = find_issue(issue_raw["key"]) semaphore.synchronize { success_items.push(JiraApi::Issue.new(issue)) } rescue MultiJson::ParseError => e html = e.message title = html[%r|(.*?)|, 1] # 401 due to high number of concurrent requests with current account # The number of concurrent requests is not fixed by every account # Hence catch the error item and retry later raise title if title != "Unauthorized (401)" semaphore.synchronize { fail_items.push(issue_raw) error_object = e } end end retry_count += 1 rate_limit = calculate_rate_limit(rate_limit, issues_raw.length, fail_items.length, retry_count) issues_raw = fail_items fail_items = [] raise error_object if retry_count > DEFAULT_SEARCH_RETRY_TIMES && !error_object.nil? # Sleep after some seconds for JIRA API perhaps under the overload sleep retry_count if fail_items.length > 0 end success_items end end def search(jql, options={}) timeout_and_retry(SEARCH_ISSUES_TIMEOUT_SECONDS) do Jiralicious.search(jql, options) end end def total_count(jql) search(jql, max_results: 1).num_results end def check_user_credential(username) Jiralicious::User.search(username) rescue Jiralicious::JqlError, Jiralicious::AuthenticationError, Jiralicious::NotLoggedIn, Jiralicious::InvalidLogin => e raise Embulk::ConfigError.new(e.message) rescue ::SocketError => e # wrong `uri` option given raise Embulk::ConfigError.new(e.message) rescue MultiJson::ParseError => e html = e.message title = html[%r|(.*?)|, 1] #=> e.g. "Unauthorized (401)" raise ConfigError.new("Can not authorize with your credential.") if title == 'Unauthorized (401)' end # Calculate rate limit based on previous run result # Return 2 MIN_RATE_LIMIT in case turning from the 5th times or success_items is less than 2 # Otherwise return the min number between fail_items, success_items and current_limit def calculate_rate_limit(current_limit, all_items, fail_items, times) success_items = all_items - fail_items return MIN_RATE_LIMIT if times >= DEFAULT_SEARCH_RETRY_TIMES/2 || success_items < MIN_RATE_LIMIT return [fail_items, success_items, current_limit].min end private def timeout_and_retry(wait, retry_times = DEFAULT_SEARCH_RETRY_TIMES, &block) count = 0 begin Timeout.timeout(wait) do yield end rescue Jiralicious::JqlError, Jiralicious::AuthenticationError, Jiralicious::NotLoggedIn, Jiralicious::InvalidLogin => e raise Embulk::ConfigError.new(e.message) rescue ::SocketError => e # wrong `uri` option given raise Embulk::ConfigError.new(e.message) rescue MultiJson::ParseError => e # same as this Mailchimp plugin issue: https://github.com/treasure-data/embulk-output-mailchimp/issues/10 # (a) JIRA returns error as HTML, but HTTParty try to parse it as JSON. # And (b) `search_issues` method has race-condition bug. If it occurred, MultiJson::ParseError raised too. html = e.message title = html[%r|(.*?)|, 1] #=> e.g. "Unauthorized (401)" raise title if title == "Atlassian Cloud Notifications - Page Unavailable" count += 1 raise title.nil? ? "Unknown Error" : title if count > retry_times Embulk.logger.warn "JIRA returns error: #{title == 'Unauthorized (401)' ? title + " due to overloading API requests. Retrying on failed items only" : title}." sleep count retry rescue Timeout::Error => e count += 1 raise e if count > retry_times Embulk.logger.warn "Time out error." sleep count # retry after some seconds for JIRA API perhaps under the overload retry end end def find_issue(issue_key) @rate_limiter.shift Jiralicious::Issue.find(issue_key) end end end end end