data_fetcher.rb in relaton-iso-1.19.0

- old
+ new

@@ -93,27 +93,26 @@
     def get_redirection(path) # rubocop:disable Metrics/MethodLength
       try = 0
       uri = URI(Scrapper::DOMAIN + path)
       begin
         get_response uri
-      rescue Net::OpenTimeout, Net::ReadTimeout => e
+      rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED => e
         try += 1
         retry if check_try try, uri
 
-        warn "Error fetching #{uri}"
-        warn e.message
+        Util.error "Error fetching #{uri}, #{e.message}"
       end
     end
 
     def get_response(uri)
       resp = Net::HTTP.get_response(uri)
       resp.code == "302" ? get_redirection(resp["location"]) : resp
     end
 
     def check_try(try, uri)
       if try < 3
-        warn "Timeout fetching #{uri}, retrying..."
+        Util.warn "Timeout fetching #{uri}, retrying..."
         sleep 1
         true
       end
     end
 
@@ -135,13 +134,12 @@
       # path = docpath.sub(/\.html$/, "")
       # hit = Hit.new({ path: docpath }, nil)
       doc = Scrapper.parse_page docpath
       @mutex.synchronize { save_doc doc, docpath }
     rescue StandardError => e
-      warn "Error fetching document: #{Scrapper::DOMAIN}#{docpath}"
-      warn e.message
-      warn e.backtrace
+      Util.error "Error fetching document: #{Scrapper::DOMAIN}#{docpath}\n" \
+        "#{e.message}\n#{e.backtrace}"
     end
 
     # def compare_docids(id1, id2)
     #   Pubid::Iso::Identifier.create(**id1).to_s <=> Pubid::Iso::Identifier.create(**id2).to_s
     # end
@@ -156,10 +154,10 @@
     def save_doc(doc, docpath) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
       docid = doc.docidentifier.detect(&:primary)
       file_name = docid.id.gsub(/[\s\/:]+/, "-").downcase
       file = File.join @output, "#{file_name}.#{@ext}"
       if @files.include? file
-        warn "Duplicate file #{file} for #{docid.id} from #{Scrapper::DOMAIN}#{docpath}"
+        Util.warn "Duplicate file #{file} for #{docid.id} from #{Scrapper::DOMAIN}#{docpath}"
       else
         @files << file
         index.add_or_update docid.to_h, file
         File.write file, serialize(doc), encoding: "UTF-8"
       end