Sha256: ea72917a35e688183c7c3b6603a43233dbd49343f071e429b2c1ad022c317e06
Contents?: true
Size: 1.58 KB
Versions: 6
Compression:
Stored size: 1.58 KB
Contents
#!/usr/bin/env ruby
require "concurrent"
require "stepmod/utils/stepmod_file_annotator"
stepmod_dir = ARGV.first || Dir.pwd
# build resource_docs cache
resource_docs_cache_file = `mktemp`
`"#{File.join(__dir__,
"stepmod-build-resource-docs-cache")}" "#{stepmod_dir}" > "#{resource_docs_cache_file}"`
# annotate each file
files = `"#{File.join(__dir__,
"stepmod-find-express-files")}" "#{stepmod_dir}"`.strip.split("\n")
MAX_THREADS = [2, Concurrent.processor_count].max * 2
MAX_QUEUE_SIZE = MAX_THREADS * 4
# https://github.com/ruby-concurrency/concurrent-ruby/blob/master/docs-source/thread_pools.md
pool = Concurrent::ThreadPoolExecutor.new(
min_threads: MAX_THREADS,
max_threads: MAX_THREADS,
max_queue: MAX_QUEUE_SIZE,
fallback_policy: :caller_runs,
)
files.each_slice(MAX_QUEUE_SIZE) do |batch|
puts("Queueing next batch")
batch.each do |file|
pool.post do
puts("#{Thread.current.object_id}: Queued processing #{file}")
annotated = Stepmod::Utils::StepmodFileAnnotator.new(
express_file: file,
resource_docs_cache_file: resource_docs_cache_file,
stepmod_dir: stepmod_dir
).call
annotated_file_name = "#{File.basename(file, '.exp')}_annotated.exp"
annotated_file_path = File.join(File.dirname(file), annotated_file_name)
File.open(annotated_file_path, "w") do |file|
file.puts(annotated)
end
puts("#{Thread.current.object_id}: Done processing #{File.basename(file)} => #{annotated_file_path}.")
end
end
pool.shutdown
end
# cleanup
`rm "#{resource_docs_cache_file}"`
Version data entries
6 entries across 6 versions & 1 rubygems