test_runner.rb in kaba-0.4.0

- old
+ new

@@ -24,23 +24,25 @@
   def scan(
     limit: nil, 
     model: 'spark-general-4.0', 
     judge_model: 'spark-general-4.0',  
     judge_temperature: 0.1, 
-    temperature: 0.1
+    temperature: 0.1,
+    semaphore_limit: 5
     )
 
     progressbar = TTY::ProgressBar.new(
       "Test: [:bar] :percent :current/:total",
        total: @test_files.first(limit || @test_files.size).size
     )
 
     progressbar.start
 
     Async do
+      semaphore = Async::Semaphore.new(semaphore_limit)
       _each(limit: limit) do |row|
-        Async do |task|
+        semaphore.async do |task|
           input = @prompt.render(File.read row.input_file)
 
           target = <<~Markdown
           ```json
           #{JSON.pretty_generate(JSON.parse(File.read(row.target_path)))}
@@ -62,10 +64,10 @@
 
           type_check_response = JSON.parse @validate.run(output_json).body
           @type_right_total += 1 if type_check_response["success"]
           
           judge_input = Judge.new(input: input, output: output, target: target).render
-          judge_response = Application.llm_client.chat(
+          judge_response = Application.judge_llm_client.chat(
             parameters: {
               model: judge_model,
               messages: [ { role: 'user', content: judge_input } ],
               temperature: judge_temperature,
             }