test/integration_tests.rb in crowdflower-0.11.0 vs test/integration_tests.rb in crowdflower-0.12.0

- old
+ new

@@ -1,13 +1,14 @@ $: << File.dirname(__FILE__) + "/../lib" -require 'rubygems' require 'crowdflower' require 'json' -API_KEY = ENV["API_KEY"] -DOMAIN_BASE = ENV["DOMAIN_BASE"] || "https://api.localdev.crowdflower.com:8443" +CONFIG = YAML::load(File.read("config/account_info.yml")) +API_KEY = CONFIG["API_KEY"] +PROJECT_NUM = CONFIG["PROJECT_NUM"] +DOMAIN_BASE = "https://api.crowdflower.com" || "https://api.localdev.crowdflower.com:8443" CrowdFlower::Job.connect! API_KEY, DOMAIN_BASE unless API_KEY && API_KEY.size > 3 puts <<EOF @@ -21,18 +22,14 @@ EOF exit 1 end -# If you turn this on, tasks will be posted on CrowdFlower and your -# account will be charged. This is inadvisable for anyone other than -# CrowdFlower employees. -I_AM_RICH = ENV["CF_LIVE_TRANSACTIONS"] == "true" +RUNNING_TESTS = ENV["CF_LIVE_TRANSACTIONS"] == "true" -if I_AM_RICH - puts "*** LIVE TRANSACTIONS ENABLED - THIS TEST RUN WILL BE CHARGED ***" - puts +if RUNNING_TESTS + puts "*** LIVE TRANSACTIONS ENABLED - THIS TEST RUN WILL BE CHARGED TO YOUR ACCOUNT ***" end def wait_until 10.times do if yield @@ -60,113 +57,255 @@ def say(msg) $stdout.puts msg end +################################################# +# API CONNECTION +################################################# +say "Connecting to the API" +CrowdFlower.connect! API_KEY, DOMAIN_BASE -say "defining multiple api keys" -(job_subclass_with_valid_custom_key = Class.new(CrowdFlower::Job)).connect! API_KEY, DOMAIN_BASE -(job_subclass_with_invalid_custom_key = Class.new(CrowdFlower::Job)).connect! 'invalid api key', DOMAIN_BASE -job_subclass_with_no_custom_key = Class.new(CrowdFlower::Job) +################################################# +# CREATE JOB +################################################# +say "Creating a blank job." +job_1 = CrowdFlower::Job.create("Job_1: Connection Check") +p "job_1 id: #{job_1.get["id"]}" +p "job_1 units_count: #{job_1.get["units_count"]}" -say "no default api key" -assert_exception_raised(CrowdFlower::UsageError) {CrowdFlower::Job.create("job creation should fail")} -assert_exception_raised(CrowdFlower::UsageError) {job_subclass_with_no_custom_key.create("job creation should fail")} -assert_exception_raised(CrowdFlower::APIError) {job_subclass_with_invalid_custom_key.create("job creation should fail")} -assert job_subclass_with_valid_custom_key.create("should be ok").units.ping['count'] +################################################# +# CHECK/ ENABLE CHANNELS +################################################# +say "Checking that job_1 does not have any enabled channels." +assert job_1.channels["enabled_channels"].empty? -say "invalid default api key" -CrowdFlower.connect! "invalid default api key", DOMAIN_BASE -assert_exception_raised(CrowdFlower::APIError) {CrowdFlower::Job.create("job creation should fail")} -assert_exception_raised(CrowdFlower::APIError) {job_subclass_with_no_custom_key.create("job creation should fail")} -assert_exception_raised(CrowdFlower::APIError) {job_subclass_with_invalid_custom_key.create("job creation should fail")} -assert job_subclass_with_valid_custom_key.create("should be ok").units.ping['count'] +say "Enabling the cf_internal channel." +job_1.enable_channels("cf_internal") +assert job_1.channels["enabled_channels"] == ["cf_internal"] -say "Connecting to the API" -CrowdFlower.connect! API_KEY, DOMAIN_BASE +################################################# +# UPLOAD DATA/ CREATE NEW JOB +################################################# +say "Uploading CSV to create job_2." +job_2 = CrowdFlower::Job.upload(File.dirname(__FILE__) + "/crowdshopping.csv", "text/csv") +job_2_id = job_2.get["id"] +p "job_2 id: #{job_2_id}" -assert CrowdFlower::Job.create("should be ok").units.ping['count'] -assert job_subclass_with_no_custom_key.create("should be ok").units.ping['count'] -assert job_subclass_with_valid_custom_key.create("should be ok").units.ping['count'] -assert_exception_raised(CrowdFlower::APIError) {job_subclass_with_invalid_custom_key.create("job creation should fail")} -# Add this test to check your URL -#assert CrowdFlower::Base.connection.public_url == "localdev.crowdflower.com:80" +################################################# +# ADD UNITS +################################################# +say "-- Waiting for CrowdFlower to process the data." +wait_until { job_2.get["units_count"] == 6 } -say "Uploading a test CSV" -job = CrowdFlower::Job.upload(File.dirname(__FILE__) + "/sample.csv", "text/csv") +say "Adding some more data." +job_2.upload(File.dirname(__FILE__) + "/crowdshopping.csv", "text/csv") -say "Trying to get all jobs" -assert CrowdFlower::Job.all.first["id"] == job.id +say "-- Waiting for CrowdFlower to process the data." +wait_until { job_2.get["units_count"] == 12 } -say "-- Waiting for CrowdFlower to process the data" -wait_until { job.get["units_count"] == 4 } +################################################# +# PING UNITS +################################################# +say "Pinging job_2 units." +assert job_2.units.ping['count'] == 12 +assert job_2.units.ping['done'] == true -say "Adding some more data" -job.upload(File.dirname(__FILE__) + "/sample.csv", "text/csv") +################################################# +# COUNT UNITS +################################################# +say "Checking for 12 units in job_2." +assert job_2.units.all.size == 12 -say "-- Waiting for CrowdFlower to process the data" -# You could also register a webhook to have CrowdFlower notify your -# server. -wait_until { job.get["units_count"] == 8 } +################################################# +# UPDATE JOB_2 +################################################# +say "Adding title, project number, instructions, CML" +job_2.update({:title => 'Job_2: CrowdShopping', + :project_number => PROJECT_NUM, + :instructions => '<p>There are six questions to this task. In this order, the questions ask if you were able to find a pair of Lita shoes for sale in red glitter, gold glitter, multi glitter, silver glitter, black glitter, or other color of glitter.</p><p>There is a photo of the shoe in correlating color as the question right below the question. It will give you a better idea of what to look for.</p>', + :cml => '<cml:radios label="Were you able to find an online retailer selling Jeffery Campbell Lita Booties in {{glitter_color}}?" validates="required" name="color_found" instructions="If you found the shoes we are looking for, click yes to fill in the website url."><p class="shoe-img">Example Photo: <img src="{{image}}" width="100" /></p><cml:radio label="Yes, I found an online retailer selling Lita shoes in {{glitter_color}}." value="yes"></cml:radio><cml:radio label="No, I could not find an online retailer selling Lita shoes in {{glitter_color}}." value="no"></cml:radio></cml:radios><br /><cml:text label="Please enter the name of the online retailer." default="Example: Karmaloop" validates="required" only-if="color_found:[yes]" name="please_enter_the_name_of_the_online_retailer"></cml:text><br /><cml:text label="Please enter the url to the shoes you found." default="Example: www.karmaloop.com/jeffery-campbell-litas-multiglitter" validates="required url" only-if="color_found:[yes]" name="please_enter_the_url_to_the_shoes_you_found"></cml:text>'}) -say "Checking ping." -assert job.units.ping['count'] == 8 -assert job.units.ping['done'] == true +################################################# +# ADD/ UPDATE/ REMOVE TAGS +################################################# +say "Checking if tags exist." +assert job_2.tags.empty? -say "Getting the units for this job." -assert job.units.all.size == 8 +say "Adding 'shoes' and 'glitter' to tags." +job_2.add_tags ["shoes", "glitter"] +assert job_2.tags.map{|t| t["name"]}.sort == ["glitter", "shoes"] -say "Checking the status of the job." -assert job.status["tainted_judgments"] == 0 +say "Removing 'shoes' tag." +job_2.remove_tags ["shoes"] +assert job_2.tags.map{|t| t["name"]} == ["glitter"] -say "Adding title, instructions, and problem to the job." -job.update({:title => 'testtt', - :instructions => 'testttt fdsf sfds fsdfs fesfsdf', - :cml => '<cml:text label="Text" class="unmodified" validates="required"/>'}) +say "Updating tags to 'fashion' 'fun' and 'glitter'." +job_2.update_tags ["fashion", "fun", "glitter"] +assert job_2.tags.map{|t| t["name"]} == ["fashion", "fun", "glitter"] -say "Registering a webhook." -job.update :webhook_uri => "http://localhost:8080/crowdflower" +################################################# +# CHECK JOB_2 CHANNELS +################################################# +say "Checking that channels are turned on." +assert !job_2.channels["enabled_channels"].empty? +p "job_2 enabled_channels: #{job_2.channels["enabled_channels"]}" -say "Tags" -assert job.tags.empty? -job.update_tags ["testing_123", "testing_456"] -assert job.tags.map{|t| t["name"]}.sort == ["testing_123", "testing_456"] -job.remove_tags ["testing_123"] -assert job.tags.map{|t| t["name"]} == ["testing_456"] -job.add_tags ["testing_789"] -assert job.tags.map{|t| t["name"]} == ["testing_456", "testing_789"] +################################################# +# ORDER JOB +################################################# +say "Ordering (launching) job_2 with 12 units." +order = CrowdFlower::Order.new(job_2) +order.debit(12, ["4x4bux_com", "bitcoinget", "cf_internal"]) +wait_until { job_2.get["state"].casecmp('running') == 0} +# turning channels from on_demand work force adds "on_demand" to enabled_channels +assert job_2.channels["enabled_channels"] == ["4x4bux_com", "bitcoinget", "cf_internal", "on_demand"] -say "Copying the existing job to a new one." -job2 = job.copy :all_units => true +################################################# +# DISABLE A CHANNEL +################################################# +say "Disabling 'bitcoinget' channel." +job_2.disable_channel("bitcoinget") +assert job_2.channels["enabled_channels"] == ["4x4bux_com", "cf_internal", "on_demand"] -say "-- Waiting for CrowdFlower to finish copying the job." -# You could also register a webhook to have CrowdFlower notify your -# server. -wait_until { job2.get["units_count"] == 8 } +################################################# +# UNIT METHODS +################################################# +say "Setting up units." +unit_1 = job_2.units.all.to_a[0][0] +unit_2 = job_2.units.all.to_a[1][0] -say "Ordering the job." -order = CrowdFlower::Order.new(job) -unit_count = 8 -order.debit(8) -wait_until { job.get["state"].casecmp('running') == 0} +unit = CrowdFlower::Unit.new(job_2) +wait_until { unit.get(unit_1)["state"] == "judgable" } +p "unit_1 id: #{unit_1}" +wait_until { unit.get(unit_1)["state"] == "judgable" } +p "unit_2 id: #{unit_2}" -say "Checking enabled_channels." -assert !job.channels["enabled_channels"].empty? -# To see exact list of enabled jobs: -# p job.channels["enabled_channels"] +say "Making unit_1 a test question (gold)." +unit.make_gold(unit_1) +assert unit.get(unit_1)["state"] == "golden" -say "Order job2 to check channels when specifying channel parameter." -order = CrowdFlower::Order.new(job2) -order.debit(8, "channel"=>"cf_internal") -assert job2.channels["enabled_channels"] == ["cf_internal"] +say "Copying unit_2." +unit.copy(unit_2, job_2_id, "glitter_color"=>"blue") +assert job_2.get["units_count"] == 13 -say "Canceling the unit." -unit_id = job.units.all.to_a[0][0] +say "Creating a new test question unit." +unit.create({"glitter_color"=>"white"}, true) +assert job_2.get["units_count"] == 14 +assert job_2.get["golds_count"] == 2 + +say "Canceling unit_2." +unit.cancel(unit_2) +assert unit.get(unit_2)["state"] == "canceled" + +################################################# +# PAUSE/ RESUME/ CANCEL JOB +################################################# +say "Pausing job_2." +job_2.pause +assert job_2.get["state"] == "paused" +p "job_2 state: #{job_2.get["state"]}" + +say "Resuming job_2." +job_2.resume +assert job_2.get["state"] == "running" +p "job_2 state: #{job_2.get["state"]}" + +say "Canceling job_2." +job_2.cancel +assert job_2.get["state"] == "canceled" +p "job_2 state: #{job_2.get["state"]}" + +say "Deleting job_1." +job_1.delete +assert job_1.get["state"] == "unordered" + +################################################# +# JOB LEGEND +################################################# +say "Checking job_2 legend." +assert !job_2.legend.empty? + +################################################# +# COPY JOB_2 +################################################# +say "Copying job_2." +job_3 = job_2.copy(:all_units => true) + +say "Updating job_3 title." +job_3.update(:title => 'Job_3: Copy of Job_2') +assert job_3.get["title"] == "Job_3: Copy of Job_2" + +say "-- Waiting for CrowdFlower to process the data." +wait_until { job_3.get["units_count"] == 14 } +assert job_3.get["units_count"] == 14 + +################################################# +# WORKER METHODS +################################################# +# Worker tests run against an existing job with an internal CF worker_id number +say "Starting Worker tests for job #422830." +job = CrowdFlower::Job.new(422830) +worker = CrowdFlower::Worker.new(job) +worker_id = 23542619 + +say "Notifying worker." +worker.notify(worker_id, "Testing notify method.") + +say "Bonusing a worker." +worker.bonus(worker_id, 1, "Here's a bonus for your awesome work!") + +say "Flagging worker from one of my jobs." +worker.flag(worker_id, "Testing flag method.", :persist => false) + +say "Flagging worker from all my jobs." +worker.flag(worker_id, "Testing flag method across all jobs.", :persist => true) + +say "Deflagging worker." +worker.deflag(worker_id, "Testing deflag method.") + +################################################# +# JUDGMENT METHODS +################################################# +# Judgment tests run against the same job as Worker tests +say "Starting Judgment tests for job #422830." +judgment = CrowdFlower::Judgment.new(job) unit = CrowdFlower::Unit.new(job) -wait_until { unit.get(unit_id)['state'] == 'judgable' } -puts unit.cancel(unit_id).inspect -assert unit.get(unit_id)['state'] == 'canceled' -say "Webhook test needs to be written." -#job.test_webhook +say "Getting all judgments." +assert judgment.all.count == job.get["judgments_count"] +say "Checking the judgment number to test judgment.get method." +assert judgment.get(1243027889)["judgment"] == 1 + +say "Checking the number of judgments for a unit to see if a unit's judgment info can be returned." +assert job.units.judgments(447664267).count == unit.get(447664267)["judgments_count"] + +################################################# +# DOWNLOAD REPORTS - missing assetions +################################################# +# These run against the completed job from readme examples; they print status codes as they go +say "Startiing report tests for job #418404." +job = CrowdFlower::Job.new(418404) + +say "Downloading Full CSV" +job.download_csv(:full, "full_report.zip") + +say "Downloading Aggregated CSV" +job.download_csv(:aggregated, "agg_report.zip") + +say "Downloading Source CSV" +job.download_csv(:source, "source_report.zip") + +say "Downloading Test Questions CSV" +job.download_csv(:gold_report, "gold_report.zip") + +say "Downloading Worker CSV" +job.download_csv(:workset, "workset_report.zip") + +say "Downloading JSON" +job.download_csv(:json, "json_report.zip") + +################################################# +# END OF TESTS +################################################# say ">-< Tests complete. >-<"