.default-rules: rules: - if: '$CI_COMMIT_TAG || $RELEASE' when: never - if: '$RELEASE == null && $CI_JOB_NAME =~ /staging/' when: manual - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_REF_NAME == "master"' image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-ruby-2.6 stages: - check - release - test - notify variables: DOCKER_DRIVER: overlay2 DOCKER_HOST: tcp://docker:2375 QA_ARTIFACTS_DIR: $CI_PROJECT_DIR QA_CAN_TEST_GIT_PROTOCOL_V2: "true" QA_TESTCASES_REPORTING_PROJECT: "gitlab-org/quality/testcases" cache: key: "ruby:2.6" paths: - vendor/ruby before_script: - bundle version - bundle install --clean --jobs=$(nproc) --path=vendor --retry=3 --quiet && bundle check - if [ -n "$TRIGGERED_USER" ] && [ -n "$TRIGGER_SOURCE" ]; then echo "Pipeline triggered by $TRIGGERED_USER at $TRIGGER_SOURCE"; fi - export LANG=C.UTF-8 .check-base: extends: .default-rules stage: check script: - bundle exec $CI_JOB_NAME rubocop: extends: .check-base rspec: extends: .check-base release: stage: release rules: - if: '$CI_COMMIT_TAG' script: - gem update --system - ruby --version - gem env version - gem build gitlab-qa.gemspec - gem push gitlab-qa*.gem artifacts: paths: - gitlab-qa*.gem expire_in: 30 days .test: stage: test services: - docker:19.03.0-dind tags: - docker artifacts: when: always expire_in: 10d paths: - ./gitlab-qa-run-* reports: junit: gitlab-qa-run-*/**/rspec-*.xml script: - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"' - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$? - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN" - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi - exit $test_run_exit_code .ce-qa: variables: DEFAULT_RELEASE: "CE" rules: - if: '$CI_COMMIT_TAG || $RELEASE =~ /gitlab-ee/' when: never - if: '$RELEASE == null && $CI_JOB_NAME =~ /quarantine|custom/' when: manual - if: '$RELEASE =~ /gitlab-ce/ && $CI_JOB_NAME =~ /quarantine|custom/' when: manual - if: '$CI_MERGE_REQUEST_ID && $CI_JOB_NAME =~ /quarantine|custom/' when: manual - if: '$RELEASE == null || $RELEASE =~ /gitlab-ce/ || $CI_MERGE_REQUEST_ID || $CI_COMMIT_REF_NAME == "master"' .ee-qa: variables: DEFAULT_RELEASE: "EE" rules: - if: '$CI_COMMIT_TAG || $RELEASE =~ /gitlab-ce/' when: never - if: '$RELEASE == null && $CI_JOB_NAME =~ /quarantine|custom/' when: manual - if: '$RELEASE =~ /gitlab-ee/ && $CI_JOB_NAME =~ /quarantine|custom/' when: manual - if: '$CI_MERGE_REQUEST_ID && $CI_JOB_NAME =~ /quarantine|custom/' when: manual - if: '$RELEASE == null || $RELEASE =~ /gitlab-ee/ || $CI_MERGE_REQUEST_ID || $CI_COMMIT_REF_NAME == "master"' .only-qa: extends: .default-rules .high-capacity: tags: - docker - 7gb - triggered-packages .knapsack-variables: variables: KNAPSACK_REPORT_PATH: "knapsack/master_report.json" KNAPSACK_TEST_FILE_PATTERN: "qa/specs/features/**/*_spec.rb" KNAPSACK_GENERATE_REPORT: "true" .rspec-report-opts: variables: FILE_SAFE_JOB_NAME: $(echo $CI_JOB_NAME | sed 's/[ /]/_/g') RSPEC_REPORT_OPTS: "--format RspecJunitFormatter --out \"tmp/rspec-${CI_JOB_ID}.xml\" --format html --out \"tmp/rspec-${FILE_SAFE_JOB_NAME}.htm\" --color --format documentation" .quarantine: allow_failure: true variables: QA_RSPEC_TAGS: "--tag quarantine" ce:sanity-framework: script: - ./bin/expect_exit_code_and_text "exe/gitlab-qa Test::Instance::Image ${RELEASE:=CE} -- --tag framework" 1 "2 examples, 1 failure" extends: - .test - .high-capacity - .ce-qa ee:sanity-framework: script: - ./bin/expect_exit_code_and_text "exe/gitlab-qa Test::Instance::Image ${RELEASE:=EE} -- --tag framework" 1 "2 examples, 1 failure" extends: - .test - .high-capacity - .ee-qa # The custom jobs are for manually running specific/alternative tests in MRs, so we don't report them in issues ce:custom-parallel: script: - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"' - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS extends: - .test - .high-capacity - .ce-qa - .rspec-report-opts allow_failure: true parallel: 10 ee:custom-parallel: script: - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"' - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts allow_failure: true parallel: 10 ce:instance: extends: - .test - .high-capacity - .ce-qa - .knapsack-variables - .rspec-report-opts parallel: 5 ce:instance-quarantine: extends: - .test - .high-capacity - .ce-qa - .quarantine - .rspec-report-opts variables: QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated" ee:instance: extends: - .test - .high-capacity - .ee-qa - .knapsack-variables - .rspec-report-opts parallel: 5 ee:instance-quarantine: extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts variables: QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated" ce:relative_url: extends: - .test - .high-capacity - .ce-qa - .knapsack-variables - .rspec-report-opts parallel: 5 variables: QA_SCENARIO: "Test::Instance::RelativeUrl" ce:relative_url-quarantine: extends: - .test - .high-capacity - .ce-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Instance::RelativeUrl" QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated" ee:relative_url: extends: - .test - .high-capacity - .ee-qa - .knapsack-variables - .rspec-report-opts parallel: 5 variables: QA_SCENARIO: "Test::Instance::RelativeUrl" ee:relative_url-quarantine: extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Instance::RelativeUrl" QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated" ce:repository_storage: extends: - .test - .high-capacity - .ce-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Instance::RepositoryStorage" ce:repository_storage-quarantine: extends: - .test - .high-capacity - .ce-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Instance::RepositoryStorage" ee:repository_storage: extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Instance::RepositoryStorage" ee:repository_storage-quarantine: extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Instance::RepositoryStorage" # The Test::Omnibus::Image scenarios don't run the E2E tests so they don't need to report test results ce:image: script: - exe/gitlab-qa Test::Omnibus::Image ${RELEASE:=CE} extends: - .test - .ce-qa ee:image: script: - exe/gitlab-qa Test::Omnibus::Image ${RELEASE:=EE} extends: - .test - .ee-qa # The Test::Omnibus::Update scenarios require the release to be specified twice, which can't be done dynamically using the `variables` parameter # So instead we include the script here again, with two release variables ce:update: script: - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$? - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN" - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi - exit $test_run_exit_code extends: - .test - .high-capacity - .ce-qa - .rspec-report-opts - .knapsack-variables parallel: 5 ce:update-quarantine: script: - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$? - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN" - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi - exit $test_run_exit_code extends: - .test - .high-capacity - .ce-qa - .quarantine - .rspec-report-opts ee:update: script: - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$? - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN" - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi - exit $test_run_exit_code extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts - .knapsack-variables parallel: 5 ee:update-quarantine: script: - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$? - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN" - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi - exit $test_run_exit_code extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts # The Test::Omnibus::Upgrade scenario isn't run on master (because it always uses the latest CE/EE image) so we don't report the test results in issues ce:upgrade: script: - exe/gitlab-qa Test::Omnibus::Upgrade CE -- $RSPEC_REPORT_OPTS extends: - .test - .high-capacity - .only-qa - .rspec-report-opts - .knapsack-variables parallel: 5 ce:upgrade-quarantine: script: - exe/gitlab-qa Test::Omnibus::Upgrade CE -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS extends: - .test - .high-capacity - .only-qa - .quarantine - .rspec-report-opts ee-previous-to-ce:update: script: - exe/gitlab-qa Test::Omnibus::Update EE CE -- $RSPEC_REPORT_OPTS extends: - .test - .high-capacity - .only-qa - .rspec-report-opts ee-previous-to-ce:update-quarantine: script: - exe/gitlab-qa Test::Omnibus::Update EE CE -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS extends: - .test - .high-capacity - .only-qa - .quarantine - .rspec-report-opts ce:mattermost: extends: - .test - .high-capacity - .ce-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::Mattermost" ce:mattermost-quarantine: extends: - .test - .high-capacity - .ce-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::Mattermost" ee:mattermost: extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::Mattermost" ee:mattermost-quarantine: extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::Mattermost" ee:geo: extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::Geo" ee:geo-quarantine: extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::Geo" ce:ldap_no_tls: extends: - .test - .high-capacity - .ce-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::LDAPNoTLS" ce:ldap_no_tls-quarantine: extends: - .test - .high-capacity - .ce-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::LDAPNoTLS" ee:ldap_no_tls: extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::LDAPNoTLS" ee:ldap_no_tls-quarantine: extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::LDAPNoTLS" ce:ldap_tls: extends: - .test - .high-capacity - .ce-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::LDAPTLS" ce:ldap_tls-quarantine: extends: - .test - .high-capacity - .ce-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::LDAPTLS" ee:ldap_tls: extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::LDAPTLS" ee:ldap_tls-quarantine: extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::LDAPTLS" ee:ldap_no_server: extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::LDAPNoServer" ee:ldap_no_server-quarantine: extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::LDAPNoServer" ce:instance_saml: extends: - .test - .high-capacity - .ce-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::InstanceSAML" ce:instance_saml-quarantine: extends: - .test - .high-capacity - .ce-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::InstanceSAML" ee:instance_saml: extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::InstanceSAML" ee:instance_saml-quarantine: extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::InstanceSAML" ee:group_saml: extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::GroupSAML" ee:group_saml-quarantine: extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::GroupSAML" ce:kubernetes: extends: - .test - .high-capacity - .ce-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::Kubernetes" ce:kubernetes-quarantine: extends: - .test - .high-capacity - .ce-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::Kubernetes" ee:kubernetes: extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::Kubernetes" ee:kubernetes-quarantine: extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::Kubernetes" ce:object_storage: extends: - .test - .high-capacity - .ce-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::ObjectStorage" ce:object_storage-quarantine: extends: - .test - .high-capacity - .ce-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::ObjectStorage" ee:object_storage: extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::ObjectStorage" ee:object_storage-quarantine: extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::ObjectStorage" ce:oauth: extends: - .test - .high-capacity - .ce-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::OAuth" ce:oauth-quarantine: extends: - .test - .high-capacity - .ce-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::OAuth" ee:oauth: extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::OAuth" ee:oauth-quarantine: extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::OAuth" ee:packages: extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::Packages" ee:packages-quarantine: extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::Packages" ee:elasticsearch: extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::Elasticsearch" ee:elasticsearch-quarantine: extends: - .test - .high-capacity - .ee-qa - .quarantine - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::Elasticsearch" ce:praefect: extends: - .test - .high-capacity - .ce-qa - .knapsack-variables - .rspec-report-opts parallel: 5 variables: QA_SCENARIO: "Test::Integration::Praefect" ee:praefect: extends: - .test - .high-capacity - .ee-qa - .knapsack-variables - .rspec-report-opts parallel: 5 variables: QA_SCENARIO: "Test::Integration::Praefect" ce:smtp: extends: - .test - .high-capacity - .ce-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::SMTP" ee:smtp: extends: - .test - .high-capacity - .ee-qa - .rspec-report-opts variables: QA_SCENARIO: "Test::Integration::SMTP" # This job requires the `GITLAB_QA_ACCESS_TOKEN` and `GITLAB_QA_DEV_ACCESS_TOKEN` # variable to be passed when triggered. staging: script: - unset EE_LICENSE - exe/gitlab-qa Test::Instance::Staging extends: - .test - .high-capacity - .only-qa allow_failure: true .notify_upstream_commit: stage: notify image: ruby:2.6 before_script: - gem install gitlab --no-document notify_upstream_commit:success: extends: .notify_upstream_commit script: - bin/notify_upstream_commit success rules: - if: '$TOP_UPSTREAM_SOURCE_PROJECT && $TOP_UPSTREAM_SOURCE_SHA' when: on_success notify_upstream_commit:failure: extends: .notify_upstream_commit script: - bin/notify_upstream_commit failure rules: - if: '$TOP_UPSTREAM_SOURCE_PROJECT && $TOP_UPSTREAM_SOURCE_SHA' when: on_failure .notify_slack: image: alpine stage: notify dependencies: [] cache: {} before_script: - apk update && apk add git curl bash notify_slack: extends: - .notify_slack rules: - if: '$TOP_UPSTREAM_SOURCE_JOB && $NOTIFY_CHANNEL' when: on_failure - if: '$TOP_UPSTREAM_SOURCE_JOB && $TOP_UPSTREAM_SOURCE_REF == "master"' when: on_failure script: - export RELEASE=${TOP_UPSTREAM_SOURCE_REF:-$RELEASE} - echo "NOTIFY_CHANNEL is ${NOTIFY_CHANNEL:=qa-$TOP_UPSTREAM_SOURCE_REF}" - echo "RELEASE is ${RELEASE}" - echo "CI_PIPELINE_URL is $CI_PIPELINE_URL" - echo "TOP_UPSTREAM_SOURCE_JOB is $TOP_UPSTREAM_SOURCE_JOB" - bin/slack $NOTIFY_CHANNEL "☠️ QA against $RELEASE failed! ☠️ See $CI_PIPELINE_URL (triggered from $TOP_UPSTREAM_SOURCE_JOB)" ci_failing