name: CI Build Test
on:
pull_request:
branches-ignore:
- /^release\/.*/
- main
jobs:
build:
runs-on: ubuntu-20.04
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Setup Ruby and install gems
uses: ruby/setup-ruby@v1
with:
bundler-cache: true
ruby-version: 2.6.1
- name: Install dependencies
run: |
sudo ci_scripts/install_dep.sh
- name: Builder
run: |
rake build -t -v
cp -R pkg /tmp
- name: Cache pkg
uses: actions/cache@v1
with:
path: /tmp
key: ${{ runner.os }}-build
unit-test:
runs-on: ubuntu-20.04
needs:
- build
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Install dependencies
run: |
sudo ci_scripts/install_dep.sh
- uses: actions/cache@v2
with:
path: /tmp
key: ${{ runner.os }}-build
- name: Run unit tests
run: |
bundle exec rake test -t -v
func-test:
needs:
- unit-test
runs-on: ubuntu-20.04
env:
CI_SPLUNK_PORT: 8089
CI_SPLUNK_USERNAME: admin
CI_SPLUNK_HEC_TOKEN: a6b5e77f-d5f6-415a-bd43-930cecb12959
CI_SPLUNK_PASSWORD: helloworld
CI_INDEX_EVENTS: ci_events
CI_INDEX_OBJECTS: ci_objects
CI_INDEX_METRICS: ci_metrics
KUBERNETES_VERSION: v1.23.2
MINIKUBE_VERSION: v1.24.0
MINIKUBE_NODE_COUNTS: 2
GITHUB_ACTIONS: true
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Prepare container build
id: prep
run: |
VERSION=`cat VERSION`
TAGS=splunk/k8s-metrics-aggr:recent
echo ::set-output name=tags::${TAGS}
echo ::set-output name=version::${VERSION}
- name: Set up QEMU
uses: docker/setup-qemu-action@master
with:
platforms: all
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@master
- name: Build multi-arch kubernetes-metrics image
uses: docker/build-push-action@v2
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
file: ./docker/Dockerfile
platforms: linux/amd64
push: false
load: true
tags: ${{ steps.prep.outputs.tags }}
build-args: VERSION=${{ steps.prep.outputs.version }}
- name: Build k8s-metrics-aggr image
run: |
docker image ls
- name: Setup Minikube
run: |
# Install Kubectl
curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl
chmod +x kubectl
sudo mv kubectl /usr/local/bin/
mkdir -p ${HOME}/.kube
touch ${HOME}/.kube/config
# Install Minikube
curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
chmod +x minikube
sudo mv minikube /usr/local/bin/
# Start Minikube and Wait
minikube start --driver=docker --container-runtime=docker --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --no-vtx-check -n=${MINIKUBE_NODE_COUNTS}
export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
sleep 1;
done
- name: Install Splunk
run: |
# Wait until minikube is ready
export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
echo "wait for minikube ready ..."
sleep 1;
done
kubectl get nodes
until kubectl get sa | grep -q 'default'; do
sleep 1;
done
# Install Splunk on minikube
kubectl apply -f ci_scripts/k8s-splunk.yml
# Wait until splunk is ready
until kubectl logs splunk --tail=2 | grep -q 'Ansible playbook complete'; do
sleep 1;
done
export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}})
# Setup Indexes
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_EVENTS -d datatype=event
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_OBJECTS -d datatype=event
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_METRICS -d datatype=metric
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=default-events -d datatype=event
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=ns-anno -d datatype=event
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=pod-anno -d datatype=event
# Enable HEC services
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http/http/enable
# Create new HEC token
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=splunk_hec_token&token=a6b5e77f-d5f6-415a-bd43-930cecb12959&disabled=0&index=default-events&indexes=default-events,$CI_INDEX_METRICS,$CI_INDEX_OBJECTS,$CI_INDEX_EVENTS,ns-anno,pod-anno" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http
# Restart Splunk
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/server/control/restart -X POST
- name: Deploy k8s connector
run: |
export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}})
ci_scripts/deploy_connector.sh
- name: Deploy log generator
run: |
cd /opt/splunk-connect-for-kubernetes
kubectl apply -f test/test_setup.yaml
sleep 65
- uses: actions/setup-python@v2
with:
python-version: 3.7
- name: Run functional tests
run: |
echo "check the pods"
kubectl get pods -A
cd /opt/splunk-connect-for-kubernetes
kubectl get nodes
export PYTHONWARNINGS="ignore:Unverified HTTPS request"
export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}})
cd test
pip install --upgrade pip
pip install -r requirements.txt
echo "Running functional tests....."
python -m pytest \
--splunkd-url https://$CI_SPLUNK_HOST:8089 \
--splunk-user admin \
--splunk-password $CI_SPLUNK_PASSWORD \
--nodes-count $MINIKUBE_NODE_COUNTS\
-p no:warnings -s -n auto
fossa-scan:
continue-on-error: true
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: run fossa anlyze and create report
run: |
curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install-latest.sh | bash
fossa analyze --include-unused-deps --debug
fossa report attribution --format text > /tmp/THIRDPARTY
env:
FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }}
- name: upload THIRDPARTY file
uses: actions/upload-artifact@v2
with:
name: THIRDPARTY
path: /tmp/THIRDPARTY
- name: run fossa test
run: |
fossa test --debug
env:
FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }}
semgrep:
runs-on: ubuntu-latest
name: security-sast-semgrep
if: github.actor != 'dependabot[bot]'
steps:
- uses: actions/checkout@v3
- name: Semgrep
id: semgrep
uses: returntocorp/semgrep-action@v1
with:
publishToken: ${{ secrets.SEMGREP_PUBLISH_TOKEN }}