forked from kubeflow/spark-operator
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Various CI improvements (kubeflow#1418)
This change renames the github workflows to be clearer about their purpose, and adds a set of tests which aim to force developers to increment the appVersion if they have changed anything with the spark-operator docker container or the chart version if they have updated the chart. Signed-off-by: Tom Hellier <[email protected]>
- Loading branch information
1 parent
d04160e
commit ba16242
Showing
14 changed files
with
732 additions
and
454 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,7 +10,36 @@ on: | |
| - master | ||
|
|
||
| jobs: | ||
| build: | ||
|
|
||
| build-api-docs: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout source code | ||
| uses: actions/checkout@v2 | ||
| with: | ||
| fetch-depth: "0" | ||
|
|
||
| - name: The API should not change once published | ||
| run: | | ||
| if ! git diff --quiet origin/master -- pkg/apis/sparkoperator.k8s.io/v1beta1; then | ||
| echo "sparkoperator.k8s.io/v1beta1 api has changed" | ||
| false | ||
| fi | ||
| if ! git diff --quiet origin/master -- pkg/apis/sparkoperator.k8s.io/v1beta2; then | ||
| echo "sparkoperator.k8s.io/v1beta2 api has changed" | ||
| false | ||
| fi | ||
| - name: The API documentation hasn't changed | ||
| run: | | ||
| make build-api-docs | ||
| if ! git diff --quiet -- docs/api-docs.md; then | ||
| echo "Need to re-run 'make build-api-docs' and commit the changes" | ||
| git diff -- docs/api-docs.md; | ||
| false | ||
| fi | ||
| build-sparkctl: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Set up Go | ||
|
|
@@ -21,22 +50,15 @@ jobs: | |
| - name: Checkout source code | ||
| uses: actions/checkout@v2 | ||
| with: | ||
| fetch-depth: 2 | ||
|
|
||
| - name: fmt check | ||
| run: make fmt-check | ||
| fetch-depth: "0" | ||
|
|
||
| - name: unit tests | ||
| run: make test | ||
| - name: build sparkctl | ||
| run: | | ||
| make all | ||
| it: | ||
| build-spark-operator: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v2 | ||
| with: | ||
| fetch-depth: 0 | ||
|
|
||
| - name: Set up Go | ||
| uses: actions/setup-go@v2 | ||
| with: | ||
|
|
@@ -45,32 +67,75 @@ jobs: | |
| - name: Checkout source code | ||
| uses: actions/checkout@v2 | ||
| with: | ||
| fetch-depth: 2 | ||
| fetch-depth: "0" | ||
|
|
||
| - name: Run gofmt check | ||
| run: make fmt-check | ||
|
|
||
| - name: Run static analysis | ||
| run: make static-analysis | ||
|
|
||
| - name: Run unit tests | ||
| run: make unit-test | ||
|
|
||
| - name: Build Spark-Operator Docker Image | ||
| run: | | ||
| tag=$(git describe --tags --dirty)_v3.1.1 | ||
| docker build -t gcr.io/spark-operator/spark-operator:${tag} . | ||
| docker build -t gcr.io/spark-operator/spark-operator:local . | ||
| DOCKER_TAG=$(cat charts/spark-operator-chart/Chart.yaml | grep "appVersion: .*" | cut -c13-) | ||
| docker build -t gcr.io/spark-operator/spark-operator:${DOCKER_TAG} . | ||
| - name: Check changes in resources used in docker file | ||
| run: | | ||
| DOCKERFILE_RESOURCES=$(cat Dockerfile | grep -P -o "COPY [a-zA-Z0-9].*? " | cut -c6-) | ||
| for resource in $DOCKERFILE_RESOURCES; do | ||
| # If the resource is different | ||
| if ! git diff --quiet origin/master -- $resource; then | ||
| ## And the appVersion hasn't been updated | ||
| if ! git diff --quiet charts/spark-operator-chart/Chart.yaml | grep +appVersion; then | ||
| echo "resource used in gcr.io/spark-operator/spark-operator has changed in $resource, need to update the appVersion in charts/spark-operator-chart/Chart.yaml" | ||
| git diff origin/master -- $resource; | ||
| echo "failing the build... " && false | ||
| fi | ||
| fi | ||
| done | ||
| build-helm-chart: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout source code | ||
| uses: actions/checkout@v2 | ||
| with: | ||
| fetch-depth: "0" | ||
|
|
||
| - name: Install Helm | ||
| uses: azure/setup-helm@v1 | ||
| with: | ||
| version: v3.4.0 | ||
| version: v3.7.1 | ||
|
|
||
| - uses: actions/setup-python@v2 | ||
| with: | ||
| python-version: 3.7 | ||
| - name: Produce the helm documentation | ||
| run: | | ||
| make helm-docs | ||
| if ! git diff --quiet -- charts/spark-operator-chart/README.md; then | ||
| echo "Need to re-run 'make helm-docs' and commit the changes" | ||
| false | ||
| fi | ||
| - name: Set up chart-testing | ||
| uses: helm/[email protected] | ||
|
|
||
| - name: Print ct version information and List files | ||
| run: ct version && ls -lh | ||
| - name: Print chart-testing version information | ||
| run: ct version | ||
|
|
||
| - name: Run chart-testing (lint) | ||
| run: ct lint | ||
|
|
||
| - name: Run chart-testing (list-changed) | ||
| id: list-changed | ||
| run: | | ||
| changed=$(ct list-changed) | ||
| if [[ -n "$changed" ]]; then | ||
| echo "::set-output name=changed::true" | ||
| fi | ||
| - name: Detect CRDs drift between chart and manifest | ||
| run: make detect-crds-drift | ||
|
|
||
|
|
@@ -84,13 +149,39 @@ jobs: | |
|
|
||
| - name: Run chart-testing (install) | ||
| run: | | ||
| tag=$(git describe --tags --dirty)_v3.1.1 | ||
| docker build -t gcr.io/spark-operator/spark-operator:local . | ||
| minikube image load gcr.io/spark-operator/spark-operator:local | ||
| ct install | ||
| integration-test: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Set up Go | ||
| uses: actions/setup-go@v2 | ||
| with: | ||
| go-version: 1.15 | ||
|
|
||
| - name: Checkout source code | ||
| uses: actions/checkout@v2 | ||
| with: | ||
| fetch-depth: "0" | ||
|
|
||
| - name: setup minikube | ||
| uses: manusa/[email protected] | ||
| with: | ||
| minikube version: "v1.24.0" | ||
| kubernetes version: "v1.20.8" | ||
| start args: --memory 6g --cpus=2 --addons ingress | ||
| github token: ${{ inputs.github-token }} | ||
|
|
||
| - name: Build local spark-operator docker image for minikube testing | ||
| run: | | ||
| docker build -t gcr.io/spark-operator/spark-operator:local . | ||
| minikube image load gcr.io/spark-operator/spark-operator:local | ||
| # The integration tests are currently broken see: https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/issues/1416 | ||
| # - name: Run chart-testing (integration test) | ||
| # run: make it-test | ||
| # run: make integation-test | ||
|
|
||
| - name: Setup tmate session | ||
| if: failure() | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,7 +12,7 @@ jobs: | |
| - name: Checkout | ||
| uses: actions/checkout@v2 | ||
| with: | ||
| fetch-depth: 0 | ||
| fetch-depth: "0" | ||
|
|
||
| - name: Configure Git | ||
| run: | | ||
|
|
@@ -22,11 +22,11 @@ jobs: | |
| - name: Install Helm | ||
| uses: azure/setup-helm@v1 | ||
| with: | ||
| version: v3.4.0 | ||
| version: v3.7.1 | ||
|
|
||
| - uses: actions/setup-python@v2 | ||
| with: | ||
| python-version: 3.7 | ||
| python-version: 3.10 | ||
|
|
||
| # TODO: Maintainer of repository to follow: | ||
| # https://github.com/docker/login-action#google-container-registry-gcr to add credentials so | ||
|
|
@@ -38,42 +38,17 @@ jobs: | |
| # username: ${{ secrets.DOCKER_USERNAME }} | ||
| # password: ${{ secrets.DOCKER_PASSWORD }} | ||
|
|
||
| - name: Build Spark-Operator Docker Image | ||
| - name: Release Spark-Operator Docker Image | ||
| run: | | ||
| tag=$(git describe --tags --dirty)_v3.1.1 | ||
| docker build -t gcr.io/spark-operator/spark-operator:${tag} . | ||
| docker build -t gcr.io/spark-operator/spark-operator:local . | ||
| DOCKER_TAG=$(cat charts/spark-operator-chart/Chart.yaml | grep "appVersion: .*" | cut -c13-) | ||
| docker build -t gcr.io/spark-operator/spark-operator:${DOCKER_TAG} . | ||
| echo "Ideally, we'd release the docker container at this point, but the maintainer of this repo needs to approve..." | ||
| echo "docker push gcr.io/spark-operator/spark-operator:${tag}" | ||
| - name: Set up chart-testing | ||
| uses: helm/[email protected] | ||
|
|
||
| - name: Run chart-testing (list-changed) | ||
| id: list-changed | ||
| run: | | ||
| changed=$(ct list-changed) | ||
| if [[ -n "$changed" ]]; then | ||
| echo "::set-output name=changed::true" | ||
| if ! docker pull docker pull gcr.io/spark-operator/spark-operator:${DOCKER_TAG}; then | ||
| echo "docker push gcr.io/spark-operator/spark-operator:${DOCKER_TAG}" | ||
| git tag $DOCKER_TAG | ||
| git push origin $DOCKER_TAG | ||
| fi | ||
| - name: Run chart-testing (lint) | ||
| run: ct lint | ||
|
|
||
| - name: setup minikube | ||
| uses: manusa/[email protected] | ||
| with: | ||
| minikube version: "v1.24.0" | ||
| kubernetes version: "v1.20.8" | ||
| start args: --memory 6g --cpus=2 --addons ingress | ||
| github token: ${{ inputs.github-token }} | ||
|
|
||
| - name: Run chart-testing (install) | ||
| run: | | ||
| tag=$(git describe --tags --dirty)_v3.1.1 | ||
| minikube image load gcr.io/spark-operator/spark-operator:local | ||
| ct install | ||
| - name: Run chart-releaser | ||
| uses: helm/[email protected] | ||
| env: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.