diff --git a/.github/actions/setup-dwo-smoke/action.yml b/.github/actions/setup-dwo-smoke/action.yml new file mode 100644 index 000000000..2f2d48402 --- /dev/null +++ b/.github/actions/setup-dwo-smoke/action.yml @@ -0,0 +1,204 @@ +# +# Copyright (c) 2019-2026 Red Hat, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Install pull request controller on minikube +description: >- + Start a small minikube cluster, install this checkout's controller image, + and clone the DevWorkspace operator load-tests repository so a later step + can create one DevWorkspace. The controller and webhook deployments are + pinned to imagePullPolicy Never so the cluster runs the image built from + the pull request instead of quay.io/devfile/devworkspace-controller:next. + +inputs: + image-tarball: + description: gzip-compressed docker save archive of the controller image + required: true + image-ref: + description: Image reference stored in the archive and installed as DWO_IMG + required: false + default: docker.io/library/devworkspace-controller:ci + kubernetes-version: + description: Kubernetes version started by minikube + required: false + default: v1.33.0 + dwo-namespace: + description: Namespace the operator is installed into + required: false + default: devworkspace-controller + load-test-repo: + description: Load-test repository cloned after the operator is ready + required: false + default: https://github.com/rohankanojia-testing/devworkspace-operator-load-tests.git + load-test-ref: + description: Git ref cloned from the load-test repository + required: false + default: main + +outputs: + load-test-dir: + description: Absolute path of the cloned load-test repository + value: ${{ steps.clone-load-tests.outputs.dir }} + +runs: + using: composite + steps: + - name: Setup Minikube-Kubernetes + uses: manusa/actions-setup-minikube@001b9f30165b2a875b835ec5183ced75b94bc269 # v2.19.0 + with: + minikube version: v1.39.0 + kubernetes version: ${{ inputs.kubernetes-version }} + github token: ${{ github.token }} + + - name: Enable metrics-server + shell: bash + run: | + minikube addons enable metrics-server + kubectl rollout status deployment/metrics-server -n kube-system --timeout=300s + + - name: Install cert-manager + shell: bash + run: | + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml + kubectl rollout status deployment/cert-manager -n cert-manager --timeout=300s + kubectl rollout status deployment/cert-manager-webhook -n cert-manager --timeout=300s + kubectl rollout status deployment/cert-manager-cainjector -n cert-manager --timeout=300s + for _ in $(seq 1 30); do + if kubectl api-versions | grep -q '^cert-manager.io/v1$'; then + exit 0 + fi + sleep 5 + done + echo "cert-manager.io/v1 did not become available" + exit 1 + + - name: Enable ingress + shell: bash + run: | + minikube addons enable ingress + kubectl rollout status deployment/ingress-nginx-controller -n ingress-nginx --timeout=300s + + - name: Install deployment tools and k6 + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y jq gettext-base + pip install yq || pip install --break-system-packages yq || true + # Ubuntu 26 apt ignores the keybox written by gpg --recv-keys. Dearmor + # the published key into the binary keyring format apt accepts. + curl -fsSL https://dl.k6.io/key.gpg | gpg --dearmor | sudo tee /usr/share/keyrings/k6-archive-keyring.gpg >/dev/null + sudo chmod 644 /usr/share/keyrings/k6-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list + sudo apt-get update + sudo apt-get install -y k6 + k6 version + + - name: Load the pull request image into minikube + shell: bash + env: + IMAGE_TARBALL: ${{ inputs.image-tarball }} + IMAGE_REF: ${{ inputs.image-ref }} + run: | + set -euo pipefail + archive="${RUNNER_TEMP}/devworkspace-controller-ci.tar" + gzip -dc "${IMAGE_TARBALL}" > "${archive}" + minikube image load "${archive}" + rm -f "${archive}" + echo "Images visible to minikube:" + minikube image ls + if ! minikube image ls | grep -q 'devworkspace-controller:ci'; then + echo "minikube does not have ${IMAGE_REF}" + exit 1 + fi + + - name: Install DevWorkspace Operator from this checkout + shell: bash + env: + NAMESPACE: ${{ inputs.dwo-namespace }} + DWO_IMG: ${{ inputs.image-ref }} + run: | + set -euo pipefail + echo "Installing ${DWO_IMG} into namespace ${NAMESPACE}" + kubectl config current-context + make install + + # The generated deployment and the webhook Deployment created at + # controller startup both set imagePullPolicy Always. Always ignores + # the image loaded into minikube, so pin Never after install. The + # controller writes the webhook Deployment once during startup and + # does not reconcile it afterwards. + wait_and_pin_image() { + local deploy="$1" + local container="$2" + local deadline=$((SECONDS + 360)) + local idx policy image + + while (( SECONDS < deadline )); do + if ! kubectl -n "${NAMESPACE}" get deployment "${deploy}" >/dev/null 2>&1; then + echo "waiting for deployment/${deploy}" + sleep 5 + continue + fi + + idx="$(kubectl -n "${NAMESPACE}" get deployment "${deploy}" -o json | jq -r --arg name "${container}" ' + .spec.template.spec.containers + | to_entries[] + | select(.value.name == $name) + | .key')" + if [[ -z "${idx}" || "${idx}" == "null" ]]; then + echo "container ${container} not found on deployment/${deploy}" + kubectl -n "${NAMESPACE}" get deployment "${deploy}" -o yaml + return 1 + fi + + kubectl -n "${NAMESPACE}" patch deployment "${deploy}" --type=json \ + -p="[{\"op\":\"replace\",\"path\":\"/spec/template/spec/containers/${idx}/imagePullPolicy\",\"value\":\"Never\"}]" + sleep 2 + policy="$(kubectl -n "${NAMESPACE}" get deployment "${deploy}" -o jsonpath="{.spec.template.spec.containers[${idx}].imagePullPolicy}")" + image="$(kubectl -n "${NAMESPACE}" get deployment "${deploy}" -o jsonpath="{.spec.template.spec.containers[${idx}].image}")" + echo "${deploy}: image=${image} imagePullPolicy=${policy}" + if [[ "${policy}" == "Never" && "${image}" == "${DWO_IMG}" ]]; then + if kubectl -n "${NAMESPACE}" rollout status "deployment/${deploy}" --timeout=180s; then + policy="$(kubectl -n "${NAMESPACE}" get deployment "${deploy}" -o jsonpath="{.spec.template.spec.containers[${idx}].imagePullPolicy}")" + image="$(kubectl -n "${NAMESPACE}" get deployment "${deploy}" -o jsonpath="{.spec.template.spec.containers[${idx}].image}")" + if [[ "${policy}" == "Never" && "${image}" == "${DWO_IMG}" ]]; then + return 0 + fi + fi + fi + sleep 5 + done + + echo "timed out pinning deployment/${deploy} to ${DWO_IMG}" + kubectl -n "${NAMESPACE}" get deploy,pods -o wide || true + kubectl -n "${NAMESPACE}" logs -l app.kubernetes.io/name=devworkspace-controller --tail=100 || true + return 1 + } + + wait_and_pin_image devworkspace-controller-manager devworkspace-controller + wait_and_pin_image devworkspace-webhook-server webhook-server + kubectl get pods -n "${NAMESPACE}" -o wide + + - name: Clone load tests + id: clone-load-tests + shell: bash + env: + LOAD_TEST_REPO: ${{ inputs.load-test-repo }} + LOAD_TEST_REF: ${{ inputs.load-test-ref }} + run: | + set -euo pipefail + dest="${RUNNER_TEMP}/devworkspace-operator-load-tests" + git clone --depth 1 --branch "${LOAD_TEST_REF}" "${LOAD_TEST_REPO}" "${dest}" + git -C "${dest}" log -1 --oneline + echo "dir=${dest}" >> "${GITHUB_OUTPUT}" diff --git a/.github/scripts/collect-smoke-diagnostics.sh b/.github/scripts/collect-smoke-diagnostics.sh new file mode 100644 index 000000000..794204bda --- /dev/null +++ b/.github/scripts/collect-smoke-diagnostics.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright (c) 2019-2026 Red Hat, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Dump the objects that explain a failed minikube verification. Invoked with +# `if: failure()`, so a missing cluster must not fail the step. +set +e + +namespace="${DWO_NAMESPACE:-devworkspace-controller}" + +echo "===== pods =====" +kubectl get pods -A -o wide + +echo "===== controller logs =====" +kubectl logs -n "${namespace}" -l app.kubernetes.io/name=devworkspace-controller --tail=200 + +echo "===== webhook logs =====" +kubectl logs -n "${namespace}" -l app.kubernetes.io/name=devworkspace-webhook-server --tail=200 + +echo "===== devworkspaces =====" +kubectl get devworkspaces -A -o yaml + +echo "===== devworkspaceoperatorconfigs =====" +kubectl get devworkspaceoperatorconfigs -A -o yaml + +echo "===== backup jobs =====" +kubectl get jobs -A -l controller.devfile.io/backup-job=true -o yaml + +echo "===== warning events =====" +kubectl get events -A --field-selector type=Warning + +exit 0 diff --git a/.github/workflows/verify-devworkspace.yaml b/.github/workflows/verify-devworkspace.yaml new file mode 100644 index 000000000..45aa74764 --- /dev/null +++ b/.github/workflows/verify-devworkspace.yaml @@ -0,0 +1,195 @@ +# +# Copyright (c) 2019-2026 Red Hat, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Creates one DevWorkspace on minikube against the controller built from this +# checkout, then repeats that check for the webhook and for backup. This is a +# functional check on a tiny cluster, not a load test. Installing +# quay.io/devfile/devworkspace-controller:next would not exercise the pull request. +name: Verify DevWorkspace + +on: + pull_request: + branches: [ main ] + push: + branches: [ main ] + workflow_dispatch: + +concurrency: + group: verify-devworkspace-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + DWO_NAMESPACE: devworkspace-controller + DWO_IMG: docker.io/library/devworkspace-controller:ci + LOAD_TEST_REPO: https://github.com/rohankanojia-testing/devworkspace-operator-load-tests.git + LOAD_TEST_REF: main + +jobs: + build-controller-image: + name: Build pull request image + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Check out code + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + + - name: Build controller and webhook image + run: | + docker build -t "${DWO_IMG}" -f build/Dockerfile . + docker save "${DWO_IMG}" | gzip > "${GITHUB_WORKSPACE}/dwo-image.tar.gz" + + - name: Upload controller image + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: dwo-controller-image + path: dwo-image.tar.gz + retention-days: 1 + if-no-files-found: error + + verify-controller: + name: Minikube v1.39.0 / K8s v1.33.0 / One DevWorkspace + needs: build-controller-image + runs-on: ubuntu-latest + timeout-minutes: 50 + steps: + - name: Check out code + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + + - name: Download controller image + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: dwo-controller-image + path: ${{ github.workspace }}/controller-image + + - name: Install pull request controller + id: setup + uses: ./.github/actions/setup-dwo-smoke + with: + image-tarball: ${{ github.workspace }}/controller-image/dwo-image.tar.gz + image-ref: ${{ env.DWO_IMG }} + dwo-namespace: ${{ env.DWO_NAMESPACE }} + load-test-repo: ${{ env.LOAD_TEST_REPO }} + load-test-ref: ${{ env.LOAD_TEST_REF }} + + - name: Create one DevWorkspace + working-directory: ${{ steps.setup.outputs.load-test-dir }} + env: + LOAD_TEST_NAMESPACE: loadtest-devworkspaces + run: | + export KUBECONFIG="${HOME}/.kube/config" + make test_load ARGS="--max-vus 1 --max-devworkspaces 1 --dwo-namespace ${LOAD_TEST_NAMESPACE} --delete-devworkspace-after-ready false --test-duration-minutes 5" + + - name: Collect cluster diagnostics + if: failure() + run: | + bash "${GITHUB_WORKSPACE}/.github/scripts/collect-smoke-diagnostics.sh" + + verify-webhook: + name: Minikube v1.39.0 / K8s v1.33.0 / Webhook + needs: build-controller-image + runs-on: ubuntu-latest + timeout-minutes: 50 + steps: + - name: Check out code + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + + - name: Download controller image + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: dwo-controller-image + path: ${{ github.workspace }}/controller-image + + - name: Install pull request controller + id: setup + uses: ./.github/actions/setup-dwo-smoke + with: + image-tarball: ${{ github.workspace }}/controller-image/dwo-image.tar.gz + image-ref: ${{ env.DWO_IMG }} + dwo-namespace: ${{ env.DWO_NAMESPACE }} + load-test-repo: ${{ env.LOAD_TEST_REPO }} + load-test-ref: ${{ env.LOAD_TEST_REF }} + + - name: Check the webhook with one user + working-directory: ${{ steps.setup.outputs.load-test-dir }} + env: + LOAD_TEST_NAMESPACE: dw-webhook-loadtest + DEVWORKSPACE_LINK: "" + run: | + export KUBECONFIG="${HOME}/.kube/config" + make test_webhook_load ARGS="--number-of-users 1 --load-test-namespace ${LOAD_TEST_NAMESPACE} --webhook-server-namespace ${DWO_NAMESPACE}" + + - name: Collect cluster diagnostics + if: failure() + run: | + bash "${GITHUB_WORKSPACE}/.github/scripts/collect-smoke-diagnostics.sh" + + verify-backup: + name: Minikube v1.39.0 / K8s v1.33.0 / Backup + needs: build-controller-image + runs-on: ubuntu-latest + timeout-minutes: 50 + steps: + - name: Check out code + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + + - name: Download controller image + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: dwo-controller-image + path: ${{ github.workspace }}/controller-image + + - name: Install pull request controller + id: setup + uses: ./.github/actions/setup-dwo-smoke + with: + image-tarball: ${{ github.workspace }}/controller-image/dwo-image.tar.gz + image-ref: ${{ env.DWO_IMG }} + dwo-namespace: ${{ env.DWO_NAMESPACE }} + load-test-repo: ${{ env.LOAD_TEST_REPO }} + load-test-ref: ${{ env.LOAD_TEST_REF }} + + - name: Back up one DevWorkspace + working-directory: ${{ steps.setup.outputs.load-test-dir }} + env: + LOAD_TEST_NAMESPACE: loadtest-devworkspaces + REGISTRY_USERNAME: anonymous + REGISTRY_PASSWORD: anonymous + run: | + export KUBECONFIG="${HOME}/.kube/config" + kubectl create namespace "${LOAD_TEST_NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - + random_id="backup-test-${GITHUB_RUN_ID}-$(date +%s)" + export BACKUP_DEVWORKSPACE_TEMPLATE="${PWD}/common/templates/dw-minimal-per-workspace-storage-scale.json" + make test_backup \ + MAX_DEVWORKSPACES=1 \ + BACKUP_MONITOR_DURATION=5 \ + LOAD_TEST_NAMESPACE="${LOAD_TEST_NAMESPACE}" \ + DWO_NAMESPACE="${DWO_NAMESPACE}" \ + REGISTRY_PATH="ttl.sh/${random_id}" \ + REGISTRY_SECRET=quay-push-secret \ + DWOC_CONFIG_TYPE=correct \ + SEPARATE_NAMESPACE=false \ + BACKUP_SCHEDULE="*/2 * * * *" \ + VERIFY_RESTORE=true \ + MAX_RESTORE_SAMPLES=1 \ + WAIT_FOR_READY=true \ + WAIT_TIMEOUT=10 + + - name: Collect cluster diagnostics + if: failure() + run: | + bash "${GITHUB_WORKSPACE}/.github/scripts/collect-smoke-diagnostics.sh"