Merge commit 'a17045a9096bade0f8bbe669f8ef42db771a062f' into update-release-tools
This commit is contained in:
Aaron Crickenberger
@@ -1,5 +1,5 @@
|
||||
#! /bin/bash
|
||||
#
|
||||
|
||||
# Copyright 2019 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@@ -33,8 +33,6 @@
|
||||
# The expected environment is:
|
||||
# - $GOPATH/src/<import path> for the repository that is to be tested,
|
||||
# with PR branch merged (when testing a PR)
|
||||
# - optional: bazel installed (when testing against Kubernetes master),
|
||||
# must be recent enough for Kubernetes master
|
||||
# - running on linux-amd64
|
||||
# - kind (https://github.com/kubernetes-sigs/kind) installed
|
||||
# - optional: Go already installed
|
||||
@@ -65,7 +63,22 @@ get_versioned_variable () {
|
||||
echo "$value"
|
||||
}
|
||||
|
||||
configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
|
||||
# This takes a version string like CSI_PROW_KUBERNETES_VERSION and
|
||||
# maps it to the corresponding git tag, branch or commit.
|
||||
version_to_git () {
|
||||
version="$1"
|
||||
shift
|
||||
case "$version" in
|
||||
latest|master) echo "master";;
|
||||
release-*) echo "$version";;
|
||||
*) echo "v$version";;
|
||||
esac
|
||||
}
|
||||
|
||||
# the list of windows versions was matched from:
|
||||
# - https://hub.docker.com/_/microsoft-windows-nanoserver
|
||||
# - https://hub.docker.com/_/microsoft-windows-servercore
|
||||
configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm -arm; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2; windows amd64 .exe nanoserver:ltsc2022 servercore:ltsc2022" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
|
||||
|
||||
# If we have a vendor directory, then use it. We must be careful to only
|
||||
# use this for "make" invocations inside the project's repo itself because
|
||||
@@ -73,37 +86,12 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64
|
||||
# which is disabled with GOFLAGS=-mod=vendor).
|
||||
configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory"
|
||||
|
||||
# Go versions can be specified seperately for different tasks
|
||||
# If the pre-installed Go is missing or a different
|
||||
# version, the required version here will get installed
|
||||
# from https://golang.org/dl/.
|
||||
go_from_travis_yml () {
|
||||
grep "^ *- go:" "${RELEASE_TOOLS_ROOT}/travis.yml" | sed -e 's/.*go: *//'
|
||||
}
|
||||
configvar CSI_PROW_GO_VERSION_BUILD "$(go_from_travis_yml)" "Go version for building the component" # depends on component's source code
|
||||
configvar CSI_PROW_GO_VERSION_BUILD "1.16" "Go version for building the component" # depends on component's source code
|
||||
configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e
|
||||
configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below
|
||||
configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below
|
||||
configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building ginkgo" # depends on CSI_PROW_GINKGO_VERSION below
|
||||
|
||||
# kind version to use. If the pre-installed version is different,
|
||||
# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases
|
||||
# (if available), otherwise it is built from source.
|
||||
configvar CSI_PROW_KIND_VERSION "v0.9.0" "kind"
|
||||
|
||||
# kind images to use. Must match the kind version.
|
||||
# The release notes of each kind release list the supported images.
|
||||
configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.19.1@sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600
|
||||
kindest/node:v1.18.8@sha256:f4bcc97a0ad6e7abaf3f643d890add7efe6ee4ab90baeb374b4f41a4c95567eb
|
||||
kindest/node:v1.17.11@sha256:5240a7a2c34bf241afb54ac05669f8a46661912eab05705d660971eeb12f6555
|
||||
kindest/node:v1.16.15@sha256:a89c771f7de234e6547d43695c7ab047809ffc71a0c3b65aa54eda051c45ed20
|
||||
kindest/node:v1.15.12@sha256:d9b939055c1e852fe3d86955ee24976cab46cba518abcb8b13ba70917e6547a6
|
||||
kindest/node:v1.14.10@sha256:ce4355398a704fca68006f8a29f37aafb49f8fc2f64ede3ccd0d9198da910146
|
||||
kindest/node:v1.13.12@sha256:1c1a48c2bfcbae4d5f4fa4310b5ed10756facad0b7a2ca93c7a4b5bae5db29f5" "kind images"
|
||||
|
||||
# Use kind node-image --type=bazel by default, but allow to disable that.
|
||||
configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation"
|
||||
|
||||
# ginkgo test runner version to use. If the pre-installed version is
|
||||
# different, the desired version is built from source.
|
||||
configvar CSI_PROW_GINKGO_VERSION v1.7.0 "Ginkgo"
|
||||
@@ -140,10 +128,36 @@ configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes"
|
||||
# when a Prow job just defines the Kubernetes version.
|
||||
csi_prow_kubernetes_version_suffix="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | tr . _ | tr '[:lower:]' '[:upper:]' | sed -e 's/^RELEASE-//' -e 's/\([0-9]*\)_\([0-9]*\).*/\1_\2/')"
|
||||
|
||||
# Work directory. It has to allow running executables, therefore /tmp
|
||||
# is avoided. Cleaning up after the script is intentionally left to
|
||||
# the caller.
|
||||
configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory"
|
||||
# Only the latest KinD is (eventually) guaranteed to work with the
|
||||
# latest Kubernetes. For example, KinD 0.10.0 failed with Kubernetes
|
||||
# 1.21.0-beta1. Therefore the default version of KinD is "main"
|
||||
# for that, otherwise the latest stable release for which we then
|
||||
# list the officially supported images below.
|
||||
kind_version_default () {
|
||||
case "${CSI_PROW_KUBERNETES_VERSION}" in
|
||||
latest|master)
|
||||
echo main;;
|
||||
*)
|
||||
echo v0.11.1;;
|
||||
esac
|
||||
}
|
||||
|
||||
# kind version to use. If the pre-installed version is different,
|
||||
# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases
|
||||
# (if available), otherwise it is built from source.
|
||||
configvar CSI_PROW_KIND_VERSION "$(kind_version_default)" "kind"
|
||||
|
||||
# kind images to use. Must match the kind version.
|
||||
# The release notes of each kind release list the supported images.
|
||||
configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.22.0@sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047
|
||||
kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6
|
||||
kindest/node:v1.20.7@sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9
|
||||
kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729
|
||||
kindest/node:v1.18.19@sha256:7af1492e19b3192a79f606e43c35fb741e520d195f96399284515f077b3b622c
|
||||
kindest/node:v1.17.17@sha256:66f1d0d91a88b8a001811e2f1054af60eef3b669a9a74f9b6db871f2f1eeed00
|
||||
kindest/node:v1.16.15@sha256:83067ed51bf2a3395b24687094e283a7c7c865ccc12a8b1d7aa673ba0c5e8861
|
||||
kindest/node:v1.15.12@sha256:b920920e1eda689d9936dfcf7332701e80be12566999152626b2c9d730397a95
|
||||
kindest/node:v1.14.10@sha256:f8a66ef82822ab4f7569e91a5bccaf27bceee135c1457c512e54de8c6f7219f8" "kind images"
|
||||
|
||||
# By default, this script tests sidecars with the CSI hostpath driver,
|
||||
# using the install_csi_driver function. That function depends on
|
||||
@@ -171,8 +185,8 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip
|
||||
# CSI_PROW_DEPLOYMENT variable can be set in the
|
||||
# .prow.sh of each component when there are breaking changes
|
||||
# that require using a non-default deployment. The default
|
||||
# is a deployment named "kubernetes-x.yy" (if available),
|
||||
# otherwise "kubernetes-latest".
|
||||
# is a deployment named "kubernetes-x.yy${CSI_PROW_DEPLOYMENT_SUFFIX}" (if available),
|
||||
# otherwise "kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}".
|
||||
# "none" disables the deployment of the hostpath driver.
|
||||
#
|
||||
# When no deploy script is found (nothing in `deploy` directory,
|
||||
@@ -184,6 +198,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip
|
||||
configvar CSI_PROW_DRIVER_VERSION "v1.3.0" "CSI driver version"
|
||||
configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo"
|
||||
configvar CSI_PROW_DEPLOYMENT "" "deployment"
|
||||
configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files"
|
||||
|
||||
# The install_csi_driver function may work also for other CSI drivers,
|
||||
# as long as they follow the conventions of the CSI hostpath driver.
|
||||
@@ -208,16 +223,7 @@ configvar CSI_PROW_DRIVER_CANARY_REGISTRY "gcr.io/k8s-staging-sig-storage" "regi
|
||||
# all generated files are present.
|
||||
#
|
||||
# CSI_PROW_E2E_REPO=none disables E2E testing.
|
||||
tag_from_version () {
|
||||
version="$1"
|
||||
shift
|
||||
case "$version" in
|
||||
latest) echo "master";;
|
||||
release-*) echo "$version";;
|
||||
*) echo "v$version";;
|
||||
esac
|
||||
}
|
||||
configvar CSI_PROW_E2E_VERSION "$(tag_from_version "${CSI_PROW_KUBERNETES_VERSION}")" "E2E version"
|
||||
configvar CSI_PROW_E2E_VERSION "$(version_to_git "${CSI_PROW_KUBERNETES_VERSION}")" "E2E version"
|
||||
configvar CSI_PROW_E2E_REPO "https://github.com/kubernetes/kubernetes" "E2E repo"
|
||||
configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package"
|
||||
|
||||
@@ -227,8 +233,8 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package"
|
||||
# of the cluster. The alternative would have been to (cross-)compile csi-sanity
|
||||
# and install it inside the cluster, which is not necessarily easier.
|
||||
configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo"
|
||||
configvar CSI_PROW_SANITY_VERSION 5421d9f3c37be3b95b241b44a094a3db11bee789 "csi-test version" # latest master
|
||||
configvar CSI_PROW_SANITY_IMPORT_PATH github.com/kubernetes-csi/csi-test "csi-test package"
|
||||
configvar CSI_PROW_SANITY_VERSION v4.2.0 "csi-test version"
|
||||
configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package"
|
||||
configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock"
|
||||
configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver"
|
||||
configvar CSI_PROW_SANITY_CONTAINER "hostpath" "Kubernetes container with CSI driver"
|
||||
@@ -287,13 +293,26 @@ tests_need_alpha_cluster () {
|
||||
tests_enabled "parallel-alpha" "serial-alpha"
|
||||
}
|
||||
|
||||
# Enabling mock tests adds the "CSI mock volume" tests from https://github.com/kubernetes/kubernetes/blob/HEAD/test/e2e/storage/csi_mock_volume.go
|
||||
# to the e2e.test invocations (serial, parallel, and the corresponding alpha variants).
|
||||
# When testing canary images, those get used instead of the images specified
|
||||
# in the e2e.test's normal YAML files.
|
||||
#
|
||||
# The default is to enable this for all jobs which use canary images
|
||||
# and the latest Kubernetes because those images will be used for mock
|
||||
# testing once they are released. Using them for mock testing with
|
||||
# older Kubernetes releases is too risky because the deployment files
|
||||
# can be very old (for example, still using a removed -provisioner
|
||||
# parameter in external-provisioner).
|
||||
configvar CSI_PROW_E2E_MOCK "$(if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ] && [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ]; then echo true; else echo false; fi)" "enable CSI mock volume tests"
|
||||
|
||||
# Regex for non-alpha, feature-tagged tests that should be run.
|
||||
#
|
||||
configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for latest Kubernetes version"
|
||||
configvar CSI_PROW_E2E_FOCUS "$(get_versioned_variable CSI_PROW_E2E_FOCUS "${csi_prow_kubernetes_version_suffix}")" "non-alpha, feature-tagged tests"
|
||||
|
||||
# Serial vs. parallel is always determined by these regular expressions.
|
||||
# Individual regular expressions are seperated by spaces for readability
|
||||
# Individual regular expressions are separated by spaces for readability
|
||||
# and expected to not contain spaces. Use dots instead. The complete
|
||||
# regex for Ginkgo will be created by joining the individual terms.
|
||||
configvar CSI_PROW_E2E_SERIAL '\[Serial\] \[Disruptive\]' "tags for serial E2E tests"
|
||||
@@ -345,16 +364,25 @@ configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external
|
||||
# whether they can run with the current cluster provider, but until
|
||||
# they are, we filter them out by name. Like the other test selection
|
||||
# variables, this is again a space separated list of regular expressions.
|
||||
#
|
||||
# "different node" test skips can be removed once
|
||||
# https://github.com/kubernetes/kubernetes/pull/82678 has been backported
|
||||
# to all the K8s versions we test against
|
||||
configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped"
|
||||
configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped"
|
||||
|
||||
# This is the directory for additional result files. Usually set by Prow, but
|
||||
# if not (for example, when invoking manually) it defaults to the work directory.
|
||||
configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts"
|
||||
mkdir -p "${ARTIFACTS}"
|
||||
# This creates directories that are required for testing.
|
||||
ensure_paths () {
|
||||
# Work directory. It has to allow running executables, therefore /tmp
|
||||
# is avoided. Cleaning up after the script is intentionally left to
|
||||
# the caller.
|
||||
configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory"
|
||||
|
||||
# This is the directory for additional result files. Usually set by Prow, but
|
||||
# if not (for example, when invoking manually) it defaults to the work directory.
|
||||
configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts"
|
||||
mkdir -p "${ARTIFACTS}"
|
||||
|
||||
# For additional tools.
|
||||
CSI_PROW_BIN="${CSI_PROW_WORK}/bin"
|
||||
mkdir -p "${CSI_PROW_BIN}"
|
||||
PATH="${CSI_PROW_BIN}:$PATH"
|
||||
}
|
||||
|
||||
run () {
|
||||
echo "$(date) $(go version | sed -e 's/.*version \(go[^ ]*\).*/\1/') $(if [ "$(pwd)" != "${REPO_DIR}" ]; then pwd; fi)\$" "$@" >&2
|
||||
@@ -374,11 +402,6 @@ die () {
|
||||
exit 1
|
||||
}
|
||||
|
||||
# For additional tools.
|
||||
CSI_PROW_BIN="${CSI_PROW_WORK}/bin"
|
||||
mkdir -p "${CSI_PROW_BIN}"
|
||||
PATH="${CSI_PROW_BIN}:$PATH"
|
||||
|
||||
# Ensure that PATH has the desired version of the Go tools, then run command given as argument.
|
||||
# Empty parameter uses the already installed Go. In Prow, that version is kept up-to-date by
|
||||
# bumping the container image regularly.
|
||||
@@ -407,7 +430,7 @@ install_kind () {
|
||||
chmod u+x "${CSI_PROW_WORK}/bin/kind"
|
||||
else
|
||||
git_checkout https://github.com/kubernetes-sigs/kind "${GOPATH}/src/sigs.k8s.io/kind" "${CSI_PROW_KIND_VERSION}" --depth=1 &&
|
||||
(cd "${GOPATH}/src/sigs.k8s.io/kind" && make install INSTALL_DIR="${CSI_PROW_WORK}/bin")
|
||||
(cd "${GOPATH}/src/sigs.k8s.io/kind" && run_with_go "$CSI_PROW_GO_VERSION_KIND" make install INSTALL_DIR="${CSI_PROW_WORK}/bin")
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -465,20 +488,22 @@ git_checkout () {
|
||||
|
||||
# This clones a repo ("https://github.com/kubernetes/kubernetes")
|
||||
# in a certain location ("$GOPATH/src/k8s.io/kubernetes") at
|
||||
# a the head of a specific branch (i.e., release-1.13, master).
|
||||
# The directory cannot exist.
|
||||
git_clone_branch () {
|
||||
local repo path branch parent
|
||||
# a the head of a specific branch (i.e., release-1.13, master),
|
||||
# tag (v1.20.0) or commit.
|
||||
#
|
||||
# The directory must not exist.
|
||||
git_clone () {
|
||||
local repo path name parent
|
||||
repo="$1"
|
||||
shift
|
||||
path="$1"
|
||||
shift
|
||||
branch="$1"
|
||||
name="$1"
|
||||
shift
|
||||
|
||||
parent="$(dirname "$path")"
|
||||
mkdir -p "$parent"
|
||||
(cd "$parent" && run git clone --single-branch --branch "$branch" "$repo" "$path") || die "cloning $repo" failed
|
||||
(cd "$parent" && run git clone --single-branch --branch "$name" "$repo" "$path") || die "cloning $repo" failed
|
||||
# This is useful for local testing or when switching between different revisions in the same
|
||||
# repo.
|
||||
(cd "$path" && run git clean -fdx) || die "failed to clean $path"
|
||||
@@ -562,16 +587,12 @@ start_cluster () {
|
||||
if [ "$version" = "latest" ]; then
|
||||
version=master
|
||||
fi
|
||||
if ${CSI_PROW_USE_BAZEL}; then
|
||||
type="bazel"
|
||||
else
|
||||
type="docker"
|
||||
fi
|
||||
git_clone_branch https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version" || die "checking out Kubernetes $version failed"
|
||||
git_clone https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$(version_to_git "$version")" || die "checking out Kubernetes $version failed"
|
||||
|
||||
go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes"
|
||||
# Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910
|
||||
(cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --type="$type" --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed"
|
||||
# shellcheck disable=SC2046
|
||||
(cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed"
|
||||
csi_prow_kind_have_kubernetes=true
|
||||
fi
|
||||
image="csiprow/node:latest"
|
||||
@@ -605,11 +626,16 @@ EOF
|
||||
|
||||
# Deletes kind cluster inside a prow job
|
||||
delete_cluster_inside_prow_job() {
|
||||
local name="$1"
|
||||
|
||||
# Inside a real Prow job it is better to clean up at runtime
|
||||
# instead of leaving that to the Prow job cleanup code
|
||||
# because the later sometimes times out (https://github.com/kubernetes-csi/csi-release-tools/issues/24#issuecomment-554765872).
|
||||
#
|
||||
# This is also a good time to collect logs.
|
||||
if [ "$JOB_NAME" ]; then
|
||||
if kind get clusters | grep -q csi-prow; then
|
||||
run kind export logs --name=csi-prow "${ARTIFACTS}/cluster-logs/$name"
|
||||
run kind delete cluster --name=csi-prow || die "kind delete failed"
|
||||
fi
|
||||
unset KUBECONFIG
|
||||
@@ -619,24 +645,38 @@ delete_cluster_inside_prow_job() {
|
||||
# Looks for the deployment as specified by CSI_PROW_DEPLOYMENT and CSI_PROW_KUBERNETES_VERSION
|
||||
# in the given directory.
|
||||
find_deployment () {
|
||||
local dir file
|
||||
dir="$1"
|
||||
|
||||
# Fixed deployment name? Use it if it exists, otherwise fail.
|
||||
if [ "${CSI_PROW_DEPLOYMENT}" ]; then
|
||||
file="$dir/${CSI_PROW_DEPLOYMENT}/deploy.sh"
|
||||
if ! [ -e "$file" ]; then
|
||||
return 1
|
||||
fi
|
||||
echo "$file"
|
||||
return 0
|
||||
fi
|
||||
local dir="$1"
|
||||
local file
|
||||
|
||||
# major/minor without release- prefix.
|
||||
local k8sver
|
||||
# Ignore: See if you can use ${variable//search/replace} instead.
|
||||
# shellcheck disable=SC2001
|
||||
file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy.sh"
|
||||
k8sver="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/^release-//' -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')"
|
||||
|
||||
# Desired deployment, either specified completely, including version, or derived from other variables.
|
||||
local deployment
|
||||
deployment=${CSI_PROW_DEPLOYMENT:-kubernetes-${k8sver}${CSI_PROW_DEPLOYMENT_SUFFIX}}
|
||||
|
||||
# Fixed deployment name? Use it if it exists.
|
||||
if [ "${CSI_PROW_DEPLOYMENT}" ]; then
|
||||
file="$dir/${CSI_PROW_DEPLOYMENT}/deploy.sh"
|
||||
if [ -e "$file" ]; then
|
||||
echo "$file"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# CSI_PROW_DEPLOYMENT=kubernetes-x.yy must be mapped to kubernetes-latest
|
||||
# as fallback. Same for kubernetes-distributed-x.yy.
|
||||
fi
|
||||
|
||||
file="$dir/${deployment}/deploy.sh"
|
||||
if ! [ -e "$file" ]; then
|
||||
file="$dir/kubernetes-latest/deploy.sh"
|
||||
# Replace the first xx.yy number with "latest", for example
|
||||
# kubernetes-1.21-test -> kubernetes-latest-test.
|
||||
# Ignore: See if you can use ${variable//search/replace} instead.
|
||||
# shellcheck disable=SC2001
|
||||
file="$dir/$(echo "$deployment" | sed -e 's/[0-9][0-9]*\.[0-9][0-9]*/latest/')/deploy.sh"
|
||||
if ! [ -e "$file" ]; then
|
||||
return 1
|
||||
fi
|
||||
@@ -696,7 +736,7 @@ install_csi_driver () {
|
||||
fi
|
||||
}
|
||||
|
||||
# Installs all nessesary snapshotter CRDs
|
||||
# Installs all nessesary snapshotter CRDs
|
||||
install_snapshot_crds() {
|
||||
# Wait until volumesnapshot CRDs are in place.
|
||||
CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd"
|
||||
@@ -743,7 +783,7 @@ install_snapshot_controller() {
|
||||
fi
|
||||
echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt"
|
||||
cnt=$((cnt + 1))
|
||||
sleep 10
|
||||
sleep 10
|
||||
done
|
||||
|
||||
SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml"
|
||||
@@ -756,7 +796,7 @@ install_snapshot_controller() {
|
||||
kind load docker-image --name csi-prow ${NEW_IMG} || die "could not load the snapshot-controller:csiprow image into the kind cluster"
|
||||
|
||||
# deploy snapshot-controller
|
||||
echo "Deploying snapshot-controller"
|
||||
echo "Deploying snapshot-controller from ${SNAPSHOT_CONTROLLER_YAML} with $NEW_IMG."
|
||||
# Replace image in SNAPSHOT_CONTROLLER_YAML with snapshot-controller:csiprow and deploy
|
||||
# NOTE: This logic is similar to the logic here:
|
||||
# https://github.com/kubernetes-csi/csi-driver-host-path/blob/v1.4.0/deploy/util/deploy-hostpath.sh#L155
|
||||
@@ -793,25 +833,37 @@ install_snapshot_controller() {
|
||||
echo "$modified"
|
||||
exit 1
|
||||
fi
|
||||
echo "kubectl apply -f ${SNAPSHOT_CONTROLLER_YAML}(modified)"
|
||||
done
|
||||
elif [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then
|
||||
echo "Deploying snapshot-controller from ${SNAPSHOT_CONTROLLER_YAML} with canary images."
|
||||
yaml="$(kubectl apply --dry-run=client -o yaml -f "$SNAPSHOT_CONTROLLER_YAML")"
|
||||
# Ignore: See if you can use ${variable//search/replace} instead.
|
||||
# shellcheck disable=SC2001
|
||||
modified="$(echo "$yaml" | sed -e "s;image: .*/\([^/:]*\):.*;image: ${CSI_PROW_DRIVER_CANARY_REGISTRY}/\1:canary;")"
|
||||
diff <(echo "$yaml") <(echo "$modified")
|
||||
if ! echo "$modified" | kubectl apply -f -; then
|
||||
echo "modified version of $SNAPSHOT_CONTROLLER_YAML:"
|
||||
echo "$modified"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "kubectl apply -f ${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml"
|
||||
kubectl apply -f "${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml"
|
||||
echo "kubectl apply -f $SNAPSHOT_CONTROLLER_YAML"
|
||||
kubectl apply -f "$SNAPSHOT_CONTROLLER_YAML"
|
||||
fi
|
||||
|
||||
cnt=0
|
||||
expected_running_pods=$(curl https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${CSI_SNAPSHOTTER_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml | grep replicas | cut -d ':' -f 2-)
|
||||
while [ "$(kubectl get pods -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do
|
||||
expected_running_pods=$(kubectl apply --dry-run=client -o "jsonpath={.spec.replicas}" -f "$SNAPSHOT_CONTROLLER_YAML")
|
||||
expected_namespace=$(kubectl apply --dry-run=client -o "jsonpath={.metadata.namespace}" -f "$SNAPSHOT_CONTROLLER_YAML")
|
||||
while [ "$(kubectl get pods -n "$expected_namespace" -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do
|
||||
if [ $cnt -gt 30 ]; then
|
||||
echo "snapshot-controller pod status:"
|
||||
kubectl describe pods -l app=snapshot-controller
|
||||
kubectl describe pods -n "$expected_namespace" -l app=snapshot-controller
|
||||
echo >&2 "ERROR: snapshot controller not ready after over 5 min"
|
||||
exit 1
|
||||
fi
|
||||
echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt"
|
||||
cnt=$((cnt + 1))
|
||||
sleep 10
|
||||
sleep 10
|
||||
done
|
||||
}
|
||||
|
||||
@@ -856,6 +908,29 @@ start_loggers () {
|
||||
done
|
||||
}
|
||||
|
||||
# Patches the image versions of test/e2e/testing-manifests/storage-csi/mock in the k/k
|
||||
# source code, if needed.
|
||||
patch_kubernetes () {
|
||||
local source="$1" target="$2"
|
||||
|
||||
if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then
|
||||
# We cannot replace k8s.gcr.io/sig-storage with gcr.io/k8s-staging-sig-storage because
|
||||
# e2e.test does not support it (see test/utils/image/manifest.go). Instead we
|
||||
# invoke the e2e.test binary with KUBE_TEST_REPO_LIST set to a file that
|
||||
# overrides that registry.
|
||||
find "$source/test/e2e/testing-manifests/storage-csi/mock" -name '*.yaml' -print0 | xargs -0 sed -i -e 's;k8s.gcr.io/sig-storage/\(.*\):v.*;k8s.gcr.io/sig-storage/\1:canary;'
|
||||
cat >"$target/e2e-repo-list" <<EOF
|
||||
sigStorageRegistry: gcr.io/k8s-staging-sig-storage
|
||||
EOF
|
||||
cat >&2 <<EOF
|
||||
|
||||
Using a modified version of k/k/test/e2e:
|
||||
$(cd "$source" && git diff 2>&1)
|
||||
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
# Makes the E2E test suite binary available as "${CSI_PROW_WORK}/e2e.test".
|
||||
install_e2e () {
|
||||
if [ -e "${CSI_PROW_WORK}/e2e.test" ]; then
|
||||
@@ -864,6 +939,7 @@ install_e2e () {
|
||||
|
||||
git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1 &&
|
||||
if [ "${CSI_PROW_E2E_IMPORT_PATH}" = "k8s.io/kubernetes" ]; then
|
||||
patch_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_WORK}" &&
|
||||
go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}")}" &&
|
||||
run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
|
||||
ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}"
|
||||
@@ -879,8 +955,8 @@ install_sanity () (
|
||||
return
|
||||
fi
|
||||
|
||||
git_checkout "${CSI_PROW_SANITY_REPO}" "${GOPATH}/src/${CSI_PROW_SANITY_IMPORT_PATH}" "${CSI_PROW_SANITY_VERSION}" --depth=1 || die "checking out csi-sanity failed"
|
||||
run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed"
|
||||
git_checkout "${CSI_PROW_SANITY_REPO}" "${GOPATH}/src/${CSI_PROW_SANITY_PACKAGE_PATH}" "${CSI_PROW_SANITY_VERSION}" --depth=1 || die "checking out csi-sanity failed"
|
||||
( cd "${GOPATH}/src/${CSI_PROW_SANITY_PACKAGE_PATH}/cmd/csi-sanity" && run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go build -o "${CSI_PROW_WORK}/csi-sanity" ) || die "building csi-sanity failed"
|
||||
)
|
||||
|
||||
# Captures pod output while running some other command.
|
||||
@@ -909,25 +985,33 @@ run_e2e () (
|
||||
# the full Kubernetes E2E testsuite while only running a few tests.
|
||||
move_junit () {
|
||||
if ls "${ARTIFACTS}"/junit_[0-9]*.xml 2>/dev/null >/dev/null; then
|
||||
run_filter_junit -t="External Storage" -o "${ARTIFACTS}/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml && rm -f "${ARTIFACTS}"/junit_[0-9]*.xml
|
||||
run_filter_junit -t="External.Storage|CSI.mock.volume" -o "${ARTIFACTS}/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml && rm -f "${ARTIFACTS}"/junit_[0-9]*.xml
|
||||
fi
|
||||
}
|
||||
trap move_junit EXIT
|
||||
|
||||
cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
|
||||
run_with_loggers ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml"
|
||||
run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml"
|
||||
)
|
||||
|
||||
# Run csi-sanity against installed CSI driver.
|
||||
run_sanity () (
|
||||
install_sanity || die "installing csi-sanity failed"
|
||||
|
||||
if [[ "${CSI_PROW_SANITY_POD}" =~ " " ]]; then
|
||||
# Contains spaces, more complex than a simple pod name.
|
||||
# Evaluate as a shell command.
|
||||
pod=$(eval "${CSI_PROW_SANITY_POD}") || die "evaluation failed: CSI_PROW_SANITY_POD=${CSI_PROW_SANITY_POD}"
|
||||
else
|
||||
pod="${CSI_PROW_SANITY_POD}"
|
||||
fi
|
||||
|
||||
cat >"${CSI_PROW_WORK}/mkdir_in_pod.sh" <<EOF
|
||||
#!/bin/sh
|
||||
kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- mkdir "\$@" && echo "\$@"
|
||||
kubectl exec "$pod" -c "${CSI_PROW_SANITY_CONTAINER}" -- mkdir "\$@" && echo "\$@"
|
||||
EOF
|
||||
# Using "rm -rf" as fallback for "rmdir" is a workaround for:
|
||||
# Node Service
|
||||
# Node Service
|
||||
# should work
|
||||
# /nvme/gopath.tmp/src/github.com/kubernetes-csi/csi-test/pkg/sanity/node.go:624
|
||||
# STEP: reusing connection to CSI driver at dns:///172.17.0.2:30896
|
||||
@@ -949,11 +1033,29 @@ EOF
|
||||
# why it happened.
|
||||
cat >"${CSI_PROW_WORK}/rmdir_in_pod.sh" <<EOF
|
||||
#!/bin/sh
|
||||
if ! kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- rmdir "\$@"; then
|
||||
kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- rm -rf "\$@"
|
||||
if ! kubectl exec "$pod" -c "${CSI_PROW_SANITY_CONTAINER}" -- rmdir "\$@"; then
|
||||
kubectl exec "$pod" -c "${CSI_PROW_SANITY_CONTAINER}" -- rm -rf "\$@"
|
||||
exit 1
|
||||
fi
|
||||
EOF
|
||||
|
||||
cat >"${CSI_PROW_WORK}/checkdir_in_pod.sh" <<EOF
|
||||
#!/bin/sh
|
||||
CHECK_PATH=\$(cat <<SCRIPT
|
||||
if [ -f "\$@" ]; then
|
||||
echo "file"
|
||||
elif [ -d "\$@" ]; then
|
||||
echo "directory"
|
||||
elif [ -e "\$@" ]; then
|
||||
echo "other"
|
||||
else
|
||||
echo "not_found"
|
||||
fi
|
||||
SCRIPT
|
||||
)
|
||||
kubectl exec "$pod" -c "${CSI_PROW_SANITY_CONTAINER}" -- /bin/sh -c "\${CHECK_PATH}"
|
||||
EOF
|
||||
|
||||
chmod u+x "${CSI_PROW_WORK}"/*dir_in_pod.sh
|
||||
|
||||
# This cannot run in parallel, because -csi.junitfile output
|
||||
@@ -969,6 +1071,7 @@ EOF
|
||||
-csi.createmountpathcmd "${CSI_PROW_WORK}/mkdir_in_pod.sh" \
|
||||
-csi.removestagingpathcmd "${CSI_PROW_WORK}/rmdir_in_pod.sh" \
|
||||
-csi.removemountpathcmd "${CSI_PROW_WORK}/rmdir_in_pod.sh" \
|
||||
-csi.checkpathcmd "${CSI_PROW_WORK}/checkdir_in_pod.sh" \
|
||||
)
|
||||
|
||||
ascii_to_xml () {
|
||||
@@ -998,7 +1101,7 @@ make_test_to_junit () {
|
||||
echo "$line" # pass through
|
||||
if echo "$line" | grep -q "^### [^ ]*:$"; then
|
||||
if [ "$testname" ]; then
|
||||
# previous test succesful
|
||||
# previous test successful
|
||||
echo " </system-out>" >>"$out"
|
||||
echo " </testcase>" >>"$out"
|
||||
fi
|
||||
@@ -1072,7 +1175,7 @@ make_test_to_junit () {
|
||||
# version_gt 1.3.1 v1.2.0 (returns true)
|
||||
# version_gt 1.1.1 release-1.2.0 (returns false)
|
||||
# version_gt 1.2.0 1.2.2 (returns false)
|
||||
function version_gt() {
|
||||
function version_gt() {
|
||||
versions=$(for ver in "$@"; do ver=${ver#release-}; ver=${ver#kubernetes-}; echo "${ver#v}"; done)
|
||||
greaterVersion=${1#"release-"};
|
||||
greaterVersion=${greaterVersion#"kubernetes-"};
|
||||
@@ -1084,6 +1187,9 @@ main () {
|
||||
local images ret
|
||||
ret=0
|
||||
|
||||
# Set up work directory.
|
||||
ensure_paths
|
||||
|
||||
images=
|
||||
if ${CSI_PROW_BUILD_JOB}; then
|
||||
# A successful build is required for testing.
|
||||
@@ -1132,7 +1238,7 @@ main () {
|
||||
if [ "$rbac_file_path" == "" ]; then
|
||||
rbac_file_path="$(pwd)/deploy/kubernetes/rbac.yaml"
|
||||
fi
|
||||
|
||||
|
||||
if [ -e "$rbac_file_path" ]; then
|
||||
# This is one of those components which has its own RBAC rules (like external-provisioner).
|
||||
# We are testing a locally built image and also want to test with the the current,
|
||||
@@ -1143,6 +1249,12 @@ main () {
|
||||
done
|
||||
fi
|
||||
|
||||
# Run the external driver tests and optionally also mock tests.
|
||||
local focus="External.Storage"
|
||||
if "$CSI_PROW_E2E_MOCK"; then
|
||||
focus="($focus|CSI.mock.volume)"
|
||||
fi
|
||||
|
||||
if tests_need_non_alpha_cluster; then
|
||||
start_cluster || die "starting the non-alpha cluster failed"
|
||||
|
||||
@@ -1150,6 +1262,7 @@ main () {
|
||||
install_snapshot_crds
|
||||
install_snapshot_controller
|
||||
|
||||
|
||||
# Installing the driver might be disabled.
|
||||
if ${CSI_PROW_DRIVER_INSTALL} "$images"; then
|
||||
collect_cluster_info
|
||||
@@ -1164,7 +1277,7 @@ main () {
|
||||
# Ignore: Double quote to prevent globbing and word splitting.
|
||||
# shellcheck disable=SC2086
|
||||
if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \
|
||||
-focus="External.Storage" \
|
||||
-focus="$focus" \
|
||||
-skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then
|
||||
warn "E2E parallel failed"
|
||||
ret=1
|
||||
@@ -1174,7 +1287,7 @@ main () {
|
||||
# Ignore: Double quote to prevent globbing and word splitting.
|
||||
# shellcheck disable=SC2086
|
||||
if ! run_e2e parallel-features ${CSI_PROW_GINKO_PARALLEL} \
|
||||
-focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \
|
||||
-focus="$focus.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \
|
||||
-skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then
|
||||
warn "E2E parallel features failed"
|
||||
ret=1
|
||||
@@ -1183,14 +1296,14 @@ main () {
|
||||
|
||||
if tests_enabled "serial"; then
|
||||
if ! run_e2e serial \
|
||||
-focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \
|
||||
-focus="$focus.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \
|
||||
-skip="$(regex_join "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then
|
||||
warn "E2E serial failed"
|
||||
ret=1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
delete_cluster_inside_prow_job
|
||||
delete_cluster_inside_prow_job non-alpha
|
||||
fi
|
||||
|
||||
if tests_need_alpha_cluster && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then
|
||||
@@ -1209,7 +1322,7 @@ main () {
|
||||
# Ignore: Double quote to prevent globbing and word splitting.
|
||||
# shellcheck disable=SC2086
|
||||
if ! run_e2e parallel-alpha ${CSI_PROW_GINKO_PARALLEL} \
|
||||
-focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \
|
||||
-focus="$focus.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \
|
||||
-skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_SKIP}")"; then
|
||||
warn "E2E parallel alpha failed"
|
||||
ret=1
|
||||
@@ -1218,14 +1331,14 @@ main () {
|
||||
|
||||
if tests_enabled "serial-alpha"; then
|
||||
if ! run_e2e serial-alpha \
|
||||
-focus="External.Storage.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \
|
||||
-focus="$focus.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \
|
||||
-skip="$(regex_join "${CSI_PROW_E2E_SKIP}")"; then
|
||||
warn "E2E serial alpha failed"
|
||||
ret=1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
delete_cluster_inside_prow_job
|
||||
delete_cluster_inside_prow_job alpha
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -1245,6 +1358,9 @@ gcr_cloud_build () {
|
||||
# Required for "docker buildx build --push".
|
||||
gcloud auth configure-docker
|
||||
|
||||
# Might not be needed here, but call it just in case.
|
||||
ensure_paths
|
||||
|
||||
if find . -name Dockerfile | grep -v ^./vendor | xargs --no-run-if-empty cat | grep -q ^RUN; then
|
||||
# Needed for "RUN" steps on non-linux/amd64 platforms.
|
||||
# See https://github.com/multiarch/qemu-user-static#getting-started
|
||||
|
||||
Reference in New Issue
Block a user