test/system: Decouple image caching from Zuul

Since the rewrite of the system test suite[0] we've relied on the Zuul
playbooks for taking care of caching images using Skopeo for increasing
the reliability of the tests (in the past the instability of the Fedora
registry caused problems). This state is problematic if we want to use
the tests in other environments than the Zuul CI. This moves the caching
from Zuul into the system tests.

Currently, Bats does not support officially suite-wide setup and
teardown functions. The solution I chose was to add two new test files
that are executed before and after all tests. This may complicate the
execution of cherry-picked tests but that is not a very common use case
anyway.

The tests are now to some extent capable of adjusting to the host
environment. This is meant in the sense of: I'm running on RHEL, the
"default image" is UBI; I'm running on Fedora, the "default image" is
fedora-toolbox. This mechanism relies on os-release, which is the same
as what Toolbox itself uses.

[0] https://github.com/containers/toolbox/pull/517

https://github.com/containers/toolbox/pull/774

playbooks/setup-env.yaml

@@ -46,30 +46,3 @@
 
     - name: Show podman debug information
       command: podman info --debug
-
-    - name: Pull the default image from registry.fedoraproject.org
-      command: "podman pull registry.fedoraproject.org/fedora-toolbox:{{ ansible_distribution_version }}"
-      register: _podman
-      until: _podman.rc == 0
-      retries: 5
-      delay: 10
-
-    - name: Copy the default image to a directory
-      command:
-        cmd: "skopeo copy containers-storage:registry.fedoraproject.org/fedora-toolbox:{{ ansible_distribution_version }} dir:{{ zuul.project.src_dir }}/fedora-toolbox-{{ ansible_distribution_version }}"
-        creates: "{{ zuul.project.src_dir }}/fedora-toolbox-{{ ansible_distribution_version }}/manifest.json"
-
-    - name: Pull registry.fedoraproject.org/fedora-toolbox:32
-      command: podman pull registry.fedoraproject.org/fedora-toolbox:32
-      register: _podman
-      until: _podman.rc == 0
-      retries: 5
-      delay: 10
-
-    - name: Copy registry.fedoraproject.org/fedora-toolbox:32 to a directory
-      command:
-        cmd: "skopeo copy containers-storage:registry.fedoraproject.org/fedora-toolbox:32 dir:{{ zuul.project.src_dir }}/fedora-toolbox-32"
-        creates: "{{ zuul.project.src_dir }}/fedora-toolbox-32/manifest.json"
-
-    - name: Clean up the local containers storage
-      command: podman system reset --force

test/system/000-setup.bats

@@ -0,0 +1,11 @@
+#!/usr/bin/env bats
+
+load 'libs/helpers'
+
+@test "test suite: Setup" {
+    # Cache the default image for the system
+    _pull_and_cache_distro_image $(get_system_id) $(get_system_version) || die
+    # Cache all images that will be needed during the tests
+    _pull_and_cache_distro_image fedora 32 || die
+    _pull_and_cache_distro_image busybox || die
+}

test/system/101-create.bats

@@ -22,13 +22,15 @@ teardown() {
 }
 
 @test "create: Create a container with a valid custom name ('custom-containerName')" {
+  pull_default_image
+
   run $TOOLBOX -y create -c "custom-containerName"
 
   assert_success
 }
 
 @test "create: Create a container with a custom image and name ('fedora32'; f32)" {
-  pull_image 32
+  pull_distro_image fedora 32
 
   run $TOOLBOX -y create -c "fedora32" -i fedora-toolbox:32
 
@@ -54,7 +56,7 @@ teardown() {
 }
 
 @test "create: Create a container with a distro and release options ('fedora'; f32)" {
-  pull_image 32
+  pull_distro_image fedora 32
 
   run $TOOLBOX -y create -d "fedora" -r f32
 

test/system/102-list.bats

@@ -35,7 +35,7 @@ teardown() {
 }
 
 @test "list: Run 'list' with zero toolbox's containers and images, but other image (the list should be empty)" {
-  get_busybox_image
+  pull_distro_image busybox
 
   run podman images
 
@@ -50,8 +50,9 @@ teardown() {
 @test "list: Try to list images and containers (no flag) with 3 containers and 2 images (the list should have 3 images and 2 containers)" {
   # Pull the two images
   pull_default_image
-  pull_image 32
+  pull_distro_image fedora 32
-  # Create tree containers
+
+  # Create three containers
   create_default_container
   create_container non-default-one
   create_container non-default-two
@@ -60,14 +61,14 @@ teardown() {
   run $TOOLBOX list --images
 
   assert_success
-  assert_output --partial "fedora-toolbox:${DEFAULT_FEDORA_VERSION}"
+  assert_output --partial "$(get_system_id)-toolbox:$(get_system_version)"
   assert_output --partial "fedora-toolbox:32"
 
   # Check containers
   run $TOOLBOX list --containers
 
   assert_success
-  assert_output --partial "fedora-toolbox-${DEFAULT_FEDORA_VERSION}"
+  assert_output --partial "$(get_system_id)-toolbox-$(get_system_version)"
   assert_output --partial "non-default-one"
   assert_output --partial "non-default-two"
 
@@ -75,9 +76,9 @@ teardown() {
   run $TOOLBOX list
 
   assert_success
-  assert_output --partial "fedora-toolbox:${DEFAULT_FEDORA_VERSION}"
+  assert_output --partial "$(get_system_id)-toolbox:$(get_system_version)"
   assert_output --partial "fedora-toolbox:32"
-  assert_output --partial "fedora-toolbox-${DEFAULT_FEDORA_VERSION}"
+  assert_output --partial "$(get_system_id)-toolbox-$(get_system_version)"
   assert_output --partial "non-default-one"
   assert_output --partial "non-default-two"
 }

test/system/999-teardown.bats

@@ -0,0 +1,7 @@
+#!/usr/bin/env bats
+
+load 'libs/helpers'
+
+@test "test suite: Teardown" {
+  _clean_cached_images
+}

test/system/libs/helpers.bash

@@ -1,16 +1,20 @@
 #!/usr/bin/env bash
 
+load 'libs/bats-support/load'
+
 # Podman and Toolbox commands to run
 readonly PODMAN=${PODMAN:-podman}
 readonly TOOLBOX=${TOOLBOX:-toolbox}
 readonly SKOPEO=$(command -v skopeo)
-readonly PROJECT_DIR=${PWD}
 
 # Helpful globals
-current_os_version=$(awk -F= '/VERSION_ID/ {print $2}' /etc/os-release)
+readonly PROJECT_DIR=${PWD}
-readonly DEFAULT_FEDORA_VERSION=${DEFAULT_FEDORA_VERSION:-${current_os_version}}
+readonly IMAGE_CACHE_DIR="${PROJECT_DIR}/image-cache"
-readonly REGISTRY_URL=${REGISTRY_URL:-"registry.fedoraproject.org"}
+
-readonly BUSYBOX_IMAGE="docker.io/library/busybox"
+# Images
+declare -Ag IMAGES=([busybox]="docker.io/library/busybox" \
+                   [fedora]="registry.fedoraproject.org/fedora-toolbox" \
+                   [rhel]="registry.access.redhat.com/ubi8")
 
 
 function cleanup_all() {
@@ -23,46 +27,176 @@ function cleanup_containers() {
 }
 
 
-function get_busybox_image() {
+# Pulls an image using Podman and saves it to a image dir using Skopeo
-  $PODMAN pull "$BUSYBOX_IMAGE" >/dev/null \
+#
-    || fail "Podman couldn't pull the image."
+# Parameters
+# ==========
+# - distro - os-release field ID (e.g., fedora, rhel)
+# - version - os-release field VERSION_ID (e.g., 33, 34, 8.4)
+#
+# Only use during test suite setup for caching all images to be used throught
+# tests.
+function _pull_and_cache_distro_image() {
+  local num_of_retries=5
+  local timeout=10
+  local pulled=false
+  local distro
+  local version
+  local image
+  local image_archive
+
+  distro="$1"
+  version="$2"
+
+  if [ ! -v IMAGES[$distro] ]; then
+    fail "Requested distro (${distro}) does not have a matching image"
+  fi
+
+  image="${IMAGES[$distro]}"
+  image_archive="${distro}-toolbox"
+
+  if [[ $# -eq 2 ]]; then
+    image="${image}:${version}"
+    image_archive="${image_archive}-${version}"
+  fi
+
+  for ((i = ${num_of_retries}; i > 0; i--)); do
+    run $PODMAN pull ${image}
+
+    if [ "$status" -eq 0 ]; then
+      pulled=true
+      break
+    fi
+
+    sleep $timeout
+  done
+  
+  if !pulled; then
+    echo "Failed to pull image ${image}"
+    assert_success
+  fi
+
+  if [ ! -d ${IMAGE_CACHE_DIR} ]; then
+    mkdir -p ${IMAGE_CACHE_DIR}
+  fi
+
+  run $SKOPEO copy --dest-compress containers-storage:${image} dir:${IMAGE_CACHE_DIR}/${image_archive}
+
+  if [ "$status" -ne 0 ]; then
+    echo "Failed to cache image ${image} to ${IMAGE_CACHE_DIR}/${image_archive}"
+    assert_success
+  fi
+
+  cleanup_all
 }
 
 
-function pull_image() {
+# Removes the folder with cached images
+function _clean_cached_images() {
+  rm -rf ${IMAGE_CACHE_DIR}
+}
+
+
+# Copies an image from local storage to Podman's image store
+# 
+# Call before creating any container. Network failures are not nice.
+#
+# An image has to be cached first. See _pull_and_cache_distro_image()
+#
+# Parameters:
+# ===========
+# - distro - os-release field ID (e.g., fedora, rhel)
+# - version - os-release field VERSION_ID (e.g., 33, 34, 8.4)
+function pull_distro_image() {
+  local distro
   local version
   local image
-  version="$1"
+  local image_archive
-  image="${REGISTRY_URL}/fedora-toolbox:${version}"
+
+  distro="$1"
+  version="$2"
+
+  if [ ! -v IMAGES[$distro] ]; then
+    fail "Requested distro (${distro}) does not have a matching image"
+  fi
+
+  image="${IMAGES[$distro]}"
+  image_archive="${distro}-toolbox"
+
+  if [[ -n $version ]]; then
+    image="${image}:${version}"
+    image_archive="${image_archive}-${version}"
+  fi
+
+  # No need to copy if the image is already available in Podman
+  run $PODMAN image exists ${image}
+  if [[ "$status" -eq 0 ]]; then
+    return
+  fi
+
+  run $SKOPEO copy "dir:${IMAGE_CACHE_DIR}/${image_archive}" "containers-storage:${image}"
+  if [ "$status" -ne 0 ]; then
+    echo "Failed to load image ${image} from cache ${IMAGE_CACHE_DIR}/${image_archive}"
+    assert_success
+  fi
 
-  $SKOPEO copy "dir:${PROJECT_DIR}/fedora-toolbox-${version}" "containers-storage:${image}"
   $PODMAN images
 }
 
 
+# Copies the system's default image to Podman's image store
+#
+# See pull_default_image() for more info.
 function pull_default_image() {
-  pull_image "${DEFAULT_FEDORA_VERSION}"
+  pull_distro_image $(get_system_id) $(get_system_version)
 }
 
 
+# Creates a container with specific name, distro and version
+#
+# Pulling of an image is taken care of by the function
+#
+# Parameters:
+# ===========
+# - distro - os-release field ID (e.g., fedora, rhel)
+# - version - os-release field VERSION_ID (e.g., 33, 34, 8.4)
+# - container_name - name of the container
+function create_distro_container() {
+  local distro
+  local version
+  local container_name
+
+  distro="$1"
+  version="$2"
+  container_name="$3"
+
+  pull_distro_image ${distro} ${version}
+
+  $TOOLBOX --assumeyes create --container "${container_name}" --distro "${distro}" --release "${version}" >/dev/null \
+    || fail "Toolbox couldn't create container '$container_name'"
+}
+
+
+# Creates a container with specific name matching the system
+#
+# Parameters:
+# ===========
+# - container_name - name of the container
 function create_container() {
   local container_name
-  local version
+
-  local image
   container_name="$1"
-  version="$DEFAULT_FEDORA_VERSION"
-  image="${REGISTRY_URL}/fedora-toolbox:${version}"
 
-  pull_image "$version"
+  create_distro_container $(get_system_id) $(get_system_version) $container_name
-
-  $TOOLBOX --assumeyes create --container "$container_name" \
-    --image "$image" >/dev/null \
-    || fail "Toolbox couldn't create the container '$container_name'"
 }
 
 
+# Creates a default container
 function create_default_container() {
-  create_container "fedora-toolbox-${DEFAULT_FEDORA_VERSION}"
+  pull_default_image
+
+  $TOOLBOX --assumeyes create >/dev/null \
+    || fail "Toolbox couldn't create default container"
 }
 
 
@@ -95,3 +229,45 @@ function list_images() {
 function list_containers() {
   $PODMAN ps --all --quiet | wc -l
 }
+
+
+# Returns the path to os-release
+function find_os_release() {
+  if [[ -f "/etc/os-release" ]]; then
+    echo "/etc/os-release"
+  elif [[ -f "/usr/lib/os-release" ]]; then
+    echo "/usr/lib/os-release"
+  else
+    echo ""
+  fi
+}
+
+
+# Returns the content of field ID in os-release
+function get_system_id() {
+    local os_release
+
+    os_release="$(find_os_release)"
+
+    if [[ -z "$os_release" ]]; then
+        echo ""
+        return
+    fi
+
+    echo $(awk -F= '/ID/ {print $2}' $os_release | head -n 1)
+}
+
+
+# Returns the content of field VERSION_ID in os-release
+function get_system_version() {
+    local os_release
+
+    os_release="$(find_os_release)"
+
+    if [[ -z "$os_release" ]]; then
+        echo ""
+        return
+    fi
+
+    echo $(awk -F= '/VERSION_ID/ {print $2}' $os_release | head -n 1)
+}