From b27795a03ec8ae3139e2dad93b4efca13e96fb01 Mon Sep 17 00:00:00 2001 From: Juanje Ojeda Date: Mon, 27 Jul 2020 14:35:17 +0100 Subject: [PATCH] test/system: Refactor tests using bats-support and bats-assert The bats-support[0] and bats-assert[1] libraries extend the capabilities of bats[2]. Mainly, bats-assert is very useful for clean checking of values/outputs/return codes. Apart from updating the cases to use the libraries, the test cases have been restructured in a way that they don't depend on each other anymore. This required major changes in the helpers.bats file. Overall, the tests are cleaner to read and easier to extend due to the test cases being independent. Some slight changes were made to the test cases themselves. Should not alter their final behaviour. There will be a follow up commit that will take care of downloading of the tested images locally and caching them using Skopeo to speedup the tests and try to resolve network problems when pulling the images that we experienced in the past. [0] https://github.com/bats-core/bats-support [1] https://github.com/bats-core/bats-assert [2] https://github.com/bats-core/bats-core --- .gitignore | 2 + playbooks/fedora-31/setup-env.yaml | 13 + playbooks/fedora-32/setup-env.yaml | 12 + playbooks/fedora-33/setup-env.yaml | 12 + playbooks/fedora-rawhide/setup-env.yaml | 12 + test/system/001-version.bats | 9 +- test/system/002-help.bats | 34 ++- test/system/101-create.bats | 54 +++++ test/system/101-list.bats | 24 -- test/system/102-create.bats | 20 -- test/system/102-list.bats | 83 +++++++ test/system/103-list.bats | 15 -- test/system/103-run.bats | 52 ++++ test/system/104-rm.bats | 70 ++++++ test/system/105-rmi.bats | 65 +++++ test/system/201-run.bats | 29 --- test/system/301-rm.bats | 28 --- test/system/302-rmi.bats | 8 - test/system/README.md | 60 ++--- test/system/helpers.bash | 307 ------------------------ test/system/libs/helpers.bash | 109 +++++++++ 21 files changed, 551 insertions(+), 467 deletions(-) create mode 100644 test/system/101-create.bats delete mode 100644 test/system/101-list.bats delete mode 100644 test/system/102-create.bats create mode 100644 test/system/102-list.bats delete mode 100644 test/system/103-list.bats create mode 100644 test/system/103-run.bats create mode 100644 test/system/104-rm.bats create mode 100644 test/system/105-rmi.bats delete mode 100644 test/system/201-run.bats delete mode 100644 test/system/301-rm.bats delete mode 100644 test/system/302-rmi.bats delete mode 100644 test/system/helpers.bash create mode 100644 test/system/libs/helpers.bash diff --git a/.gitignore b/.gitignore index 225f1db..a6cfcb6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ src/toolbox +test/system/libs/bats-assert +test/system/libs/bats-support diff --git a/playbooks/fedora-31/setup-env.yaml b/playbooks/fedora-31/setup-env.yaml index 9dd2772..4160518 100644 --- a/playbooks/fedora-31/setup-env.yaml +++ b/playbooks/fedora-31/setup-env.yaml @@ -4,6 +4,7 @@ - name: Install requirements become: yes package: + use: dnf name: - golang - golang-github-cpuguy83-md2man @@ -16,6 +17,18 @@ - udisks2 - podman + - name: Install bats-support library + git: + repo: https://github.com/bats-core/bats-support + dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-support' + depth: 1 + + - name: Install bats-assert library + git: + repo: https://github.com/bats-core/bats-assert + dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-assert' + depth: 1 + - name: Setup environment become: yes command: diff --git a/playbooks/fedora-32/setup-env.yaml b/playbooks/fedora-32/setup-env.yaml index 1d66e90..e05b1fb 100644 --- a/playbooks/fedora-32/setup-env.yaml +++ b/playbooks/fedora-32/setup-env.yaml @@ -17,6 +17,18 @@ - udisks2 - podman + - name: Install bats-support library + git: + repo: https://github.com/bats-core/bats-support + dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-support' + depth: 1 + + - name: Install bats-assert library + git: + repo: https://github.com/bats-core/bats-assert + dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-assert' + depth: 1 + - name: Setup environment become: yes command: diff --git a/playbooks/fedora-33/setup-env.yaml b/playbooks/fedora-33/setup-env.yaml index 15042a0..f997680 100644 --- a/playbooks/fedora-33/setup-env.yaml +++ b/playbooks/fedora-33/setup-env.yaml @@ -17,6 +17,18 @@ - udisks2 - podman + - name: Install bats-support library + git: + repo: https://github.com/bats-core/bats-support + dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-support' + depth: 1 + + - name: Install bats-assert library + git: + repo: https://github.com/bats-core/bats-assert + dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-assert' + depth: 1 + - name: Setup environment become: yes command: diff --git a/playbooks/fedora-rawhide/setup-env.yaml b/playbooks/fedora-rawhide/setup-env.yaml index 2d59b87..4d3bb07 100644 --- a/playbooks/fedora-rawhide/setup-env.yaml +++ b/playbooks/fedora-rawhide/setup-env.yaml @@ -17,6 +17,18 @@ - udisks2 - podman + - name: Install bats-support library + git: + repo: https://github.com/bats-core/bats-support + dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-support' + depth: 1 + + - name: Install bats-assert library + git: + repo: https://github.com/bats-core/bats-assert + dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-assert' + depth: 1 + - name: Setup environment become: yes command: diff --git a/test/system/001-version.bats b/test/system/001-version.bats index d6d439f..9334dd6 100644 --- a/test/system/001-version.bats +++ b/test/system/001-version.bats @@ -1,7 +1,10 @@ #!/usr/bin/env bats -load helpers +load 'libs/bats-support/load' +load 'libs/bats-assert/load' -@test "Output version number using full flag" { - run_toolbox --version +@test "version: Check version using option --version" { + run toolbox --version + + assert_output --regexp '^toolbox version [0-9]+\.[0-9]+\.[0-9]+$' } diff --git a/test/system/002-help.bats b/test/system/002-help.bats index 1c857e4..ad6cd9d 100644 --- a/test/system/002-help.bats +++ b/test/system/002-help.bats @@ -1,8 +1,34 @@ #!/usr/bin/env bats -load helpers +load 'libs/bats-support/load' +load 'libs/bats-assert/load' -@test "Show usage screen when no command is given" { - run_toolbox 1 - is "${lines[0]}" "Error: missing command" "Usage line 1" +@test "help: Try to run toolbox with no command (shows usage screen)" { + run toolbox + + assert_failure + assert_line --index 0 "Error: missing command" + assert_output --partial "Run 'toolbox --help' for usage." +} + +@test "help: Run command 'help'" { + run toolbox help + + assert_success + assert_output --partial "toolbox - Unprivileged development environment" +} + +@test "help: Use flag '--help' (it should show usage screen)" { + run toolbox --help + + assert_success + assert_output --partial "toolbox - Unprivileged development environment" +} + +@test "help: Try to run toolbox with non-existent command (shows usage screen)" { + run toolbox foo + + assert_failure + assert_line --index 0 "Error: unknown command \"foo\" for \"toolbox\"" + assert_line --index 1 "Run 'toolbox --help' for usage." } diff --git a/test/system/101-create.bats b/test/system/101-create.bats new file mode 100644 index 0000000..98bae34 --- /dev/null +++ b/test/system/101-create.bats @@ -0,0 +1,54 @@ +#!/usr/bin/env bats + +load 'libs/bats-support/load' +load 'libs/bats-assert/load' +load 'libs/helpers' + +setup() { + cleanup_containers +} + +teardown() { + cleanup_containers +} + + +@test "create: Create the default container" { + pull_default_image + + run toolbox -y create + + assert_success +} + +@test "create: Create a container with a valid custom name ('custom-containerName')" { + run toolbox -y create -c "custom-containerName" + + assert_success +} + +@test "create: Create a container with a custom image and name ('fedora29'; f29)" { + pull_image 29 + + run toolbox -y create -c "fedora29" -i fedora-toolbox:29 + + assert_success +} + +@test "create: Try to create a container with invalid custom name ('ßpeci@l.Nam€'; using positional argument)" { + run toolbox -y create "ßpeci@l.Nam€" + + assert_failure + assert_line --index 0 "Error: invalid argument for 'CONTAINER'" + assert_line --index 1 "Container names must match '[a-zA-Z0-9][a-zA-Z0-9_.-]*'" + assert_line --index 2 "Run 'toolbox --help' for usage." +} + +@test "create: Try to create a container with invalid custom name ('ßpeci@l.Nam€'; using option --container)" { + run toolbox -y create -c "ßpeci@l.Nam€" + + assert_failure + assert_line --index 0 "Error: invalid argument for '--container'" + assert_line --index 1 "Container names must match '[a-zA-Z0-9][a-zA-Z0-9_.-]*'" + assert_line --index 2 "Run 'toolbox --help' for usage." +} diff --git a/test/system/101-list.bats b/test/system/101-list.bats deleted file mode 100644 index 485370e..0000000 --- a/test/system/101-list.bats +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bats - -load helpers - -@test "Run list with zero containers and two images" { - run_toolbox list - is "${#lines[@]}" "3" "Expected number of lines of the output is 3 (Img: 3 + Spc: 0 + Cont: 0)" - - is "${lines[1]}" ".*registry.fedoraproject.org/.*" "First of the two images" - is "${lines[2]}" ".*registry.fedoraproject.org/.*" "Second of the two images" -} - -@test "Run list with zero containers (-c flag)" { - run_toolbox list -c - is "$output" "" "Output of list should be blank" -} - -@test "Run list with zero images (-i flag)" { - run_toolbox list -i - is "${#lines[@]}" "3" "Expected number of lines of the output is 3" - - is "${lines[1]}" ".*registry.fedoraproject.org/.*" "First of the two images" - is "${lines[2]}" ".*registry.fedoraproject.org/.*" "Second of the two images" -} diff --git a/test/system/102-create.bats b/test/system/102-create.bats deleted file mode 100644 index 9041c25..0000000 --- a/test/system/102-create.bats +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bats - -load helpers - -@test "Create the default container" { - run_toolbox -y create -} - -@test "Create a container with a valid custom name ('not-running'; name passed as an argument)" { - run_toolbox -y create "not-running" -} - -@test "Create a container with a custom image and name ('running';f29; name passed as an option)" { - run_toolbox -y create -c "running" -i fedora-toolbox:29 -} - -@test "Try to create a container with invalid custom name" { - run_toolbox 1 -y create "ßpeci@l.Nam€" - is "${lines[0]}" "Error: invalid argument for 'CONTAINER'" "Toolbox should fail to create a container with such name" -} diff --git a/test/system/102-list.bats b/test/system/102-list.bats new file mode 100644 index 0000000..49256df --- /dev/null +++ b/test/system/102-list.bats @@ -0,0 +1,83 @@ +#!/usr/bin/env bats + +load 'libs/bats-support/load' +load 'libs/bats-assert/load' +load 'libs/helpers' + +setup() { + cleanup_all +} + +teardown() { + cleanup_all +} + + +@test "list: Run 'list' with zero containers and zero images (the list should be empty)" { + run toolbox list + + assert_success + assert_output "" +} + +@test "list: Run 'list -c' with zero containers (the list should be empty)" { + run toolbox list -c + + assert_success + assert_output "" +} + +@test "list: Run 'list -i' with zero images (the list should be empty)" { + run toolbox list -c + + assert_success + assert_output "" +} + +@test "list: Run 'list' with zero toolbox's containers and images, but other image (the list should be empty)" { + get_busybox_image + + run podman images + + assert_output --partial "$BUSYBOX_IMAGE" + + run toolbox list + + assert_success + assert_output "" +} + +@test "list: Try to list images and containers (no flag) with 3 containers and 2 images (the list should have 3 images and 2 containers)" { + # Pull the two images + pull_default_image + pull_image 29 + # Create tree containers + create_default_container + create_container non-default-one + create_container non-default-two + + # Check images + run toolbox list --images + + assert_success + assert_output --partial "fedora-toolbox:${DEFAULT_FEDORA_VERSION}" + assert_output --partial "fedora-toolbox:29" + + # Check containers + run toolbox list --containers + + assert_success + assert_output --partial "fedora-toolbox-${DEFAULT_FEDORA_VERSION}" + assert_output --partial "non-default-one" + assert_output --partial "non-default-two" + + # Check all together + run toolbox list + + assert_success + assert_output --partial "fedora-toolbox:${DEFAULT_FEDORA_VERSION}" + assert_output --partial "fedora-toolbox:29" + assert_output --partial "fedora-toolbox-${DEFAULT_FEDORA_VERSION}" + assert_output --partial "non-default-one" + assert_output --partial "non-default-two" +} diff --git a/test/system/103-list.bats b/test/system/103-list.bats deleted file mode 100644 index df518f7..0000000 --- a/test/system/103-list.bats +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bats - -load helpers - -@test "Run list with three containers and two images" { - run_toolbox list - is "${#lines[@]}" "7" "Expected number of lines of the output is 7 (Img: 3 + Cont: 4)" - - is "${lines[1]}" ".*registry.fedoraproject.org/.*" "The first of the two images" - is "${lines[2]}" ".*registry.fedoraproject.org/.*" "The second of the two images" - - is "${lines[4]}" ".*fedora-toolbox-.*" "The default container should be first in the list" - is "${lines[5]}" ".*not-running.*" "The container 'not-running' should be second" - is "${lines[6]}" ".*running.*" "The container 'running' should be third (last)" -} diff --git a/test/system/103-run.bats b/test/system/103-run.bats new file mode 100644 index 0000000..5ca85b3 --- /dev/null +++ b/test/system/103-run.bats @@ -0,0 +1,52 @@ +#!/usr/bin/env bats + +load 'libs/bats-support/load' +load 'libs/bats-assert/load' +load 'libs/helpers' + +# It seems like 'toolbox run' (or 'enter') doesn't work fine when +# the workdir is outside the $HOME. +# This hack is to make the tests work from outside the $HOME. +readonly CURDIR=$PWD + +setup() { + cd "$HOME" || return 1 + cleanup_containers +} + +teardown() { + cleanup_containers + cd "$CURDIR" || return 1 +} + + +@test "run: Try to run echo 'Hello World' with no containers created" { + run toolbox run echo "Hello World" + + assert_failure + assert_line --index 0 --regexp 'Error: container .* not found' + assert_output --partial "Run 'toolbox --help' for usage." +} + +#TODO: This should work without --partial +# The issue here is that toolbox output add the CRLF character at the end +@test "run: Run echo 'Hello World' inside of the default container" { + create_default_container + + run toolbox --verbose run echo "Hello World" + + assert_success + assert_output --partial "Hello World" +} + +@test "run: Run echo 'Hello World' inside a container after being stopped" { + create_container running + + start_container running + stop_container running + + run toolbox --verbose run --container running echo -n "Hello World" + + assert_success + assert_output --partial "Hello World" +} diff --git a/test/system/104-rm.bats b/test/system/104-rm.bats new file mode 100644 index 0000000..7f956ae --- /dev/null +++ b/test/system/104-rm.bats @@ -0,0 +1,70 @@ +#!/usr/bin/env bats + +load 'libs/bats-support/load' +load 'libs/bats-assert/load' +load 'libs/helpers' + +setup() { + cleanup_containers +} + +teardown() { + cleanup_containers +} + + +@test "rm: Try to remove a non-existent container" { + container_name="nonexistentcontainer" + run toolbox rm "$container_name" + + #assert_failure #BUG: it should return 1 + assert_output "Error: failed to inspect container $container_name" +} + +@test "rm: Try to remove a running container" { + skip "Bug: Fail in 'toolbox rm' does not return non-zero value" + create_container running + start_container running + + run toolbox rm running + + #assert_failure #BUG: it should return 1 + assert_output "Error: container running is running" +} + +@test "rm: Remove a not running container" { + create_container not-running + + run toolbox rm not-running + + assert_success + assert_output "" +} + +@test "rm: Force remove a running container" { + create_container running + start_container running + + run toolbox rm --force running + + assert_success + assert_output "" +} + +@test "rm: Force remove all containers (with 2 containers created and 1 running)" { + num_of_containers=$(list_containers) + assert_equal "$num_of_containers" 0 + + create_container running + create_container not-running + start_container running + + run toolbox rm --force --all + + assert_success + assert_output "" + + new_num_of_containers=$(list_containers) + + assert_equal "$new_num_of_containers" "$num_of_containers" +} diff --git a/test/system/105-rmi.bats b/test/system/105-rmi.bats new file mode 100644 index 0000000..80752c1 --- /dev/null +++ b/test/system/105-rmi.bats @@ -0,0 +1,65 @@ +#!/usr/bin/env bats + +load 'libs/bats-support/load' +load 'libs/bats-assert/load' +load 'libs/helpers' + +setup() { + cleanup_all +} + +teardown() { + cleanup_all +} + + +@test "rmi: Remove all images with the default image present" { + num_of_images=$(list_images) + assert_equal "$num_of_images" 0 + + pull_default_image + + run toolbox rmi --all + + assert_success + assert_output "" + + new_num_of_images=$(list_images) + + assert_equal "$new_num_of_images" "$num_of_images" +} + +@test "rmi: Try to remove all images with a container present and running" { + skip "Bug: Fail in 'toolbox rmi' does not return non-zero value" + num_of_images=$(list_images) + assert_equal "$num_of_images" 0 + + create_container foo + start_container foo + + run toolbox rmi --all + + assert_failure + assert_output --regexp "Error: image .* has dependent children" + + new_num_of_images=$(list_images) + + assert_equal "$new_num_of_images" "$num_of_images" +} + +@test "rmi: Force remove all images with a container present and running" { + num_of_images=$(list_images) + assert_equal "$num_of_images" 0 + + create_container foo + start_container foo + + run toolbox rmi --all --force + + assert_success + assert_output "" + + new_num_of_images=$(list_images) + + assert_equal "$new_num_of_images" "$num_of_images" +} diff --git a/test/system/201-run.bats b/test/system/201-run.bats deleted file mode 100644 index 34b7163..0000000 --- a/test/system/201-run.bats +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bats - -load helpers - -@test "Start the 'running' container and check it started alright" { - run_podman --log-level debug start running - - is_toolbox_ready running -} - -@test "Echo 'Hello World' inside of the default container" { - run_toolbox run echo "Hello World" - # is "$output" "Hello World" "Should say 'Hello World'" -} - -@test "Echo 'Hello World' inside of the 'running' container" { - run_toolbox run -c running echo "Hello World" - # is "$output" "Hello World" "Should say 'Hello World'" -} - -@test "Stop the 'running' container using 'podman stop'" { - run_podman stop running - is "${#lines[@]}" "1" "Expected number of lines of the output is 1 (with the id of the container)" -} - -@test "Echo 'hello World' again in the 'running' container after being stopped and exit" { - run_toolbox run -c running echo "Hello World" - # is "$output" "Hello World" "Should say 'Hello World'" -} diff --git a/test/system/301-rm.bats b/test/system/301-rm.bats deleted file mode 100644 index 14b3ffd..0000000 --- a/test/system/301-rm.bats +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bats - -load helpers - -@test "Try to remove a nonexistent container" { - run_toolbox rm nonexistentcontainer - is "$lines[0]" "Error: failed to inspect container nonexistentcontainer" "Toolbox should fail to remove a non-existent container" -} - -@test "Try to remove the running container 'running'" { - run_toolbox rm running - is "$output" "Error: container running is running" "Toolbox should fail to remove a running container" -} - -@test "Remove the not running container 'not-running'" { - run_toolbox rm not-running - is "$output" "" "The output should be empty" -} - -@test "Force remove the running container 'running'" { - run_toolbox rm --force running - is "$output" "" "The output should be empty" -} - -@test "Force remove all remaining containers (only 1 should be left)" { - run_toolbox rm --force --all - is "$output" "" "The output should be empty" -} diff --git a/test/system/302-rmi.bats b/test/system/302-rmi.bats deleted file mode 100644 index f294a1f..0000000 --- a/test/system/302-rmi.bats +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bats - -load helpers - -@test "Remove all images (2 should be present; --force should not be necessary)" { - run_toolbox rmi --all - is "$output" "" "The output should be empty" -} diff --git a/test/system/README.md b/test/system/README.md index 65e2d4d..673b58b 100644 --- a/test/system/README.md +++ b/test/system/README.md @@ -1,44 +1,46 @@ # System tests -These tests are built with BATS (Bash Automated Testing System). They are -strongly influenced by the [libpod](https://github.com/containers/libpod) -project. +These tests are built with BATS (Bash Automated Testing System). The tests are meant to ensure that Toolbox's functionality remains stable throughout updates of both Toolbox and Podman/libpod. -## Structure +**Warning**: The tests are not executed in an isolated environment. They affect +the system where they are run. -- **0xx (Info)** - - Commands that are not dependent on the presence/number of containers or - images. eg., version, help, etc.. -- **1xx (Initialization)** - - Commands (list, create) when Toolbox has not really been used, yet. - - It tries to list an empty list, creates several containers (default one - and several with custom names and images). -- **2xx (Usage)** - - The created containers are used for the first time testing the - initialization (CMD of the container). - - Not all containers will be used because in the *Cleanup* phase we want to - try removing containers in both running and not running states. -- **3xx (Cleanup)** - - In this section the containers and images from the previous *phases* are - removed. - - There is a difference between removing running and not running containers. - We need to check the right behaviour. +## Dependencies + +These tests use a few standard libraries for BATS which help with clarity +and consistency. In order to use it you need to download them to the `libs` +directory: + +``` +# Go to the Toolbox root folder +$ git clone https://github.com/ztombol/bats-assert test/system/libs/bats-assert +$ git clone https://github.com/ztombol/bats-support test/system/libs/bats-support +``` ## Convention -- All tests that start with *Try to..* expect non-zero return value. +- All tests should follow the nomenclature: `[command]: ...` +- When the test is expected to fail, start the test description with "Try + to..." +- When the test is to give a non obvious output, it should be put in parenthesis + at the end of the title + +Examples: + +* `@test "create: Create the default container"` +* `@test "rm: Try to remove a non-existent container"` + +- All the tests start with a clean system (no images or containers) to make sure + that there are no dependencies between tests and they are really isolated. Use + the `setup()` and `teardown()` functions for that purpose. ## How to run the tests -Make sure you have `bats` and `podman` with `toolbox` installed on your system. - -**Important** -Before you start the tests, you need to have present two images: the default -`fedora-toolbox` image for your version of Fedora and the `fedora-toolbox:29` -image. +Make sure you have `bats`, `podman` and `toolbox` installed on your system. And +have the test dependencies prepared. - Enter the toolbox root folder - Invoke command `bats ./test/system/` and the test suite should fire up @@ -48,4 +50,4 @@ By default the test suite uses the system versions of `podman` and `toolbox`. If you have a `podman` or `toolbox` installed in a nonstandard location then you can use the `PODMAN` and `TOOLBOX` environmental variables to set the path to the binaries. So the command to invoke the test suite could look something -like this: `PODMAN=/usr/libexec/podman TOOLBOX=./toolbox bats ./test/system/`. +like this: `PODMAN=/usr/libexec/podman TOOLBOX=./toolbox bats ./test/system/`. \ No newline at end of file diff --git a/test/system/helpers.bash b/test/system/helpers.bash deleted file mode 100644 index 5b0884e..0000000 --- a/test/system/helpers.bash +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/env bash - -# Podman and Toolbox commands to run -PODMAN=${PODMAN:-podman} -TOOLBOX=${TOOLBOX:-toolbox} - -# Helpful globals -LATEST_FEDORA_VERSION=${LATEST_FEDORA_VERSION:-"32"} -DEFAULT_FEDORA_VERSION=${DEFAULT_FEDORA_VERSION:-"f31"} -REGISTRY_URL=${REGISTRY_URL:-"registry.fedoraproject.org"} -TOOLBOX_DEFAULT_IMAGE=${TOOLBOX_DEFAULT_IMAGE:-"registry.fedoraproject.org/f31/fedora-toolbox:31"} -TOOLBOX_TIMEOUT=${TOOLBOX_TIMEOUT:-100} -PODMAN_TIMEOUT=${PODMAN_TIMEOUT:-100} - - -################ -# run_podman # Invoke $PODMAN, with timeout, using BATS 'run' -################ -# -# This is the preferred mechanism for invoking podman: first, it -# invokes $PODMAN, which may be 'podman-remote' or '/some/path/podman'. -# -# Second, we log the command run and its output. This doesn't normally -# appear in BATS output, but it will if there's an error. -# -# Next, we check exit status. Since the normal desired code is 0, -# that's the default; but the first argument can override: -# -# run_podman 125 nonexistent-subcommand -# run_podman '?' some-other-command # let our caller check status -# -# Since we use the BATS 'run' mechanism, $output and $status will be -# defined for our caller. -# -function run_podman() { - # Number as first argument = expected exit code; default 0 - expected_rc=0 - case "$1" in - [0-9]) expected_rc=$1; shift;; - [1-9][0-9]) expected_rc=$1; shift;; - [12][0-9][0-9]) expected_rc=$1; shift;; - '?') expected_rc= ; shift;; # ignore exit code - esac - - # stdout is only emitted upon error; this echo is to help a debugger - echo "\$ $PODMAN $*" - run timeout --foreground -v --kill=10 $PODMAN_TIMEOUT $PODMAN "$@" 3>/dev/null - # without "quotes", multiple lines are glommed together into one - if [ -n "$output" ]; then - echo "$output" - fi - if [ "$status" -ne 0 ]; then - echo -n "[ rc=$status "; - if [ -n "$expected_rc" ]; then - if [ "$status" -eq "$expected_rc" ]; then - echo -n "(expected) "; - else - echo -n "(** EXPECTED $expected_rc **) "; - fi - fi - echo "]" - fi - - if [ -n "$expected_rc" ]; then - if [ "$status" -ne "$expected_rc" ]; then - die "exit code is $status; expected $expected_rc" - fi - fi -} - -function run_toolbox() { - # Number as first argument = expected exit code; default 0 - expected_rc=0 - case "$1" in - [0-9]) expected_rc=$1; shift;; - [1-9][0-9]) expected_rc=$1; shift;; - [12][0-9][0-9]) expected_rc=$1; shift;; - '?') expected_rc= ; shift;; # ignore exit code - esac - - # stdout is only emitted upon error; this echo is to help a debugger - echo "\$ $TOOLBOX $*" - run timeout --foreground -v --kill=10 $TOOLBOX_TIMEOUT $TOOLBOX "$@" 3>/dev/null - # without "quotes", multiple lines are glommed together into one - if [ -n "$output" ]; then - echo "$output" - fi - if [ "$status" -ne 0 ]; then - echo -n "[ rc=$status "; - if [ -n "$expected_rc" ]; then - if [ "$status" -eq "$expected_rc" ]; then - echo -n "(expected) "; - else - echo -n "(** EXPECTED $expected_rc **) "; - fi - fi - echo "]" - fi - - if [ -n "$expected_rc" ]; then - if [ "$status" -ne "$expected_rc" ]; then - die "exit code is $status; expected $expected_rc" - fi - fi -} - -# Toolbox helper functions - -function is_toolbox_ready() { - toolbox_container="$1" - expected_string="Listening to file system and ticker events" - num_of_tries=5 - timeout=2 - - run_podman logs $toolbox_container - - for ((i = 0; i < $num_of_tries; i++)); do - if [[ "$output" =~ .*"$expected_string".* ]]; then - return - fi - - sleep $timeout - run_podman logs $toolbox_container - done - - echo "Output of 'podman logs $toolbox_container':" - echo "$output" - echo "" - - die "container $toolbox_container was not ready in time" -} - -# Functions to prepare environment - - -function create_toolbox() { - echo "# [create_toolbox]" - - local numberof="$1" - local naming="$2" - local image="$3" - - if [ "$numberof" = "" ]; then - numberof=${numberof:-1} - fi - - if [ "$image" = "" ]; then - image=$TOOLBOX_DEFAULT_IMAGE - fi - - for i in $(seq "$numberof"); do - if [ "$naming" = "" ]; then - run_toolbox '?' -y create -i "$image" - else - run_toolbox '?' -y create -c "$naming-$i" -i "$image" - fi - done -} - - -function get_images() { - echo "# [get_images]" - - local numberof="$1" - local image="" - - if [ "$numberof" = "" ]; then - numberof=${numberof:-1} - fi - - for i in $(seq $numberof); do - local version=$[$LATEST_FEDORA_VERSION-$i] - image="$REGISTRY_URL/f$version/fedora-toolbox:$version" - run_podman pull "$image" || echo "Podman couldn't pull the image." - done -} - - -function remove_all_images() { - echo "# [remove_all_images]" - run_podman images --all --format '{{.Repository}}:{{.Tag}} {{.ID}}' - for line in "${lines[@]}"; do - set $line - run_podman rmi --force "$1" >/dev/null 2>&1 || true - run_podman rmi --force "$2" >/dev/null 2>&1 || true - done -} - -function remove_all_images_but_default() { - echo "# [remove_all_images_but_default]" - found_needed_image=1 - run_podman images --all --format '{{.Repository}}:{{.Tag}} {{.ID}}' - for line in "${lines[@]}"; do - set $line - if [ "$1" == "$TOOLBOX_DEFAULT_IMAGE" ]; then - found_needed_image=1 - else - run_podman rmi --force "$1" >/dev/null 2>&1 || true - fi - done -} - - -function get_image_name() { - echo "# [get_image_name]" - local type="$1" - local version="$2" - - if [ -z "$type" ]; then - type=${type:-fedora} - fi - - if [ -z "$version" ]; then - version=${version:-$DEFAULT_FEDORA_VERSION} - fi - - case "$type" in - fedora) - echo "$REGISTRY_URL/f$version/fedora-toolbox:$version" - ;; - esac -} - -function remove_all_containers() { - echo "# [remove_all_containers]" - run_toolbox '?' rm --all --force -} - - -# BATS specific functions - -######### -# die # Abort with helpful message -######### -function die() { - echo "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" >&2 - echo "#| FAIL: $*" >&2 - echo "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" >&2 - false -} - - -######## -# is # Compare actual vs expected string; fail w/diagnostic if mismatch -######## -# -# Compares given string against expectations, using 'expr' to allow patterns. -# -# Examples:failed to inspect -# -# is "$actual" "$expected" "descriptive test name" -# is "apple" "orange" "name of a test that will fail in most universes" -# is "apple" "[a-z]\+" "this time it should pass" -# -function is() { - local actual="$1" - local expect="$2" - local testname="${3:-FIXME}" - - if [ -z "$expect" ]; then - if [ -z "$actual" ]; then - return - fi - expect='[no output]' - elif expr "$actual" : "$expect" >/dev/null; then - return - fi - - # This is a multi-line message, which may in turn contain multi-line - # output, so let's format it ourself, readably - local -a actual_split - readarray -t actual_split <<<"$actual" - printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2 - printf "#| FAIL: $testname\n" >&2 - printf "#| expected: '%s'\n" "$expect" >&2 - printf "#| actual: '%s'\n" "${actual_split[0]}" >&2 - local line - for line in "${actual_split[@]:1}"; do - printf "#| > '%s'\n" "$line" >&2 - done - printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2 - false -} - -################# -# parse_table # Split a table on '|' delimiters; return space-separated -################# -# -# See sample .bats scripts for examples. The idea is to list a set of -# tests in a table, then use simple logic to iterate over each test. -# Columns are separated using '|' (pipe character) because sometimes -# we need spaces in our fields. -# -function parse_table() { - while read line; do - test -z "$line" && continue - - declare -a row=() - while read col; do - dprint "col=<<$col>>" - row+=("$col") - done < <(echo "$line" | tr '|' '\012' | sed -e 's/^ *//' -e 's/\\/\\\\/g') - - printf "%q " "${row[@]}" - printf "\n" - done <<<"$1" -} diff --git a/test/system/libs/helpers.bash b/test/system/libs/helpers.bash new file mode 100644 index 0000000..b82d5f3 --- /dev/null +++ b/test/system/libs/helpers.bash @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +# Podman and Toolbox commands to run +readonly PODMAN=${PODMAN:-podman} +readonly TOOLBOX=${TOOLBOX:-toolbox} + +# Helpful globals +current_os_version=$(awk -F= '/VERSION_ID/ {print $2}' /etc/os-release) +readonly DEFAULT_FEDORA_VERSION=${DEFAULT_FEDORA_VERSION:-${current_os_version}} +readonly REGISTRY_URL=${REGISTRY_URL:-"registry.fedoraproject.org"} +readonly BUSYBOX_IMAGE="docker.io/library/busybox" + + +function cleanup_all() { + $PODMAN system reset --force >/dev/null +} + + +function cleanup_containers() { + $PODMAN rm --all --force >/dev/null +} + + +function get_busybox_image() { + $PODMAN pull "$BUSYBOX_IMAGE" >/dev/null \ + || fail "Podman couldn't pull the image." +} + + +function pull_image() { + local version + local image + local -i count + local -i max_retries + local -i wait_time + version="$1" + image="${REGISTRY_URL}/f${version}/fedora-toolbox:${version}" + count=0 + max_retries=5 + wait_time=15 + + until ${PODMAN} pull "${image}" >/dev/null ; do + sleep $wait_time + (( count += 1 )) + + if (( "$count" == "$max_retries" )); then + # Max number of retries exceeded + echo "Podman couldn't pull the image ${image}." + return 1 + fi + done +} + + +function pull_default_image() { + pull_image "${DEFAULT_FEDORA_VERSION}" +} + + +function create_container() { + local container_name + local version + local image + container_name="$1" + version="$DEFAULT_FEDORA_VERSION" + image="${REGISTRY_URL}/f${version}/fedora-toolbox:${version}" + + pull_image "$version" + + $TOOLBOX --assumeyes create --container "$container_name" \ + --image "$image" >/dev/null \ + || fail "Toolbox couldn't create the container '$container_name'" +} + + +function create_default_container() { + create_container "fedora-toolbox-${DEFAULT_FEDORA_VERSION}" +} + + +function start_container() { + local container_name + container_name="$1" + + $PODMAN start "$container_name" >/dev/null \ + || fail "Podman couldn't start the container '$container_name'" +} + + +function stop_container() { + local container_name + container_name="$1" + + # Make sure the container is running before trying to stop it + $PODMAN start "$container_name" >/dev/null \ + || fail "Podman couldn't start the container '$container_name'" + $PODMAN stop "$container_name" >/dev/null \ + || fail "Podman couldn't stop the container '$container_name'" +} + + +function list_images() { + $PODMAN images --all --quiet | wc -l +} + + +function list_containers() { + $PODMAN ps --all --quiet | wc -l +}