diff --git a/.gitignore b/.gitignore index 556a66b4c..fd63d5c83 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ .*.swp results2junitc docs_build -config_custom/*.ini cscope.files cscope.out subtests.rst diff --git a/.papr.yml b/.papr.yml deleted file mode 100644 index ecc368c46..000000000 --- a/.papr.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- - -# Example format: -# https://github.com/projectatomic/papr/blob/master/docs/sample.papr.yml - -# Internal Job (for realtime monitoring): -# https://hurl.corp.redhat.com/eb0faa3 - -container: - image: centos:latest - -# Mark this testsuite as required. This causes a special -# "required" context to be reported to GitHub on branch -# tests. The result is set to successful only if all -# testsuites marked as required are also successful. This is -# useful for integration with merge bots, so that they only -# need to watch for a single status rather than multiple -# (possibly changing) statuses. If omitted, defaults to -# false. -required: true - -# Time to allow before aborting tests. Must satisfy regex -# '[0-9]+[smh]'. If omitted, defaults to '2h', which is the -# maximum. -timeout: 30m - -# These get dumped into /etc/yum.repos.d/$name.repo -extra-repos: - - name: epel - mirrorlist: https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=$basearch - gpgkey: https://getfedora.org/static/352C64E5.txt - -packages: - - git - - make - - python-bugzilla - - pylint - - python-pep8 - - python2-mock - - python-sphinx - - python-unittest2 - - python-docutils - - python-sphinxcontrib-httpdomain - - python-sphinx_rtd_theme - - PyYAML - - libselinux-python - -env: - AUTOTEST_REPO: "https://github.com/autotest/autotest.git" - AUTOTEST_BRANCH: "master" - AUTOTEST_PATH: "/var/lib/autotest" - SPHINXOPTS: "-W" -# Put the tasks to be executed in the 'tests' key. They are -# run from the root of the repo. If the 'build' key is also -# present, tests will run after a successful build. Full -# UTF-8 values are supported. -tests: - - mkdir -p $AUTOTEST_PATH - - git clone --branch "$AUTOTEST_BRANCH" --depth 1 --single-branch "$AUTOTEST_REPO" "$AUTOTEST_PATH" - - make - - ./run_checkdocs.py - - ./run_unittests.sh - - ./run_pylint.sh --FF diff --git a/Makefile b/Makefile deleted file mode 100644 index a93106225..000000000 --- a/Makefile +++ /dev/null @@ -1,157 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = docs_build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -all: html - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -rm -rf subtests.rst - -rm -rf additional.rst - -rm -rf defaults.rst - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/DockerAutotest.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/DockerAutotest.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/DockerAutotest" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/DockerAutotest" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." diff --git a/README.md b/README.md new file mode 100644 index 000000000..091a4df55 --- /dev/null +++ b/README.md @@ -0,0 +1,90 @@ +# DANGER: CONSTRUCTION-ZONE: + +## Hard-hat, double-underwear, and dark-sunglasses required! + +### Useful documentation has been stripped for your own safety + +# Running doc^H^H^H^podman-autotest + + # cd /var/lib/autotest/client + # unset XDG_RUNTIME_DIR ! podman won't run if this is set + # ./autotest-local run docker + +This will take 25-35 minutes depending on the virt horsepower and +the version of podman (some earlier versions hang for a long time). + + +# Review results + +Open ``/var/lib/autotest/client/results/default/job_report.html`` + +You will see a table of results, with poorly-chosen non-colorblind-friendly +green/red for GOOD/FAIL respectively. Click on the `Debug` link for any +result of interest. You'll see a list of files +and have to click the right one. + +Look for boldface red; that shows the actual failure. Sometimes the actual +error is in orange (warning), especially if you see 'Postprocess' in +orange. Those are magic tests that don't actually fail in the moment +and are hard to find. The `build` test is an example of this. + +(Note that some orange warnings are OK and expected. Sorry, but you'll +have to ignore the smell for now.) + +# Debugging results + +This is a nightmare. + +Find the source code of the test: for a given test `foo` it will +be under `subtests/docker_cli/foo/` but there may be many files +in there - often it might be `foo.py` but it could also be +`subsubtestname.py` for a given subsubtest. Or, for a kill-related +test, `kill_utils.py` + +In the py file, the sequence is: + + initialize() + run_once() + postprocess() + +It can be hard to know exactly what is failing or why. My usual +process is to look in the web browser for **boldface**; this +indicates the docker commands being run. I'll see if I can +figure out what's going on just by looking at the docker commands. +Most of the time I can't, and I have to delve into the source +code. Here, again, you're on your own. Each subtest is an +unhappy family in its own unique way, and there's not really +any way I can prepare you for debugging any particular +failing test. Sorry. + +Also note that I've already picked off the low-hanging fruit: any +subtest that can be easily figured out, I've already filed podman +issues for. The remaining failures are the hard ones, those that +are not immediately obvious from the debug log, and require +delving deep into the code. Again -- sorry. + +# Rerunning tests + +Use the ``--tag`` argument to avoid clobbering ``results/default`` + + # cd /var/lib/autotest/client + # ./autotest-local run --tag=nasty_junk docker + +Then look under ``/var/lib/autotest/client/results/nasty_junk`` for +the mostly useless details. + +If you want to run just one short subtest, use --args: + + virt# ./autotest-local run docker --args docker_cli/foobar + (replace foobar with attach, images_all, build, etc) + +Short subtests can finish in 1-3 minutes instead of 20. You can +use this to instrument the code (adding printfs) then reload +in your web browser to view results. + +# Happy hacking, and please, please: do it alone. + +Whatever stumbling block you run into, I've almost certainly already +encountered it, too bad for you. Don't ping me on IRC, or send me +email. I don't care, just --force merge your change here and +we'll all hope for the best. diff --git a/README.rst b/README.rst deleted file mode 100644 index 5a14ece70..000000000 --- a/README.rst +++ /dev/null @@ -1,9 +0,0 @@ -================================= -Autotest | Client Tests | Docker -================================= - -`Full Docker Autotest documentation`__ - -.. _documentation: http://docker-autotest.readthedocs.org/en/latest/ - -__ documentation_ diff --git a/config_custom/README.rst b/config_custom/README.rst deleted file mode 100644 index 0a6ca0fa6..000000000 --- a/config_custom/README.rst +++ /dev/null @@ -1,17 +0,0 @@ -This subdirectory is searched first for ``defaults.ini``. -If found, it's options will become the defaults for all other -configuration sections. In this case, the -``config_defaults/defaults.ini`` file will be completely ignored. - -Next, all regular configuration files in the ``config_defaults`` -subdirectory are loaded and parsed. - -Finally, all regular configuration files in this subdirectory -are parsed. If any configuration sections clash, the sections -loaded from files in **this** (``config_custom``) directory will -override any previously loaded. - -Except for ``defaults.init``, the parsing order within this -subdirectory (as in ``config_defaults``) is completely undefined. -If multiple files contain duplicate sections with differing options, -the active set will be undefined. diff --git a/config_custom/control.ini b/config_custom/control.ini new file mode 100644 index 000000000..6e33a9167 --- /dev/null +++ b/config_custom/control.ini @@ -0,0 +1,19 @@ +[Control] +exclude = pretest_example,intratest_example,posttest_example,example,subexample +subthings = docker_cli/attach, + docker_cli/commit, + docker_cli/create, + docker_cli/dockerhelp, + docker_cli/history + docker_cli/import_export, + docker_cli/kill, + docker_cli/kill_bad, + docker_cli/kill_stopped, + docker_cli/logs, + docker_cli/rm, + docker_cli/run, + docker_cli/run_cgroup_parent, + docker_cli/run_cidfile, + docker_cli/start, + docker_cli/stop, + docker_cli/wait, diff --git a/config_custom/defaults.ini b/config_custom/defaults.ini new file mode 100644 index 000000000..7b76dea67 --- /dev/null +++ b/config_custom/defaults.ini @@ -0,0 +1,15 @@ +[DEFAULTS] +docker_path = /var/lib/autotest/client/tests/podman/podman_filter +docker_timeout = 600 +docker_repo_name = rhel +docker_repo_tag = latest +docker_registry_host = registry.access.redhat.com +docker_registry_user = rhel7 +remove_after_test = yes +preserve_fqins = docker.io/stackbrew/centos:7, + docker.io/stackbrew/centos:centos7, + docker.io/stackbrew/centos:latest, + registry.access.redhat.com/rhel7:latest, + registry.access.redhat.com/rhel7/rhel-tools:latest, + registry-1.docker.io/distribution/registry:2.1 +preserve_cnames = ,dummy_container_name diff --git a/config_custom/known_failures.txt b/config_custom/known_failures.txt new file mode 100644 index 000000000..87cd9ab2a --- /dev/null +++ b/config_custom/known_failures.txt @@ -0,0 +1,30 @@ +"Full NVRA, may be N-V-* as wildcard" "subsubtest path" comment describing the reason for this test exception +podman-0.3.2-* docker_cli/create/create_tmpfs "Key Error" - podman inspect doesn't show Tmpfs +podman-0.3.3-* docker_cli/create/create_remote_tag podman can't deal with colons in output filename +podman-0.3.5-* docker_cli/restart/stopped #575 stop an already-stopped container +podman-0.3.4-* docker_cli/kill/run_sigproxy_ttyoff sigproxy not implemented yet +podman-0.3.4-* docker_cli/kill/attach_sigproxy_ttyoff sigproxy not implemented yet +podman-0.3.4-* docker_cli/kill_stress/run_sigproxy_ttyoff sigproxy not implemented yet +podman-0.3.4-* docker_cli/kill_stress/attach_sigproxy_ttyoff sigproxy not implemented yet +podman-0.3.5-* docker_cli/run_dns DOS newlines in output +podman-0.3.3-* docker_cli/events events not implemented, and not likely to be +podman-0.3.2-* docker_cli/run_cgroup_parent/run_cgroup_parent_path cgroup-parent not implemented +podman-0.3.3-* docker_cli/rmi/only_tag issue #387 +podman-0.4.1-* docker_cli/rm/forced issue #527: unable to get exit code +podman-0.5.4-* docker_cli/images_all/two_images_with_parents #520 podman-history loses history +podman-0.5.4-* docker_cli/images_all/with_unused_containers #520 podman-history loses history +podman-0.4.1-* docker_cli/run/run_install no entitlements +podman-0.4.1-* docker_cli/run/run_update no entitlements +podman-0.4.1-* docker_cli/negativeusage/iv1 one of our dashes is missing +podman-0.4.1-* docker_cli/stop/force #389 exit status is 0 after signal +podman-0.3.5-* docker_cli/stop/stopped #575 stop an already-stopped container +podman-0.4.1-* docker_cli/stop/zerotime #389 exit status is 0 after signal +podman-0.4.1-* docker_cli/attach/simple #580 attach: There Can Be Only One +podman-0.4.1-* docker_cli/attach/no_stdin #580 attach: There Can Be Only One +podman-0.3.5-* docker_cli/kill/attach_sigproxy_ttyoff #580 attach: There Can Be Only One +podman-0.5.4-* docker_cli/commit/check_default_cmd #586 podman-commit doesn't create entrypoint +podman-0.4.4-* docker_cli/run_cgroups/cpu_overflow #633 typo in podman-inspect 'CpuShares' +podman-0.4.4-* docker_cli/load #639 podman prepends 'docker.io' to local image +podman-0.5.4-* docker_cli/build/expose #735 podman exposes both udp and tcp +podman-0.5.4-* docker_cli/build/onbuild buildah #658 ONBUILD not supported +podman-0.5.4-* docker_cli/create/create_remote_tag #649 podman save, multi-tag handling diff --git a/config_custom/log_versions.ini b/config_custom/log_versions.ini new file mode 100644 index 000000000..6e2d9abc4 --- /dev/null +++ b/config_custom/log_versions.ini @@ -0,0 +1,3 @@ +[log_versions] +#: CSV of rpm names to store in sysinfo/key_rpms +key_rpms = podman,buildah,container-selinux diff --git a/config_custom/subtests.ini b/config_custom/subtests.ini new file mode 100644 index 000000000..e18bc376f --- /dev/null +++ b/config_custom/subtests.ini @@ -0,0 +1,6 @@ +[docker_cli/run/run_ping] +config_version=0.8.8 +generate_generic = yes +ping_url = code.engineering.redhat.com +cmd = 'yum install -y iputils && ping -c 10 -q %(ping_url)s' +exit_status = 0 diff --git a/config_defaults/subtests/docker_cli/build.ini b/config_defaults/subtests/docker_cli/build.ini index 8171fc882..044b7107f 100644 --- a/config_defaults/subtests/docker_cli/build.ini +++ b/config_defaults/subtests/docker_cli/build.ini @@ -117,8 +117,8 @@ postproc_cmd_csv = negative(), !rx_out('foobar$'), !rx_err('Successfully built') rx_err('returned a non-zero code'), - cnt_count('1'), - img_count('1'), + cnt_count('docker=1;podman=0'), + img_count('docker=1;podman=0'), intr_exst('2'), last_cnt(), !img_exst() @@ -131,8 +131,8 @@ postproc_cmd_csv = negative(), !rx_out('foobar$'), !rx_err('Successfully built') rx_err('returned a non-zero code'), - cnt_count('1'), - img_count('1'), + cnt_count('docker=1;podman=0'), + img_count('docker=1;podman=0'), last_cnt(), !img_exst() @@ -145,7 +145,7 @@ postproc_cmd_csv = negative(), !rx_err('Successfully built') rx_err('returned a non-zero code'), cnt_count('0'), - img_count('1'), + img_count('docker=1;podman=0'), !img_exst() [docker_cli/build/rm_false] @@ -153,7 +153,7 @@ dockerfile_dir_path = /full docker_build_options = --rm=false,--no-cache=true,--quiet=false,--force-rm=false postproc_cmd_csv = positive(), rx_out('\s*Successfully built\s*(\w{64}|\w{12})'), - cnt_count('5'), + cnt_count('docker=5;podman=0'), img_count('1'), img_exst(), intr_exst() @@ -176,9 +176,9 @@ dockerfile_dir_path2 = /full docker_build_options2 = --rm=true,--no-cache=false,--quiet=false,--force-rm=false #: Second build ``postproc_cmd_csv``. postproc_cmd_csv2 = positive(), - rx_out('Using cache') + rx_out('docker=Using cache;podman=') rx_out('\s*Successfully built\s*(\w{64}|\w{12})'), - !rx_out('Schazam!$'), + !rx_out('docker=Schazam!$;podman=Nonsense_String'), cnt_count('0'), img_count('2'), img_exst(), diff --git a/config_defaults/subtests/docker_cli/create.ini b/config_defaults/subtests/docker_cli/create.ini index a3c7f90b2..02368752e 100644 --- a/config_defaults/subtests/docker_cli/create.ini +++ b/config_defaults/subtests/docker_cli/create.ini @@ -26,7 +26,7 @@ run_options_csv = --tmpfs,%(tmpfs_arg)s [docker_cli/create/create_names] cmd = sleep 2s -run_options_csv = --net,none,--restart,always +run_options_csv = --net,none #: Select whether or not the final --name value is used or the first last_name_sticks = yes #: Number of --name options to pass to create command diff --git a/config_defaults/subtests/docker_cli/dockerhelp.ini b/config_defaults/subtests/docker_cli/dockerhelp.ini index 7c9778cd1..6963a5179 100644 --- a/config_defaults/subtests/docker_cli/dockerhelp.ini +++ b/config_defaults/subtests/docker_cli/dockerhelp.ini @@ -15,9 +15,7 @@ generate_subsubtest_list = yes help_commands = attach build commit - cp diff - events export history images diff --git a/config_defaults/subtests/docker_cli/kill_bad.ini b/config_defaults/subtests/docker_cli/kill_bad.ini index 0fbca081b..bf2da9735 100644 --- a/config_defaults/subtests/docker_cli/kill_bad.ini +++ b/config_defaults/subtests/docker_cli/kill_bad.ini @@ -10,3 +10,5 @@ exec_cmd = 'while :; do sleep 0.1; done' [docker_cli/kill_bad/bad] #: csv of bad signals which will be used in test bad_signals = 0,-1,-78,96,SIGBADSIGNAL,SIG,%%,!,\\,\,"''",'""', , +docker_exit_status = 1 +podman_exit_status = 125 diff --git a/config_defaults/subtests/docker_cli/login.ini b/config_defaults/subtests/docker_cli/login.ini index c15e25156..312f29ce0 100644 --- a/config_defaults/subtests/docker_cli/login.ini +++ b/config_defaults/subtests/docker_cli/login.ini @@ -10,5 +10,25 @@ push_image = busybox #: config file into which docker login writes credentials. This is not #: likely to change. docker_auth_file = /root/.docker/config.json +podman_auth_file = /root/.podman-auth.json +#: FIXME +docker_args_csv = +podman_args_csv = --tls-verify=false,--authfile,%(podman_auth_file)s #: maximum time, in seconds, to wait for registry container to become ready. wait_for_registry_ready = 10 + +[docker_cli/login/login_fail] +docker_exit_status = 1 +podman_exit_status = 125 +docker_error_message = 401 Unauthorized +podman_error_message = invalid username/password + +[docker_cli/login/push_ok] +docker_expected_stdout_csv = ": Pushed","tag_ok: digest:" +podman_expected_stdout_csv = + +[docker_cli/login/push_fail] +docker_expected_stderr = FIXME-FIXME-FIXME test this with docker +podman_expected_stderr = unauthorized: authentication required +docker_exit_status = 1 +podman_exit_status = 125 diff --git a/config_defaults/subtests/docker_cli/mount.ini b/config_defaults/subtests/docker_cli/mount.ini new file mode 100644 index 000000000..b2ba3b8fe --- /dev/null +++ b/config_defaults/subtests/docker_cli/mount.ini @@ -0,0 +1,2 @@ +[docker_cli/mount] +subsubtests = mount_by_name, mount_by_cid, mount_twice, mount_with_no_args, umount_umounts diff --git a/config_defaults/subtests/docker_cli/negativeusage.ini b/config_defaults/subtests/docker_cli/negativeusage.ini index 6f3187cb2..fb1f2e619 100644 --- a/config_defaults/subtests/docker_cli/negativeusage.ini +++ b/config_defaults/subtests/docker_cli/negativeusage.ini @@ -8,18 +8,18 @@ subsubtests = op1, op2, op3, if1, ov1, iv1, iv3, iv4, ip1 #: {t} * ``%%(STPCNTR)s`` **-** ID of a stopped container{n} #: {t} * ``%%(NAME)s`` **-** Name of a container that doesn't exist{n} #: {n} -subcmd = +docker_subcmd = #: Optional docker subcommand arguments (CSV), also accepts same substitutions -#: as ``subcmd``. -subarg = +#: as ``docker_subcmd``. +docker_subarg = #: Required exit status integer -extcmd = 125 +docker_exit_status = 125 #: Optional regex that must match to stderr (group #1), also accepts same substitutions -#: as ``subcmd``. -stderr = +#: as ``docker_subcmd``. +docker_stderr = #: Optional regex that must match to stdout (group #1), also accepts same substitutions -#: as ``subcmd``. -stdout = +#: as ``docker_subcmd``. +docker_stdout = #op: Omit Positional - leave out a required positional parameter #if: Invalid Flag - Addition of non-existing flag @@ -28,59 +28,65 @@ stdout = #ip: Invalid Positional - Give value to non-existing positional parameter [docker_cli/negativeusage/op1] -subcmd = attach -subarg = --no-stdin,--sig-proxy -stderr = "(docker )?attach" requires( exactly)? 1 argument -extcmd = 1 +docker_subcmd = attach +docker_subarg = --no-stdin,--sig-proxy +docker_stderr = "(docker )?attach" requires( exactly)? 1 argument +podman_stderr = attach requires the name or id of one running container +docker_exit_status = 1 +podman_exit_status = 125 [docker_cli/negativeusage/op2] -subcmd = run -subarg = --interactive,%%(FQIN)s,/usr/local/sbin/fail -stderr = no such file or directory -extcmd = 127 +docker_subcmd = run +docker_subarg = --interactive,%%(FQIN)s,/usr/local/sbin/fail +docker_stderr = no such file or directory +docker_exit_status = 127 [docker_cli/negativeusage/op3] -subcmd = run -subarg = --interactive,/usr/local/sbin/fail -stderr = Unable to find image|is not a valid repository/tag +docker_subcmd = run +docker_subarg = --interactive,/usr/local/sbin/fail +docker_stderr = Unable to find image|is not a valid repository/tag +podman_stderr = error parsing reference to image|invalid reference format [docker_cli/negativeusage/if1] -subcmd = commit -subarg = --authormessage,%%(STPCNTR)s,%%(NOFQIN)s -stderr = flag provided but not defined|unknown flag: --authormessage +docker_subcmd = commit +docker_subarg = --authormessage,%%(STPCNTR)s,%%(NOFQIN)s +docker_stderr = flag provided but not defined|unknown flag: --authormessage [docker_cli/negativeusage/ov1] -subcmd = load -subarg = --input -stderr = flag needs an argument +docker_subcmd = load +docker_subarg = --input +docker_stderr = flag needs an argument [docker_cli/negativeusage/iv1] -subcmd = attach -subarg = --no-stdin=sig-proxy -stderr = invalid boolean value "sig-proxy" for --no-stdin|invalid argument "sig-proxy" for --no-stdin=sig-proxy +docker_subcmd = attach +docker_subarg = --no-stdin=sig-proxy +docker_stderr = invalid boolean value "sig-proxy" for --no-stdin|invalid argument "sig-proxy" for --no-stdin=sig-proxy +podman_stderr = invalid boolean value "sig-proxy" for -no-stdin [docker_cli/negativeusage/iv3] -subcmd = run -subarg = -p,192.168.9.1:9000,%%(FQIN)s,/bin/true -stderr = Invalid hostPort +docker_subcmd = run +docker_subarg = -p,192.168.9.1:9000,%%(FQIN)s,/bin/true +docker_stderr = Invalid hostPort [docker_cli/negativeusage/iv4] -subcmd = run -subarg = -e,PATH=/tmp,%%(FQIN)s,true +docker_subcmd = run +docker_subarg = -e,PATH=/tmp,%%(FQIN)s,true # N/B: This (commented) 'stderr' exposes RH BZ 1393572. # stderr = exec\: \"true\"\: executable file not found in \$PATH # Workaround bug to receive some benefit from this sub-subtest running/passing -stderr = executable file not found -extcmd = 127 +docker_stderr = executable file not found +docker_exit_status = 127 [docker_cli/negativeusage/ip1] -subcmd = tag -subarg = %%(RUNCNTR)s,%%(NOFQIN)s -stderr = no such id: -extcmd = 1 +docker_subcmd = tag +docker_subarg = %%(RUNCNTR)s,%%(NOFQIN)s +docker_stderr = no such id: +podman_stderr = in local storage: unable to find a name and tag match +docker_exit_status = 1 +podman_exit_status = 125 [docker_cli/negativeusage/ip2] docker_options = -subarg = daemon -g fdsafsd -b dsca -s fdsafs -e feae -- --name=flag_cGpa +docker_subarg = daemon -g fdsafsd -b dsca -s fdsafs -e feae -- --name=flag_cGpa # FIXME: BZ 1301147 closed -> add a stderr/stdout check -extcmd = 1 +docker_exit_status = 1 diff --git a/config_defaults/subtests/docker_cli/pull.ini b/config_defaults/subtests/docker_cli/pull.ini index f00e76860..a333f2f87 100644 --- a/config_defaults/subtests/docker_cli/pull.ini +++ b/config_defaults/subtests/docker_cli/pull.ini @@ -15,10 +15,12 @@ docker_registry_user = [docker_cli/pull/wrong_tag] docker_expected_exit_status = 1 +podman_expected_exit_status = 125 #: Additional tag flag **must NOT exist** docker_repo_tag = tag_does_not_exist [docker_cli/pull/wrong_registry_addr] docker_expected_exit_status = 1 +podman_expected_exit_status = 125 #: Name of an invalid remote registry server and port docker_registry_host = registry.does.not.exist.example.com:3 diff --git a/config_defaults/subtests/docker_cli/rmi.ini b/config_defaults/subtests/docker_cli/rmi.ini index 618c7096b..a444e124c 100644 --- a/config_defaults/subtests/docker_cli/rmi.ini +++ b/config_defaults/subtests/docker_cli/rmi.ini @@ -33,3 +33,4 @@ docker_commit_timeout = 60.0 [docker_cli/rmi/delete_wrong_name] docker_expected_exit_status = 1 +podman_expected_exit_status = 125 diff --git a/config_defaults/subtests/docker_cli/run.ini b/config_defaults/subtests/docker_cli/run.ini index b30ead059..f3b084f9a 100644 --- a/config_defaults/subtests/docker_cli/run.ini +++ b/config_defaults/subtests/docker_cli/run.ini @@ -1,5 +1,5 @@ [docker_cli/run] -subsubtests = run_true,run_false,run_interactive,run_attach_stdout,run_remote_tag,run_names,run_tmpfs,run_passwd,run_install,run_update +subsubtests = run_true,run_false,run_interactive,run_attach_stdout,run_remote_tag,run_names,run_tmpfs,run_passwd #: The most basic sub-subtests can be generated from a generic #: class at runtime, set 'yes' to enable this feature. generate_generic = no diff --git a/config_defaults/subtests/docker_cli/run_cgroup_parent.ini b/config_defaults/subtests/docker_cli/run_cgroup_parent.ini index 684b4ad0d..e656358bd 100644 --- a/config_defaults/subtests/docker_cli/run_cgroup_parent.ini +++ b/config_defaults/subtests/docker_cli/run_cgroup_parent.ini @@ -3,4 +3,6 @@ subsubtests = run_cgroup_parent_invalid_name,run_cgroup_parent_path,run_cgroup_p [docker_cli/run_cgroup_parent/run_cgroup_parent_invalid_name] #: Regex describing the error message we expect. Passed to re.match(). -expect_stderr = .*Error response from daemon: cgroup-parent for systemd cgroup should be a valid slice named as "xxx.slice"\. +docker_expect_stderr = .*Error response from daemon: cgroup-parent for systemd cgroup should be a valid slice named as "xxx.slice"\. +#: Regex describing the error message we expect. Passed to re.match(). +podman_expect_stderr = did not receive systemd slice as cgroup parent when using systemd to manage cgroups: invalid argument diff --git a/config_defaults/subtests/docker_cli/run_cgroups.ini b/config_defaults/subtests/docker_cli/run_cgroups.ini index 1cd7e1cd0..e2240e8ec 100644 --- a/config_defaults/subtests/docker_cli/run_cgroups.ini +++ b/config_defaults/subtests/docker_cli/run_cgroups.ini @@ -1,6 +1,4 @@ [docker_cli/run_cgroups] -#: subsystem path to the docker's cgroups on host -cgroup_path = /sys/fs/cgroup/memory/system.slice/docker #: which key is used in this test cgroup_key_value = memory.limit_in_bytes subsubtests = cpu_positive,cpu_zero,memory_positive,memory_no_cgroup,cpu_none,memory_negative,cpu_overflow @@ -25,24 +23,20 @@ memory_invalid = abcd expect_success = FAIL [docker_cli/run_cgroups/cpu_positive] -cgroup_path = /sys/fs/cgroup/cpu/system.slice/docker cgroup_key_value = cpu.shares #: Set/expected value of the ``cgroup_key_value`` cpushares_value = 10 [docker_cli/run_cgroups/cpu_zero] -cgroup_path = /sys/fs/cgroup/cpu/system.slice/docker cgroup_key_value = cpu.shares #: Set/expected value of the ``cgroup_key_value`` cpushares_value = 0 [docker_cli/run_cgroups/cpu_none] -cgroup_path = /sys/fs/cgroup/cpu/system.slice/docker #: Set/expected value of the ``cgroup_key_value`` cgroup_key_value = cpu.shares [docker_cli/run_cgroups/cpu_overflow] -cgroup_path = /sys/fs/cgroup/cpu/system.slice/docker #: Set/expected value of the ``cgroup_key_value`` cgroup_key_value = cpu.shares #: Set/expected value of the ``cgroup_key_value`` diff --git a/config_defaults/subtests/docker_cli/run_cidfile.ini b/config_defaults/subtests/docker_cli/run_cidfile.ini index 03fead58e..4e089fade 100644 --- a/config_defaults/subtests/docker_cli/run_cidfile.ini +++ b/config_defaults/subtests/docker_cli/run_cidfile.ini @@ -2,3 +2,6 @@ subsubtests = basic #: Additional options to pass when starting container run_options_csv = -i +#: Error message we expect to see if cidfile exists +docker_msg_cidfile_exists = Container ID file found, make sure the other container isn't running or delete +podman_msg_cidfile_exists = container id file exists. ensure another container is not using it or delete diff --git a/config_defaults/subtests/docker_cli/run_exec.ini b/config_defaults/subtests/docker_cli/run_exec.ini index abd61dd83..fbdd1258b 100644 --- a/config_defaults/subtests/docker_cli/run_exec.ini +++ b/config_defaults/subtests/docker_cli/run_exec.ini @@ -13,7 +13,8 @@ exit_status = 0 subsubtests = exec_true, exec_false, exec_pid_count [docker_cli/run_exec/exec_false] -exit_status = 1 +docker_exit_status = 1 +podman_exit_status = 125 [docker_cli/run_exec/exec_pid_count] #: Expected count of pid in container when command using exec is started. diff --git a/config_value.py b/config_value.py deleted file mode 100755 index 9113dadd7..000000000 --- a/config_value.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python - -"""Simple utility to extract a value from a set of ini-files""" - -import sys -from ConfigParser import SafeConfigParser - -scp = SafeConfigParser() - -if __name__ == "__main__": - section = sys.argv[1] - key = sys.argv[2] - scp.read(sys.argv[3:]) - print scp.get(section, key) diff --git a/dockertest/config.py b/dockertest/config.py index 80158816b..a5d2ae7a2 100644 --- a/dockertest/config.py +++ b/dockertest/config.py @@ -11,6 +11,7 @@ from ConfigParser import SafeConfigParser from collections import MutableMapping +from dockertest.output import DockerVersion import os.path import sys import copy @@ -245,6 +246,7 @@ class ConfigDict(MutableMapping): def __init__(self, section, defaults=None, *args, **dargs): self._config_section = ConfigSection(defaults=defaults, section=section) + self._is_podman = None # pylint: disable=E1101 super(ConfigDict, self).__init__(*args, **dargs) @@ -274,6 +276,13 @@ def __getitem__(self, key): # Don't call more methods than necessary if not self.__contains__(key): raise xceptions.DockerKeyError(key) + # Special case for podman: use 'podman_xxxxx' setting if defined + if key.startswith('docker_'): + if self.is_podman: + podman_key = key.replace('docker_', 'podman_') + if self.__contains__(podman_key): + key = podman_key + # No suffix calls regular get(), boolean wants to gobble '0' and '1' :( for suffix in ('int', 'boolean', 'float', ''): method = getattr(self._config_section, 'get%s' % suffix) @@ -307,6 +316,15 @@ def write(filelike): raise xceptions.DockerIOError("Instance does not permit writing to %s" % filelike.name) + @property + def is_podman(self): + """ + Return true if we're testing podman (instead of docker) + """ + if self._is_podman is None: + self._is_podman = DockerVersion().is_podman + return self._is_podman + class Config(dict): diff --git a/dockertest/docker_daemon.py b/dockertest/docker_daemon.py index 2311be6ca..4eca59c35 100644 --- a/dockertest/docker_daemon.py +++ b/dockertest/docker_daemon.py @@ -119,6 +119,7 @@ def which_docker(): or the name of a known docker-as-system-container service. """ docker = 'docker' + return 'podman' # FIXME FIXME FIXME # Known docker-daemon services as of July 2017 docker_services = ('docker', 'docker-latest', 'container-engine') @@ -172,6 +173,10 @@ def systemd_show(prop): def pid(): """ returns the process ID of currently-running docker daemon """ mainpid = int(systemd_show('MainPID')) + # systemd returns 0 for unknown or non-running units; this is + # expected when testing podman, which has no daemon. + if mainpid == 0: + return 0 cmd = cmdline(mainpid) if 'dockerd' in cmd[0]: return mainpid @@ -198,6 +203,9 @@ def cmdline(process_id=None): """ if process_id is None: process_id = pid() + # daemon won't be running if we're testing podman + if int(process_id) == 0: + return '' ps_command = 'ps -o command= -p %d' % int(process_id) return utils.run(ps_command).stdout.strip().split() diff --git a/dockertest/dockercmd.py b/dockertest/dockercmd.py index 3123af3ce..32c7d05d2 100644 --- a/dockertest/dockercmd.py +++ b/dockertest/dockercmd.py @@ -441,6 +441,10 @@ def container_id(self): c_pid = utils.run('docker inspect --format {{.State.Pid}} ' + cid) if int(c_pid.stdout) == int(pid): return cid + + # No match. FIXME: this is pretty likely because the PID reported by + # docker inspect is actually unrelated to ours; FIXME FIXME + self.subtest.logwarning("Could not find container_id for pid %d" % pid) return None # Override base-class property methods to give up-to-second details diff --git a/dockertest/output/dockerversion.py b/dockertest/output/dockerversion.py index 07c29cba3..7a184a32a 100644 --- a/dockertest/output/dockerversion.py +++ b/dockertest/output/dockerversion.py @@ -31,6 +31,7 @@ class DockerVersion(object): _client_info = None _server_info = None _has_distinct_exit_codes = None + _is_podman = None def __init__(self, version_string=None, docker_path=None): # If called without an explicit version string, run docker to find out @@ -248,3 +249,13 @@ def has_distinct_exit_codes(self): has = (d_run.exit_status > 120) DockerVersion._has_distinct_exit_codes = has return DockerVersion._has_distinct_exit_codes + + @property + def is_podman(self): + """ + Are we testing podman? This is expensive, so cache the result. + """ + if DockerVersion._is_podman is None: + d_help = utils.run('docker --help') + DockerVersion._is_podman = ('podman' in d_help.stdout) + return DockerVersion._is_podman diff --git a/index.rst b/index.rst deleted file mode 100644 index ff90e976b..000000000 --- a/index.rst +++ /dev/null @@ -1,984 +0,0 @@ -:tocdepth: 2 - -.. Docker Autotest documentation master file, created by - sphinx-quickstart on Tue Feb 18 09:56:35 2014. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -.. Quick reference for reStructuredText: - http://docutils.sourceforge.net/docs/user/rst/quickref.html - -.. Reference for sphinx.ext.autodoc extension: - http://sphinx-doc.org/ext/autodoc.html - - -===================================== -Docker Autotest |version| -===================================== - -.. sectnum:: - -.. toctree:: - :hidden: - :numbered: - - defaults - subtests - additional - -.. contents:: - :depth: 1 - :local: - ----------------- -Introduction ----------------- - -Docker Autotest_ is a sub-framework for standalone testing of docker_. -It does not depend on docker itself. Functionally, testing occurs within -any number of subtest modules, which in some cases also include further -nested sub-subtests. It is designed to support both extremely simple, -linear, and more complex, nested, iterative testing of arbitrary external -docker commands. - -Test content and configuration is fully modular, supporting both included -and external tests. The included content is primarily focused on -continuous-integration testing of the Docker CLI client, with a very -limited amount of negative and stress testing. - -As with other `Autotest Client Tests`_ the main entry point is the test -control file. Docker Autotest's control file utilizes Autotest's -steps-engine. This allows the overall testing state to be stored, -recovered, and resumed should a host-kernel panic or if userspace -become unresponsive. Consequently, rebooting the host system as a -testing step is also supported. - -.. _docker: http://www.docker.io - -.. _Autotest: http://github.com/autotest/autotest/wiki - -.. _python-docker-py: http://github.com/dotcloud/docker-py#readme - -.. _Autotest Client Tests: http://github.com/autotest/autotest-client-tests - -.. _docker_autotest_prereq: - ----------------- -Prerequisites ----------------- - -* Docker - - * **Clean** environment, only test-related images, containers - and dependent services (besides ``docker -d``) at the start - of **every** Autotest run. - * Docker installation (with docker daemon running) - * Default settings for Docker on your platform/OS, - including storage (LVM thin-pool + XFS). - * Sufficient available temporary storage for multiple copies of the - test-related images/container content. - -* `Supported Docker OS platform`_ - - * Red Hat Enterprise Linux 7 Server (preferred for development) - * Red Hat Enterprise Linux Atomic Host (preferred for testing) - * Fedora 22 or later including Atomic (not all tests may function) - * Other platforms (such as CentOs, SLES, Ubuntu) un-maintained but possible. - -* Platform Applications/tools - - * Autotest 0.16.0 or later. - * Coreutils or equivalent (i.e. ``cat``, ``mkdir``, ``tee``, etc.) - * Tar and supported compression programs (``bzip2``, ``gzip``, etc.) - * nfs-utils (nfs-server support daemons) - * Python 2.6 or greater (not 3.0) - * libselinux-python 2.2 or later - * Optional (for building documentation), ``make``, ``python-sphinx``, - and ``docutils`` or equivalent for your platform. - * Optional (for running unittests), ``pylint``, ``pep8``, - ``python-unittest2``, and ``python2-mock``. - * Optional, ``python-bugzilla`` for test exclusion based - upon bug status. See ``control.ini`` file comments for details. - - -* **Any specific requirements for particular** `subtest modules`_. - In particular: - - * Direct root access for running docker, restarting the daemon, - and accessing metadata / storage details. Running through - sudo may work to a degree, but is likely to cause problems. - * Access to a remote (to the host) image registry via ``docker pull``, - ``http``, ``https``, and ``git``. - * External testing dependencies must be usable by tests w/in fixed - timeout periods. If an external resource or connection is too slow, - it should be made more network-local to the test host. - * Most tests with external dependencies will have them flagged as - values to the special ``__example__`` option. See `example values`_ - for more details. - -.. _Supported Docker OS platform: https://www.docker.io/gettingstarted/#h_installation - ----------------- -Quickstart ----------------- - -All platforms -============= - -#. Double-check you meet all the requirements in `docker_autotest_prereq`_. For - the quickstart, either a RHEL 7 or Fedora 25 system is assumed with the Docker - daemon started, device-mapper or overlay2 storage configured, and at - least 10gig of registry space is available. - -#. As ``root``, shallow-clone Autotest (**non-recursive**) into ``/var/lib/autotest``, - and set/export the ``AUTOTEST_PATH`` environment variable to it's location. - - :: - - [root@docker ~]# cd /var/lib - - [root@docker lib]# git clone --single-branch --depth 1 \ - https://github.com/autotest/autotest.git autotest - - [root@docker lib]# export AUTOTEST_PATH=/var/lib/autotest - -#. Change to the autotest client subdirectory. - -#. Clone `autotest-docker`_ repository into the ``docker`` subdirectory. - Based from a `formal release`_ or the latest available. - - :: - - [root@docker lib]# cd $AUTOTEST_PATH/client - - [root@docker client]# git clone --branch $VERSION \ - https://github.com/autotest/autotest-docker.git tests/docker - - Where ``$VERSION`` is the docker-autotest release (e.g. "|version|") - **or** to use the current latest release, omit the --branch option: - - :: - - [root@docker client]# git clone \ - https://github.com/autotest/autotest-docker.git docker - - -#. Make a copy of default configuration, then edit as appropriate. - Particularly, verify the CSV list of full-image names and - container names, ``preserve_fqins`` and ``preserve_cnames`` - are set correctly. **All other images/containers will be destroyed!** - See `default configuration options`_ for more details. - - :: - - [root@docker client]# cp -abi tests/docker/config_defaults/defaults.ini \ - tests/docker/config_custom/ - - [root@docker client]# $EDITOR tests/docker/config_custom/defaults.ini - -.. _autotest-docker: https://github.com/autotest/autotest-docker.git -.. _`formal release`: https://github.com/autotest/autotest-docker/releases - -Fedora platforms -================= - -The Fedora base-images lack some essential tooling necessary for testing. In addition -to the steps above, a custom test-image must be configured for building. - -#. Edit the ``defaults.ini`` file again, and change the registry settings as follows: - - :: - - [root@docker client]# $EDITOR tests/docker/config_custom/defaults.ini - - ... - docker_registry_host = - docker_registry_user = - docker_repo_name = fedora_test_image - docker_repo_tag = latest - ... - -#. Make a copy of the ``docker_test_images.ini`` configuration file and configure - it to build the test image. - - :: - - [root@docker client]# cp -abi tests/docker/config_defaults/docker_test_images.ini \ - tests/docker/config_custom/ - - [root@docker client]# $EDITOR tests/docker/config_custom/docker_test_images.ini - - ... - build_name = fedora_test_image:latest - build_dockerfile = https://github.com/autotest/autotest-docker/raw/master/fedora_test_image.tar.gz - build_opts_csv = --no-cache,--pull,--force-rm - ... - -Execute and examine results -============================ - -For all platforms, use the standalone autotest client to select and execute subtests. -The default behavior is to run all subtests. However, the example below only -executes the ``version`` subtest for demonstration purposes. This will bring in -some additional utility "tests", such as ``docker_test_images`` and ``garbage_check``. -Other subtests may be selected via the ``--args`` `parameter or by customizing`_ -``control.ini``. - -:: - - [root@docker /]# cd $AUTOTEST_PATH/client - - [root@docker client]# ./autotest-local tests/docker/control --args=docker_cli/version - Writing results to /var/lib/autotest/client/results/default - START ---- ---- - Subtest/Sub-subtest requested: - 'docker_cli/version' - - Subtest/sub-subtest exclude list: - 'subexample' - 'pretest_example' - 'example' - 'intratest_example' - 'posttest_example' - - Executing tests: - 'docker/pretests/docker_test_images.1' - 'docker/pretests/log_sysconfig.2' - 'docker/pretests/log_versions.3' - 'docker/subtests/docker_cli/version.4' - 'docker/intratests/garbage_check.4' - - START docker/pretests/docker_test_images.1 - docker_test_images: initialize() - docker_test_images: setup() for subtest version 2055 - docker_test_images: Running sub-subtests... - puller: initialize() - puller: run_once() - puller: Pulling registry.access.redhat.com/rhel7/rhel:latest - puller: Pulling docker.io/stackbrew/centos:latest - puller: postprocess() - puller: cleanup() - builder: initialize() - builder: run_once() - builder: postprocess() - builder: cleanup() - docker_test_images: postprocess_iteration() #0 of #1 - docker_test_images: full_name:registry.access.redhat.com/rhel7/rhel:latest - docker_test_images: full_name:docker.io/stackbrew/centos:latest - docker_test_images: Updated preserve_fqins: docker.io/stackbrew/centos:latest,registry.access.redhat.com/rhel7/rhel:latest - docker_test_images: Postprocess sub-subtest results... - docker_test_images: cleanup() - GOOD docker/pretests/docker_test_images.1 - END GOOD docker/pretests/docker_test_images.1 - START docker/pretests/log_sysconfig.2 - log_sysconfig: initialize() - log_sysconfig: setup() for subtest version 0 - log_sysconfig: run_once() - log_sysconfig: postprocess_iteration() #0 of #1 - log_sysconfig: postprocess() - log_sysconfig: cleanup() - GOOD docker/pretests/log_sysconfig.2 - END GOOD docker/pretests/log_sysconfig.2 - START docker/pretests/log_versions.3 - log_versions: initialize() - log_versions: setup() for subtest version 0 - log_versions: run_once() - log_versions: Found docker version client: 1.12.6 server 1.12.6 - log_versions: postprocess_iteration() #0 of #1 - log_versions: postprocess() - log_versions: cleanup() - GOOD docker/pretests/log_versions.3 - END GOOD docker/pretests/log_versions.3 - START docker/subtests/docker_cli/version.4 - version: initialize() - version: setup() for subtest version 0 - version: run_once() - version: postprocess_iteration() #0 of #1 - version: postprocess() - version: docker version client: 1.12.6 server 1.12.6 - version: Docker cli version matches docker client API version - version: cleanup() - GOOD docker/subtests/docker_cli/version.4 - END GOOD docker/subtests/docker_cli/version.4 - START docker/intratests/garbage_check.4 - GOOD docker/intratests/garbage_check.4 - END GOOD docker/intratests/garbage_check.4 - END GOOD ---- ---- - - -*(timestamps and extra inconsequential text removed for clarity)* - -Examine the test results by changing to the ``results/default`` directory. -*Note:* The name "default" is used when no ``--tag`` option is given to the -``autotest-local`` command. - -:: - - [root@docker client]# cd $AUTOTEST_PATH/client/results/default - - [root@docker default]# ls -1 - control # Copy of the control file used for the run - control.ini # Runtime configuration from default control file - control.state # Used to support mid-test reboot / test resumption - debug # All the client / sydout/stderr recorded by log-level. - docker # Directory-tree of subtest results by name - job_report.html # Autogenerated report web-page. - status # Text-version of test run / results - status.json # Same thing, but in JSON format. - sysinfo # Directory of important log-files for the run. - - [root@docker default]# ls -1 docker/subtests/docker_cli/version.4/ - debug # Same as above, but ONLY logs for this subtest - keyval # Copy of subtest configuration, including defaults - profiling # Not used - results # Not used - status # Same as above, but ONLY for this subtest - sysinfo # Logs captured after this subtest ran. - -If you wish jUnit format results, execute the included conversion script. - -:: - - [root@docker client]# cd $AUTOTEST_PATH/client - [root@docker client]# tests/docker/results2junit --name $HOSTNAME results/default - - [root@docker client]# cat results/default/results.junit - - - - ... - -.. _`parameter or by customizing`: _selecting subthings - - -.. _subtests: - ------------------ -Subtests ------------------ - -All `subtest modules`_ reside beneath the ``subtest`` directory. A subtest -module must have the same name as the directory it is in (minus the ``.py`` -extension). Other files/directories may exist at that level, but they -will not be recognized as subtest modules by the Docker client test. This -ensures each subtest's code is kept separate from all others. Finally, -every subtest is run in its own process and context. It does not have -visibility into any other subtest code, configuration, or runtime. - -Organization and Naming -========================= - -The structure/layout of the ``subtest`` directory tree is relevant for -reference and configuration. The reference and configuration section names -for subtests are formed by its relative location under the ``subtest`` -directory. For example, the subtest module ``subtests/docker_cli/version/version.py`` -matches with the ``[docker_cli/version]`` configuration section and -``docker_cli/version`` subtest name. - -Static Content Setup -=========================== - -Subtests may source their own static content from within their directory and below. -When content needs to be built, or is in some way test-environment specific, the -``setup() method`` should be overridden. Content may be referenced from the subtest's -directory by using its ``bindir`` attribute. A version-specific directory to -contain build/setup output is available as the ``srcdir`` attribute. The ``setup()`` -method will ***only*** be called once per version number (including revisions). -State may be reset by clearing the autotest client ``tmp`` directory. - -:Note: The ``setup()`` method runs **after** the ``initialize()`` method - only once. If the configuration version has not changed, the method - will not be called in subsequent Docker Autotest runs. - -Sub-subtests -============== - -There are provisions for tests that contain, or are composed of multiple child -tests or dependent operations. They may share content and code between each other, -so long as it lives within the subtest directory or below. Optionally, they may -use their own configuration sections, named by appending their class name onto -the parent subtest's name. Sub-subtest configuration inherits undefined values -from the parent subtest configuration. Additionally, there are multiple methods -of executing sub-subtests depending on needs. - --------------------- -Images --------------------- - -It is assumed that any images required for testing are available and built -beforehand. A default image for testing purposes is required by most tests. -Its fully-qualified name is configurable, though test results will be directly -affected if it is or becomes unavailable. Individual subtests may require -specific additional images or content. If so, this will be noted -in the subtest documentation's *Prerequisites* section. - -.. _configuration: - --------------------- -Configuration --------------------- - -The default configuration files are all located under the ``config_defaults`` -subdirectory. These are intended to be bundled with the autotest docker test. -To customize any subtest or global default configuration, copies should -be made manually into the ``config_custom`` subdirectory. Any content -within ``config_custom`` will override all files and sections from -``config_defaults``. - -The subdirectory structure or relative file locations under ``config_custom`` -is irrelevant. Multiple sections may appear in the same file, even for -unrelated tests. The only exception is the ``config_custom/defaults.ini`` file -and the test ``control.ini`` file. - -When customizing subtest and sub-subtest configuration options, it is highly -recommended that you add the option ``config_version`` with the current -version number, into each section. This way, you will receive warnings when -updating Docker Autotest, if the specific custom configuration doesn't match -the API. Generally this only happens between major/minor version updates. - -Organization -============== - -Sections ---------------- - -Configuration files use the familiar *ini* style format with separate -sections (e.g. ``[
]``) preventing option names from colliding. -All configuration files are loaded into a single name-space, divided by -each section. Section names can be arbitrary, however those which exactly -match a subtest or sub-subtest's name, will be automatically loaded (see Subtests_). - -.. _default configuration options: - -Defaults ----------------- - -The Default, global values for **all** sections are located within the -special ``defaults.ini`` file's ``DEFAULTS`` section. These option -names and values are supplied for all sections which do not contain -a identical named option. This file is loaded *either* from the -``config_defaults`` *or* ``config_custom`` directory. - - -.. include:: defaults.rst - - -Formatting -============= - -Long values ----------------- - -Long values may be continued onto the next line by prefixing any run of one or -more horizontal-whitespace characters with a newline. For example: - - option_name = This option value is super duper long, - but will be un-folded into a single string with - <---- all this whitespace replaced by a single space. - -In this case, the runs of multiple whitespace following the newline will -be folded into a single space, and combined with the previous line. - -.. _example values: - -Example Values ------------------- - -In order to help dockertest operate w/o customization, many example or demonstration -value have been configured. While this is fine for development and informal testing -purposes, it adds external dependencies for production testing. Therefore every -option with default example values should be specified in a -comma-separated list to the special ``__example__`` option. - -The ``__example__`` option's value is parsed specially. It is not inherited from -defaults, or from subtest to sub-subtest. Instead, it is compounded at each level -then pruned of modified options (as compared to their default/example value). Any -unmodified options remaining at the beginning of a test will cause loud warnings -to be issued. - -Value substitution ---------------------- - -Within each section, optional inline-value substitution is supported -using the ``%(