Merge remote-tracking branch 'nipy/master'

Author: tsalo

.circle/codecov.sh

@@ -17,23 +17,27 @@ fi
 # They may need to be rebalanced in the future.
 case ${CIRCLE_NODE_INDEX} in
   0)
-    docker run --rm -it -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /src/nipype nipype/nipype:py27 /usr/bin/run_pytests.sh py27 && \
-    docker run --rm -it -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /src/nipype nipype/nipype:py35 /usr/bin/run_pytests.sh py35 && \
-    docker run --rm -it -v $WORKDIR:/work -w /src/nipype/doc nipype/nipype:py35 /usr/bin/run_builddocs.sh && \
-    docker run --rm -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d && \
-    docker run --rm -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d
+    docker run --rm=false -it -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_pytests.sh && \
+    docker run --rm=false -it -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_pytests.sh && \
+    docker run --rm=false -it -v $WORKDIR:/work -w /src/nipype/doc nipype/nipype:py35 /usr/bin/run_builddocs.sh && \
+    docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d && \
+    docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d
     ;;
   1)
-    docker run --rm -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ level1 && \
-    docker run --rm -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ l2pipeline
+    docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ level1 && \
+    docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ l2pipeline
     ;;
   2)
-    docker run --rm -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \
-    docker run --rm -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline
+    docker run --rm=false -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \
+    docker run --rm=false -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline
     ;;
   3)
-    docker run --rm -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \
-    docker run --rm -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline && \
-    docker run --rm -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow
+    docker run --rm=false -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \
+    docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline && \
+    docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow
     ;;
 esac
+
+cp ${WORKDIR}/tests/*.xml ${CIRCLE_TEST_REPORTS}/tests/
+codecov -f "coverage*.xml" -s "${WORKDIR}/tests/" -R "${HOME}/nipype/" -F unittests -e CIRCLE_NODE_INDEX
+codecov -f "smoketest*.xml" -s "${WORKDIR}/tests/" -R "${HOME}/nipype/" -F smoketests -e CIRCLE_NODE_INDEX

.circle/tests.sh

@@ -1,7 +1,2 @@
 [run]
 branch = True
-include = */nipype/*
-omit =
-    */nipype/external/*
-    */nipype/fixes/*
-    */setup.py

.coveragerc

@@ -1,6 +1,7 @@
 # python cache
-__pycache__/**/*
 __pycache__
+**/__pycache__
+**/*.pyc
 *.pyc
 
 # python distribution
@@ -23,14 +24,14 @@ src/
 # other
 docs/**/*
 docs/
+.cache/
 .circle/**/*
 .circle/
 circle.yml
-.coverage
-.coveragerc
-codecov.yml
 rtd_requirements.txt
 Vagrantfile
 .travis.yml
-.noserc
+.mailmap
 
+# Previous coverage results
+.coverage

.dockerignore

@@ -24,5 +24,6 @@ crash*.pklz
 htmlcov/
 __pycache__/
 *~
+.*.swp
 .ipynb_checkpoints/
 .ruby-version

.gitignore

@@ -42,7 +42,7 @@ before_install:
 install:
 - travis_retry pip install -e .[$NIPYPE_EXTRAS]
 script:
-- py.test --doctest-modules nipype
+- py.test -v --doctest-modules nipype
 deploy:
   provider: pypi
   user: satra

.noserc

@@ -71,6 +71,16 @@ RUN conda config --add channels conda-forge; sync && \
 RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/site-packages/matplotlib/mpl-data/matplotlibrc && \
     python -c "from matplotlib import font_manager"
 
+# Install CI scripts
+COPY docker/files/run_* /usr/bin/
+RUN chmod +x /usr/bin/run_*
+
+# Replace imglob with a Python3 compatible version
+COPY nipype/external/fsl_imglob.py /usr/bin/fsl_imglob.py
+RUN rm -rf ${FSLDIR}/bin/imglob && \
+    chmod +x /usr/bin/fsl_imglob.py && \
+    ln -s /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob
+
 # Installing dev requirements (packages that are not in pypi)
 WORKDIR /src/
 COPY requirements.txt requirements.txt
@@ -83,16 +93,6 @@ RUN cd /src/nipype && \
     pip install -e .[all] && \
     rm -rf ~/.cache/pip
 
-# Install CI scripts
-COPY docker/files/run_* /usr/bin/
-RUN chmod +x /usr/bin/run_*
-
-# Replace imglob with a Python3 compatible version
-COPY nipype/external/fsl_imglob.py /usr/bin/fsl_imglob.py
-RUN rm -rf ${FSLDIR}/bin/imglob && \
-    chmod +x /usr/bin/fsl_imglob.py && \
-    ln -s /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob
-
 WORKDIR /work/
 
 ARG BUILD_DATE

.travis.yml

@@ -17,16 +17,18 @@ dependencies:
     - "~/docker"
     - "~/examples"
     - "~/.apt-cache"
+    - "~/bin"
 
   pre:
     # Let CircleCI cache the apt archive
     - mkdir -p ~/.apt-cache/partial && sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives
     - sudo apt-get -y update && sudo apt-get install -y wget bzip2
     # Create work folder and force group permissions
     - mkdir -p $WORKDIR && sudo setfacl -d -m group:ubuntu:rwx $WORKDIR && sudo setfacl -m group:ubuntu:rwx $WORKDIR
-    - mkdir -p $HOME/docker $HOME/examples $WORKDIR/pytest $WORKDIR/logs
+    - mkdir -p $HOME/docker $HOME/examples $WORKDIR/tests $WORKDIR/logs $WORKDIR/crashfiles ${CIRCLE_TEST_REPORTS}/tests/
+    - if [[ ! -e "$HOME/bin/codecov" ]]; then mkdir -p $HOME/bin; curl -so $HOME/bin/codecov https://codecov.io/bash && chmod 755 $HOME/bin/codecov; fi
   override:
-    - if [[ -e $HOME/docker/cache.tar ]]; then docker load --input $HOME/docker/cache.tar; fi :
+    - if [[ -e "$HOME/docker/cache.tar" ]]; then docker load --input $HOME/docker/cache.tar; fi :
         timeout: 6000
     - docker images
     - docker pull nipype/base:latest
@@ -48,14 +50,13 @@ test:
     - bash .circle/tests.sh :
         timeout: 7200
         parallel: true
-  post:
-    # Send coverage data to codecov.io
-    - bash .circle/codecov.sh
 
 general:
   artifacts:
     - "~/work/docs"
     - "~/work/logs"
+    - "~/work/tests"
+    - "~/work/crashfiles"
 
 deployment:
   production:

Dockerfile

@@ -22,3 +22,7 @@ coverage:
         threshold: 100
         flags:
           - "smoketests"
+  ignore:          # files and folders that will be removed during processing
+    - "nipype/external/*"
+    - "tools/*"
+    - "doc/*"

circle.yml

@@ -137,8 +137,15 @@ Execution
     schedulers the default is set to 2 seconds.
 
 *xvfb_max_wait*
-    Maximum time (in seconds) to wait for Xvfb to start, if the _redirect_x parameter of an Interface is True.
-
+    Maximum time (in seconds) to wait for Xvfb to start, if the _redirect_x
+    parameter of an Interface is True.
+
+*crashfile_format*
+    This option controls the file type of any crashfile generated. Pklz
+    crashfiles allow interactive debugging and rerunning of nodes, while text
+    crashfiles allow portability across machines and shorter load time.
+    (possible values: ``pklz`` and ``txt``; default value: ``pklz``)
+    
 Example
 ~~~~~~~
 

codecov.yml

@@ -8,7 +8,7 @@ arr=$@
 tmp_var=$( IFS=$' '; echo "${arr[*]}" )
 example_id=${tmp_var//[^A-Za-z0-9_-]/_}
 
-mkdir -p ${HOME}/.nipype ${WORKDIR}/logs/example_${example_id}
+mkdir -p ${HOME}/.nipype ${WORKDIR}/logs/example_${example_id} ${WORKDIR}/tests ${WORKDIR}/crashfiles
 echo "[logging]" > ${HOME}/.nipype/nipype.cfg
 echo "workflow_level = DEBUG" >> ${HOME}/.nipype/nipype.cfg
 echo "interface_level = DEBUG" >> ${HOME}/.nipype/nipype.cfg
@@ -17,23 +17,16 @@ echo "log_to_file = true" >> ${HOME}/.nipype/nipype.cfg
 echo "log_directory = ${WORKDIR}/logs/example_${example_id}" >> ${HOME}/.nipype/nipype.cfg
 
 # Set up coverage
-echo "[run]" >> .coveragerc
-echo "branch = True" >> .coveragerc
-echo "source = /src/nipype" >> .coveragerc
-echo "include = */nipype/*" >> .coveragerc
-echo "omit =" >> .coveragerc
-echo "    */nipype/external/*" >> .coveragerc
-echo "    */nipype/fixes/*" >> .coveragerc
-echo "    */setup.py" >> .coveragerc
-
-
-parallel=""
+export COVERAGE_FILE=${WORKDIR}/tests/.coverage.${example_id}
 if [ "$2" == "MultiProc" ]; then
-	parallel="--concurrency=multiprocessing"
+	echo "concurrency = multiprocessing" >> /src/nipype/.coveragerc
 fi
 
-coverage run ${parallel} /src/nipype/tools/run_examples.py $@
-test_exit=$?
-coverage xml -o "${WORKDIR}/smoketest_${example_id}.xml"
+coverage run /src/nipype/tools/run_examples.py $@
+exit_code=$?
+
+# Collect crashfiles and generate xml report
+coverage xml -o ${WORKDIR}/tests/smoketest_${example_id}.xml
+find /work -name "crash-*" -maxdepth 1 -exec mv {} ${WORKDIR}/crashfiles/ \;
+exit $exit_code
 
-exit $test_exit

doc/users/config_file.rst

@@ -3,11 +3,13 @@ set -e
 set -x
 set -u
 
+
+TESTPATH=${1:-/src/nipype/}
 WORKDIR=${WORK:-/work}
 PYTHON_VERSION=$( python -c "import sys; print('{}{}'.format(sys.version_info[0], sys.version_info[1]))" )
 
 # Create necessary directories
-mkdir -p ${WORKDIR}/crashfiles ${WORKDIR}/logs/py${PYTHON_VERSION}
+mkdir -p ${WORKDIR}/tests ${WORKDIR}/crashfiles ${WORKDIR}/logs/py${PYTHON_VERSION}
 
 # Create a nipype config file
 mkdir -p ${HOME}/.nipype
@@ -21,19 +23,21 @@ if [[ "${PYTHON_VERSION}" -lt "30" ]]; then
     echo 'profile_runtime = true' >> ${HOME}/.nipype/nipype.cfg
 fi
 
-cd /src/nipype/
-make clean
 # Run tests using pytest
-py.test -n ${CIRCLE_NCPUS:-4} --doctest-modules --junitxml=${WORKDIR}/pytests_py${PYTHON_VERSION}.xml --cov-report xml:${WORKDIR}/coverage_py${PYTHON_VERSION}.xml --cov=nipype nipype
-
+export COVERAGE_FILE=${WORKDIR}/tests/.coverage.py${PYTHON_VERSION}
+py.test -v --junitxml=${WORKDIR}/tests/pytests_py${PYTHON_VERSION}.xml --cov nipype --cov-config /src/nipype/.coveragerc --cov-report xml:${WORKDIR}/tests/coverage_py${PYTHON_VERSION}.xml ${TESTPATH}
+exit_code=$?
 
 # Workaround: run here the profiler tests in python 3
 if [[ "${PYTHON_VERSION}" -ge "30" ]]; then
     echo '[execution]' >> ${HOME}/.nipype/nipype.cfg
     echo 'profile_runtime = true' >> ${HOME}/.nipype/nipype.cfg
-    py.test -n ${CIRCLE_NCPUS:-4} --junitxml=${WORKDIR}/pytests_py${PYTHON_VERSION}_profiler.xml --cov-report xml:${WORKDIR}/coverage_py${PYTHON_VERSION}_profiler.xml --cov=nipype nipype/interfaces/tests/test_runtime_profiler.py
-    py.test -n ${CIRCLE_NCPUS:-4} --junitxml=${WORKDIR}/pytests_py${PYTHON_VERSION}_multiproc.xml --cov-report xml:${WORKDIR}/coverage_py${PYTHON_VERSION}_multiproc.xml --cov=nipype nipype/pipeline/plugins/tests/test_multiproc*.py
+    export COVERAGE_FILE=${WORKDIR}/tests/.coverage.py${PYTHON_VERSION}_extra
+    py.test -v --junitxml=${WORKDIR}/tests/pytests_py${PYTHON_VERSION}_extra.xml --cov nipype --cov-report xml:${WORKDIR}/tests/coverage_py${PYTHON_VERSION}_extra.xml /src/nipype/nipype/interfaces/tests/test_runtime_profiler.py /src/nipype/nipype/pipeline/plugins/tests/test_multiproc*.py
+    exit_code=$(( $exit_code + $? ))
 fi
 
-# Copy crashfiles to scratch
-find /src/nipype/ -name "crash-*" -exec cp {} ${WORKDIR}/crashfiles/ \;
+# Collect crashfiles
+find ${WORKDIR} -name "crash-*" -maxdepth 1 -exec mv {} ${WORKDIR}/crashfiles/ \;
+
+exit ${exit_code}