From 5f94d90b7dd03c1ae495594f3f8a76b51a78babe Mon Sep 17 00:00:00 2001 From: delavega4 Date: Wed, 16 Jan 2019 15:06:59 -0500 Subject: [PATCH 01/41] Update pybids to 0.7 --- Singularity | 4 +- generate.sh | 4 +- notebooks/basic_data_input_bids.ipynb | 265 ++++++-------------------- 3 files changed, 64 insertions(+), 209 deletions(-) diff --git a/Singularity b/Singularity index d43a3b5..c3665d7 100644 --- a/Singularity +++ b/Singularity @@ -122,7 +122,7 @@ sync && conda clean -tipsy && sync bash -c "source activate neuro pip install --no-cache-dir \ https://github.com/nipy/nipype/tarball/master \ - https://github.com/INCF/pybids/tarball/0.6.5 \ + https://github.com/INCF/pybids/tarball/0.7 \ nilearn \ datalad[full] \ nipy \ @@ -232,7 +232,7 @@ echo '{ \n ], \n "pip_install": [ \n "https://github.com/nipy/nipype/tarball/master", -\n "https://github.com/INCF/pybids/tarball/0.6.5", +\n "https://github.com/INCF/pybids/tarball/0.7", \n "nilearn", \n "datalad[full]", \n "nipy", diff --git a/generate.sh b/generate.sh index 7e5cca6..4e523ef 100644 --- a/generate.sh +++ b/generate.sh @@ -17,7 +17,7 @@ generate_docker() { conda_install="python=3.6 pytest jupyter jupyterlab jupyter_contrib_nbextensions traits pandas matplotlib scikit-learn scikit-image seaborn nbformat nb_conda" \ pip_install="https://github.com/nipy/nipype/tarball/master-%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20https://github.com/INCF/pybids/tarball/0.6.5+%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20https://github.com/INCF/pybids/tarball/0.7%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20nilearn%20datalad[full]%20nipy%20duecredit%20nbval" \ create_env="neuro" \ activate=True \ @@ -55,7 +55,7 @@ generate_singularity() { conda_install="python=3.6 pytest jupyter jupyterlab jupyter_contrib_nbextensions traits pandas matplotlib scikit-learn scikit-image seaborn nbformat nb_conda" \ pip_install="https://github.com/nipy/nipype/tarball/master-%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20https://github.com/INCF/pybids/tarball/0.6.5+%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20https://github.com/INCF/pybids/tarball/0.7%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20nilearn%20datalad[full]%20nipy%20duecredit%20nbval" \ create_env="neuro" \ activate=True \ diff --git a/notebooks/basic_data_input_bids.ipynb b/notebooks/basic_data_input_bids.ipynb index b6e4d67..3275f9d 100644 --- a/notebooks/basic_data_input_bids.ipynb +++ b/notebooks/basic_data_input_bids.ipynb @@ -2,10 +2,7 @@ "cells": [ { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Data input for BIDS datasets\n", "`DataGrabber` and `SelectFiles` are great if you are dealing with generic datasets with arbitrary organization. However, if you have decided to use Brain Imaging Data Structure (BIDS) to organize your data (or got your hands on a BIDS dataset) you can take advantage of a formal structure BIDS imposes. In this short tutorial, you will learn how to do this." @@ -13,10 +10,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## `pybids` - a Python API for working with BIDS datasets\n", "`pybids` is a lightweight python API for querying BIDS folder structure for specific files and metadata. You can install it from PyPi:\n", @@ -28,10 +22,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## The `layout` object and simple queries\n", "To begin working with pybids we need to initialize a layout object. We will need it to do all of our queries" @@ -40,11 +31,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from bids.layout import BIDSLayout\n", @@ -54,11 +41,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "!tree -L 4 /data/ds000114/" @@ -66,10 +49,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Let's figure out what are the subject labels in this dataset" ] @@ -77,11 +57,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "layout.get_subjects()" @@ -89,56 +65,39 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ - "What modalities are included in this dataset?" + "What datatypes are included in this dataset?" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ - "layout.get_modalities()" + "layout.get_datatypes()" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ - "Which different data types are included in this dataset?" + "Which different data suffixes are included in this dataset?" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ - "layout.get_types(modality='func')" + "layout.get_suffixes(datatype='func')" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "What are the different tasks included in this dataset?" ] @@ -146,11 +105,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "layout.get_tasks()" @@ -158,33 +113,23 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ - "We can also ask for all of the data for a particular subject and one modality." + "We can also ask for all of the data for a particular subject and one datatype." ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ - "layout.get(subject='01', modality=\"anat\", session=\"test\")" + "layout.get(subject='01', datatype=\"anat\", session=\"test\")" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "We can also ask for a specific subset of data. Note that we are using extension filter to get just the imaging data (BIDS allows both .nii and .nii.gz so we need to include both)." ] @@ -192,22 +137,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ - "layout.get(subject='01', type='bold', extensions=['nii', 'nii.gz'])" + "layout.get(subject='01', suffix='bold', extensions=['nii', 'nii.gz'])" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "You probably noticed that this method does not only return the file paths, but objects with relevant query fields. We can easily extract just the file paths." ] @@ -215,22 +153,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ - "layout.get(subject='01', type='bold', extensions=['nii', 'nii.gz'], return_type='file')" + "layout.get(subject='01', suffix='bold', extensions=['nii', 'nii.gz'], return_type='file')" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "### Exercise 1:\n", "List all files for the \"linebisection\" task for subject 02." @@ -240,9 +171,6 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true, "solution2": "shown", "solution2_first": true }, @@ -255,9 +183,6 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true, "solution2": "shown" }, "outputs": [], @@ -270,10 +195,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## `BIDSDataGrabber`: Including `pybids` in your `nipype` workflow\n", "This is great, but what we really want is to include this into our nipype workflows. To do this, we can import `BIDSDataGrabber`, which provides an `Interface` for `BIDSLayout.get`" @@ -282,11 +204,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from nipype.interfaces.io import BIDSDataGrabber\n", @@ -299,10 +217,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "You can define static filters, that will apply to all queries, by modifying the appropriate input" ] @@ -310,11 +225,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "bg.inputs.subject = '01'\n", @@ -324,12 +235,9 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ - "Note that by default `BIDSDataGrabber` will fetch `nifti` files matching modality `func` and `anat`, and output them as two output fields. \n", + "Note that by default `BIDSDataGrabber` will fetch `nifti` files matching datatype `func` and `anat`, and output them as two output fields. \n", "\n", "To define custom fields, simply define the arguments to pass to `BIDSLayout.get` as dictionary, like so:" ] @@ -337,26 +245,19 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ - "bg.inputs.output_query = {'bolds': dict(type='bold')}\n", + "bg.inputs.output_query = {'bolds': dict(suffix='bold')}\n", "res = bg.run()\n", "res.outputs" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ - "This results in a single output field `bold`, which returns all files with `type:bold` for `subject:\"01\"` \n", + "This results in a single output field `bold`, which returns all files with `suffix:bold` for `subject:\"01\"` \n", "\n", "Now, lets put it in a workflow. We are not going to analyze any data, but for demonstration purposes, we will add a couple of nodes that pretend to analyze their inputs" ] @@ -364,11 +265,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def printMe(paths):\n", @@ -381,11 +278,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "wf = Workflow(name=\"bids_demo\")\n", @@ -395,10 +288,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "### Exercise 2:\n", "Modify the `BIDSDataGrabber` and the workflow to collect T1ws images for subject `10`." @@ -408,9 +298,6 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true, "solution2": "shown", "solution2_first": true }, @@ -423,9 +310,6 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true, "solution2": "shown" }, "outputs": [], @@ -436,7 +320,7 @@ "ex2_BIDSDataGrabber = BIDSDataGrabber()\n", "ex2_BIDSDataGrabber.inputs.base_dir = '/data/ds000114'\n", "ex2_BIDSDataGrabber.inputs.subject = '10'\n", - "ex2_BIDSDataGrabber.inputs.output_query = {'T1w': dict(modality='anat')}\n", + "ex2_BIDSDataGrabber.inputs.output_query = {'T1w': dict(datatype='anat')}\n", "\n", "ex2_res = ex2_BIDSDataGrabber.run()\n", "ex2_res.outputs" @@ -444,10 +328,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Iterating over subject labels\n", "In the previous example, we demonstrated how to use `pybids` to \"analyze\" one subject. How can we scale it for all subjects? Easy - using `iterables` (more in [Iteration/Iterables](basic_iteration.ipynb))." @@ -456,16 +337,12 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "bg_all = Node(BIDSDataGrabber(), name='bids-grabber')\n", "bg_all.inputs.base_dir = '/data/ds000114'\n", - "bg_all.inputs.output_query = {'bolds': dict(type='bold')}\n", + "bg_all.inputs.output_query = {'bolds': dict(suffix='bold')}\n", "bg_all.iterables = ('subject', layout.get_subjects()[:2])\n", "wf = Workflow(name=\"bids_demo\")\n", "wf.connect(bg_all, \"bolds\", analyzeBOLD, \"paths\")\n", @@ -474,10 +351,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Accessing additional metadata\n", "Querying different files is nice, but sometimes you want to access more metadata. For example `RepetitionTime`. `pybids` can help with that as well" @@ -486,11 +360,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "layout.get_metadata('/data/ds000114/sub-01/ses-test/func/sub-01_ses-test_task-fingerfootlips_bold.nii.gz')" @@ -498,10 +368,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Can we incorporate this into our pipeline? Yes, we can! To do so, let's use a `Function` node to use `BIDSLayout` in a custom way.\n", "(More about MapNode in [MapNode](basic_mapnodes.ipynb))" @@ -510,11 +377,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def printMetadata(path, data_dir):\n", @@ -531,9 +394,6 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true, "scrolled": false }, "outputs": [], @@ -545,10 +405,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "### Exercise 3:\n", "Modify the `printMetadata` function to also print `EchoTime` " @@ -558,9 +415,6 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true, "solution2": "shown", "solution2_first": true }, @@ -573,9 +427,6 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true, "solution2": "shown" }, "outputs": [], @@ -586,16 +437,13 @@ "ex3_BIDSDataGrabber = Node(BIDSDataGrabber(), name='bids-grabber')\n", "ex3_BIDSDataGrabber.inputs.base_dir = '/data/ds000114'\n", "ex3_BIDSDataGrabber.inputs.subject = '01'\n", - "ex3_BIDSDataGrabber.inputs.output_query = {'bolds': dict(type='bold')}" + "ex3_BIDSDataGrabber.inputs.output_query = {'bolds': dict(suffix='bold')}" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true, "solution2": "shown" }, "outputs": [], @@ -619,11 +467,18 @@ "ex3_wf.connect(ex3_BIDSDataGrabber, \"bolds\", ex3_analyzeBOLD2, \"path\")\n", "ex3_wf.run()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python [default]", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -637,7 +492,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.6" + "version": "3.6.7" } }, "nbformat": 4, From f39141375ed4a879906252afd6679f0136f4e3c4 Mon Sep 17 00:00:00 2001 From: Michael Notter Date: Thu, 2 May 2019 12:28:13 +0200 Subject: [PATCH 02/41] Update pybids to v 0.7 in Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a07d650..878bad1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -132,7 +132,7 @@ RUN export PATH="/opt/miniconda-latest/bin:$PATH" \ && bash -c "source activate neuro \ && pip install --no-cache-dir \ https://github.com/nipy/nipype/tarball/master \ - https://github.com/INCF/pybids/tarball/0.6.5 \ + https://github.com/INCF/pybids/tarball/0.7 \ nilearn \ datalad[full] \ nipy \ @@ -240,7 +240,7 @@ RUN echo '{ \ \n ], \ \n "pip_install": [ \ \n "https://github.com/nipy/nipype/tarball/master", \ - \n "https://github.com/INCF/pybids/tarball/0.6.5", \ + \n "https://github.com/INCF/pybids/tarball/0.7", \ \n "nilearn", \ \n "datalad[full]", \ \n "nipy", \ From be6755ca372d9bc2106d219b3b97236bb9cb032b Mon Sep 17 00:00:00 2001 From: Michael Notter Date: Thu, 2 May 2019 12:41:16 +0200 Subject: [PATCH 03/41] Update pybids to 0.7.1 in Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 878bad1..f3b897d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -132,7 +132,7 @@ RUN export PATH="/opt/miniconda-latest/bin:$PATH" \ && bash -c "source activate neuro \ && pip install --no-cache-dir \ https://github.com/nipy/nipype/tarball/master \ - https://github.com/INCF/pybids/tarball/0.7 \ + https://github.com/INCF/pybids/tarball/0.7.1 \ nilearn \ datalad[full] \ nipy \ @@ -240,7 +240,7 @@ RUN echo '{ \ \n ], \ \n "pip_install": [ \ \n "https://github.com/nipy/nipype/tarball/master", \ - \n "https://github.com/INCF/pybids/tarball/0.7", \ + \n "https://github.com/INCF/pybids/tarball/0.7.1", \ \n "nilearn", \ \n "datalad[full]", \ \n "nipy", \ From 496e29de2762d3d9f47a91fc1b8dd7e80364a3ee Mon Sep 17 00:00:00 2001 From: Michael Notter Date: Thu, 2 May 2019 12:41:38 +0200 Subject: [PATCH 04/41] Update pybids to 0.7.1 in Singularity --- Singularity | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Singularity b/Singularity index c3665d7..7428ca8 100644 --- a/Singularity +++ b/Singularity @@ -122,7 +122,7 @@ sync && conda clean -tipsy && sync bash -c "source activate neuro pip install --no-cache-dir \ https://github.com/nipy/nipype/tarball/master \ - https://github.com/INCF/pybids/tarball/0.7 \ + https://github.com/INCF/pybids/tarball/0.7.1 \ nilearn \ datalad[full] \ nipy \ @@ -232,7 +232,7 @@ echo '{ \n ], \n "pip_install": [ \n "https://github.com/nipy/nipype/tarball/master", -\n "https://github.com/INCF/pybids/tarball/0.7", +\n "https://github.com/INCF/pybids/tarball/0.7.1", \n "nilearn", \n "datalad[full]", \n "nipy", From 3682357f2d73c594217dc4e2255ec9e1361a2888 Mon Sep 17 00:00:00 2001 From: Michael Notter Date: Thu, 2 May 2019 12:41:56 +0200 Subject: [PATCH 05/41] Update pybids to 0.7.1 in generate.sh --- generate.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/generate.sh b/generate.sh index 4e523ef..b8f893a 100644 --- a/generate.sh +++ b/generate.sh @@ -17,7 +17,8 @@ generate_docker() { conda_install="python=3.6 pytest jupyter jupyterlab jupyter_contrib_nbextensions traits pandas matplotlib scikit-learn scikit-image seaborn nbformat nb_conda" \ pip_install="https://github.com/nipy/nipype/tarball/master-%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20https://github.com/INCF/pybids/tarball/0.7+%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20https://github.com/INCF/py+%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20/tarball/0.7.1%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20nilearn%20datalad[full]%20nipy%20duecredit%20nbval" \ create_env="neuro" \ activate=True \ @@ -55,7 +56,7 @@ generate_singularity() { conda_install="python=3.6 pytest jupyter jupyterlab jupyter_contrib_nbextensions traits pandas matplotlib scikit-learn scikit-image seaborn nbformat nb_conda" \ pip_install="https://github.com/nipy/nipype/tarball/master-%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20https://github.com/INCF/pybids/tarball/0.7+%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20https://github.com/INCF/pybids/tarball/0.7.1%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20nilearn%20datalad[full]%20nipy%20duecredit%20nbval" \ create_env="neuro" \ activate=True \ From 3d89a3d2c11f4c78f2f891e79f1c19862a8ddcc9 Mon Sep 17 00:00:00 2001 From: Michael Notter Date: Thu, 2 May 2019 13:15:52 +0200 Subject: [PATCH 06/41] FIX: Typo correction --- generate.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/generate.sh b/generate.sh index b8f893a..8be7dab 100644 --- a/generate.sh +++ b/generate.sh @@ -17,8 +17,7 @@ generate_docker() { conda_install="python=3.6 pytest jupyter jupyterlab jupyter_contrib_nbextensions traits pandas matplotlib scikit-learn scikit-image seaborn nbformat nb_conda" \ pip_install="https://github.com/nipy/nipype/tarball/master-%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20https://github.com/INCF/py-%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20/tarball/0.7.1+%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20https://github.com/INCF/pybids/tarball/0.7.1%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20nilearn%20datalad[full]%20nipy%20duecredit%20nbval" \ create_env="neuro" \ activate=True \ From 1359b0bfa99056ffcf3f43c9254d9244e94db009 Mon Sep 17 00:00:00 2001 From: miykael Date: Sun, 14 Jul 2019 10:54:51 +0200 Subject: [PATCH 07/41] Update generation file --- Dockerfile | 77 ++++++++++++++++++++++++++++++----------------------- Singularity | 75 ++++++++++++++++++++++++++++----------------------- generate.sh | 2 ++ 3 files changed, 87 insertions(+), 67 deletions(-) diff --git a/Dockerfile b/Dockerfile index a07d650..27a7ca4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -# Generated by Neurodocker version 0.4.2-dev -# Timestamp: 2018-10-15 12:01:29 UTC +# Generated by Neurodocker version 0.5.0 +# Timestamp: 2019-07-14 08:54:07 UTC # # Thank you for using Neurodocker. If you discover any issues # or ways to improve this software, please submit an issue or @@ -24,7 +24,7 @@ RUN export ND_ENTRYPOINT="/neurodocker/startup.sh" \ locales \ unzip \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && rm -rf /var/lib/apt/lists/* \ && sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ && dpkg-reconfigure --frontend=noninteractive locales \ && update-locale LANG="en_US.UTF-8" \ @@ -33,6 +33,7 @@ RUN export ND_ENTRYPOINT="/neurodocker/startup.sh" \ && if [ ! -f "$ND_ENTRYPOINT" ]; then \ echo '#!/usr/bin/env bash' >> "$ND_ENTRYPOINT" \ && echo 'set -e' >> "$ND_ENTRYPOINT" \ + && echo 'export USER="${USER:=`whoami`}"' >> "$ND_ENTRYPOINT" \ && echo 'if [ -n "$1" ]; then "$@"; else /usr/bin/env bash; fi' >> "$ND_ENTRYPOINT"; \ fi \ && chmod -R 777 /neurodocker && chmod a+s /neurodocker @@ -59,14 +60,15 @@ RUN apt-get update -qq \ octave \ netbase \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + && rm -rf /var/lib/apt/lists/* RUN sed -i '$isource /etc/fsl/fsl.sh' $ND_ENTRYPOINT ENV FORCE_SPMMCR="1" \ LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu:/opt/matlabmcr-2010a/v713/runtime/glnxa64:/opt/matlabmcr-2010a/v713/bin/glnxa64:/opt/matlabmcr-2010a/v713/sys/os/glnxa64:/opt/matlabmcr-2010a/v713/extern/bin/glnxa64" \ MATLABCMD="/opt/matlabmcr-2010a/v713/toolbox/matlab" -RUN apt-get update -qq \ +RUN export TMPDIR="$(mktemp -d)" \ + && apt-get update -qq \ && apt-get install -y -q --no-install-recommends \ bc \ libncurses5 \ @@ -75,18 +77,19 @@ RUN apt-get update -qq \ libxpm-dev \ libxt6 \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && rm -rf /var/lib/apt/lists/* \ && echo "Downloading MATLAB Compiler Runtime ..." \ && curl -sSL --retry 5 -o /tmp/toinstall.deb http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb \ && dpkg -i /tmp/toinstall.deb \ && rm /tmp/toinstall.deb \ && apt-get install -f \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ - && curl -fsSL --retry 5 -o /tmp/MCRInstaller.bin https://dl.dropbox.com/s/zz6me0c3v4yq5fd/MCR_R2010a_glnxa64_installer.bin \ - && chmod +x /tmp/MCRInstaller.bin \ - && /tmp/MCRInstaller.bin -silent -P installLocation="/opt/matlabmcr-2010a" \ - && rm -rf /tmp/* \ + && rm -rf /var/lib/apt/lists/* \ + && curl -fsSL --retry 5 -o "$TMPDIR/MCRInstaller.bin" https://dl.dropbox.com/s/zz6me0c3v4yq5fd/MCR_R2010a_glnxa64_installer.bin \ + && chmod +x "$TMPDIR/MCRInstaller.bin" \ + && "$TMPDIR/MCRInstaller.bin" -silent -P installLocation="/opt/matlabmcr-2010a" \ + && rm -rf "$TMPDIR" \ + && unset TMPDIR \ && echo "Downloading standalone SPM ..." \ && curl -fsSL --retry 5 -o /tmp/spm12.zip http://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/previous/spm12_r7219_R2010a.zip \ && unzip -q /tmp/spm12.zip -d /tmp \ @@ -97,9 +100,11 @@ RUN apt-get update -qq \ && /opt/spm12-r7219/run_spm12.sh /opt/matlabmcr-2010a/v713 quit \ && sed -i '$iexport SPMMCRCMD=\"/opt/spm12-r7219/run_spm12.sh /opt/matlabmcr-2010a/v713 script\"' $ND_ENTRYPOINT -RUN useradd --no-user-group --create-home --shell /bin/bash neuro +RUN test "$(getent passwd neuro)" || useradd --no-user-group --create-home --shell /bin/bash neuro USER neuro +WORKDIR /home/neuro + ENV CONDA_DIR="/opt/miniconda-latest" \ PATH="/opt/miniconda-latest/bin:$PATH" RUN export PATH="/opt/miniconda-latest/bin:$PATH" \ @@ -112,32 +117,32 @@ RUN export PATH="/opt/miniconda-latest/bin:$PATH" \ && conda config --system --prepend channels conda-forge \ && conda config --system --set auto_update_conda false \ && conda config --system --set show_channel_urls true \ - && sync && conda clean -tipsy && sync \ + && sync && conda clean --all && sync \ && conda create -y -q --name neuro \ && conda install -y -q --name neuro \ - python=3.6 \ - pytest \ - jupyter \ - jupyterlab \ - jupyter_contrib_nbextensions \ - traits \ - pandas \ - matplotlib \ - scikit-learn \ - scikit-image \ - seaborn \ - nbformat \ - nb_conda \ - && sync && conda clean -tipsy && sync \ + 'python=3.6' \ + 'pytest' \ + 'jupyter' \ + 'jupyterlab' \ + 'jupyter_contrib_nbextensions' \ + 'traits' \ + 'pandas' \ + 'matplotlib' \ + 'scikit-learn' \ + 'scikit-image' \ + 'seaborn' \ + 'nbformat' \ + 'nb_conda' \ + && sync && conda clean --all && sync \ && bash -c "source activate neuro \ && pip install --no-cache-dir \ - https://github.com/nipy/nipype/tarball/master \ - https://github.com/INCF/pybids/tarball/0.6.5 \ - nilearn \ - datalad[full] \ - nipy \ - duecredit \ - nbval" \ + 'https://github.com/nipy/nipype/tarball/master' \ + 'https://github.com/INCF/pybids/tarball/0.6.5' \ + 'nilearn' \ + 'datalad[full]' \ + 'nipy' \ + 'duecredit' \ + 'nbval'" \ && rm -rf ~/.cache/pip/* \ && sync \ && sed -i '$isource activate neuro' $ND_ENTRYPOINT @@ -220,6 +225,10 @@ RUN echo '{ \ \n "neuro" \ \n ], \ \n [ \ + \n "workdir", \ + \n "/home/neuro" \ + \n ], \ + \n [ \ \n "miniconda", \ \n { \ \n "miniconda_version": "4.3.31", \ diff --git a/Singularity b/Singularity index d43a3b5..736ceaf 100644 --- a/Singularity +++ b/Singularity @@ -1,5 +1,5 @@ -# Generated by Neurodocker version 0.4.2-dev -# Timestamp: 2018-10-15 12:01:30 UTC +# Generated by Neurodocker version 0.5.0 +# Timestamp: 2019-07-14 08:54:09 UTC # # Thank you for using Neurodocker. If you discover any issues # or ways to improve this software, please submit an issue or @@ -21,7 +21,7 @@ apt-get install -y -q --no-install-recommends \ locales \ unzip apt-get clean -rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +rm -rf /var/lib/apt/lists/* sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen dpkg-reconfigure --frontend=noninteractive locales update-locale LANG="en_US.UTF-8" @@ -30,6 +30,7 @@ mkdir -p /neurodocker if [ ! -f "$ND_ENTRYPOINT" ]; then echo '#!/usr/bin/env bash' >> "$ND_ENTRYPOINT" echo 'set -e' >> "$ND_ENTRYPOINT" + echo 'export USER="${USER:=`whoami`}"' >> "$ND_ENTRYPOINT" echo 'if [ -n "$1" ]; then "$@"; else /usr/bin/env bash; fi' >> "$ND_ENTRYPOINT"; fi chmod -R 777 /neurodocker && chmod a+s /neurodocker @@ -54,10 +55,11 @@ apt-get install -y -q --no-install-recommends \ octave \ netbase apt-get clean -rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +rm -rf /var/lib/apt/lists/* sed -i '$isource /etc/fsl/fsl.sh' $ND_ENTRYPOINT +export TMPDIR="$(mktemp -d)" apt-get update -qq apt-get install -y -q --no-install-recommends \ bc \ @@ -67,18 +69,19 @@ apt-get install -y -q --no-install-recommends \ libxpm-dev \ libxt6 apt-get clean -rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +rm -rf /var/lib/apt/lists/* echo "Downloading MATLAB Compiler Runtime ..." curl -sSL --retry 5 -o /tmp/toinstall.deb http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb dpkg -i /tmp/toinstall.deb rm /tmp/toinstall.deb apt-get install -f apt-get clean -rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -curl -fsSL --retry 5 -o /tmp/MCRInstaller.bin https://dl.dropbox.com/s/zz6me0c3v4yq5fd/MCR_R2010a_glnxa64_installer.bin -chmod +x /tmp/MCRInstaller.bin -/tmp/MCRInstaller.bin -silent -P installLocation="/opt/matlabmcr-2010a" -rm -rf /tmp/* +rm -rf /var/lib/apt/lists/* +curl -fsSL --retry 5 -o "$TMPDIR/MCRInstaller.bin" https://dl.dropbox.com/s/zz6me0c3v4yq5fd/MCR_R2010a_glnxa64_installer.bin +chmod +x "$TMPDIR/MCRInstaller.bin" +"$TMPDIR/MCRInstaller.bin" -silent -P installLocation="/opt/matlabmcr-2010a" +rm -rf "$TMPDIR" +unset TMPDIR echo "Downloading standalone SPM ..." curl -fsSL --retry 5 -o /tmp/spm12.zip http://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/previous/spm12_r7219_R2010a.zip unzip -q /tmp/spm12.zip -d /tmp @@ -89,9 +92,11 @@ rm -rf /tmp/* /opt/spm12-r7219/run_spm12.sh /opt/matlabmcr-2010a/v713 quit sed -i '$iexport SPMMCRCMD=\"/opt/spm12-r7219/run_spm12.sh /opt/matlabmcr-2010a/v713 script\"' $ND_ENTRYPOINT -useradd --no-user-group --create-home --shell /bin/bash neuro +test "$(getent passwd neuro)" || useradd --no-user-group --create-home --shell /bin/bash neuro su - neuro +cd /home/neuro + export PATH="/opt/miniconda-latest/bin:$PATH" echo "Downloading Miniconda installer ..." conda_installer="/tmp/miniconda.sh" @@ -102,32 +107,32 @@ conda update -yq -nbase conda conda config --system --prepend channels conda-forge conda config --system --set auto_update_conda false conda config --system --set show_channel_urls true -sync && conda clean -tipsy && sync +sync && conda clean --all && sync conda create -y -q --name neuro conda install -y -q --name neuro \ - python=3.6 \ - pytest \ - jupyter \ - jupyterlab \ - jupyter_contrib_nbextensions \ - traits \ - pandas \ - matplotlib \ - scikit-learn \ - scikit-image \ - seaborn \ - nbformat \ - nb_conda -sync && conda clean -tipsy && sync + 'python=3.6' \ + 'pytest' \ + 'jupyter' \ + 'jupyterlab' \ + 'jupyter_contrib_nbextensions' \ + 'traits' \ + 'pandas' \ + 'matplotlib' \ + 'scikit-learn' \ + 'scikit-image' \ + 'seaborn' \ + 'nbformat' \ + 'nb_conda' +sync && conda clean --all && sync bash -c "source activate neuro pip install --no-cache-dir \ - https://github.com/nipy/nipype/tarball/master \ - https://github.com/INCF/pybids/tarball/0.6.5 \ - nilearn \ - datalad[full] \ - nipy \ - duecredit \ - nbval" + 'https://github.com/nipy/nipype/tarball/master' \ + 'https://github.com/INCF/pybids/tarball/0.6.5' \ + 'nilearn' \ + 'datalad[full]' \ + 'nipy' \ + 'duecredit' \ + 'nbval'" rm -rf ~/.cache/pip/* sync sed -i '$isource activate neuro' $ND_ENTRYPOINT @@ -212,6 +217,10 @@ echo '{ \n "neuro" \n ], \n [ +\n "workdir", +\n "/home/neuro" +\n ], +\n [ \n "miniconda", \n { \n "miniconda_version": "4.3.31", diff --git a/generate.sh b/generate.sh index 7e5cca6..8c2cefe 100644 --- a/generate.sh +++ b/generate.sh @@ -13,6 +13,7 @@ generate_docker() { --add-to-entrypoint "source /etc/fsl/fsl.sh" \ --spm12 version=r7219 \ --user=neuro \ + --workdir /home/neuro \ --miniconda miniconda_version="4.3.31" \ conda_install="python=3.6 pytest jupyter jupyterlab jupyter_contrib_nbextensions traits pandas matplotlib scikit-learn scikit-image seaborn nbformat nb_conda" \ @@ -51,6 +52,7 @@ generate_singularity() { --add-to-entrypoint "source /etc/fsl/fsl.sh" \ --spm12 version=r7219 \ --user=neuro \ + --workdir /home/neuro \ --miniconda miniconda_version="4.3.31" \ conda_install="python=3.6 pytest jupyter jupyterlab jupyter_contrib_nbextensions traits pandas matplotlib scikit-learn scikit-image seaborn nbformat nb_conda" \ From 4394957465e050ea675e1cdcf38a043960043c12 Mon Sep 17 00:00:00 2001 From: Michael Notter Date: Tue, 23 Jul 2019 21:56:39 +0200 Subject: [PATCH 08/41] [FIX]: Update broken mindboggle link --- index.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.ipynb b/index.ipynb index 44a9920..edb048d 100644 --- a/index.ipynb +++ b/index.ipynb @@ -303,7 +303,7 @@ " BIDS Apps\n", " fmriprep\n", " MRIQC\n", - " Mindboggle\n", + " Mindboggle\n", " PORcupine\n", " \n", "

This section will give you helpful links and resources so that you always know where to go to learn more.

\n", From 0571c9a7224bbca98b827516a26c2f610f76fc00 Mon Sep 17 00:00:00 2001 From: Michael Notter Date: Tue, 23 Jul 2019 22:10:27 +0200 Subject: [PATCH 09/41] [FIX]: Update mindboggle link on html file --- docs/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.html b/docs/index.html index 40d8f6b..a460951 100644 --- a/docs/index.html +++ b/docs/index.html @@ -11914,7 +11914,7 @@ <a class="subject-link pure-u-1-4" target="_blank" href="http://bids-apps.neuroimaging.io">BIDS Apps</a> <a class="subject-link pure-u-1-4" target="_blank" href="http://fmriprep.readthedocs.io/en/latest/index.html">fmriprep</a> <a class="subject-link pure-u-1-4" target="_blank" href="https://mriqc.readthedocs.io/en/latest/#">MRIQC</a> - <a class="subject-link pure-u-1-4" target="_blank" href="https://www.mindboggle.info/">Mindboggle</a> + <a class="subject-link pure-u-1-4" target="_blank" href="https://mindboggle.info/">Mindboggle</a> <a class="subject-link pure-u-1-4" target="_blank" href="https://timvanmourik.github.io/Porcupine/">PORcupine</a> </div> <p>This section will give you helpful links and resources so that you always know where to go to learn more.</p> @@ -12085,7 +12085,7 @@

Useful Resources & Lin BIDS Apps fmriprep MRIQC - Mindboggle + Mindboggle PORcupine

This section will give you helpful links and resources so that you always know where to go to learn more.

From 85e21fe0b246a81ae480e86cb38335ff57e83a7f Mon Sep 17 00:00:00 2001 From: Michael Notter Date: Tue, 23 Jul 2019 22:12:55 +0200 Subject: [PATCH 10/41] [FIX]: Update broken nipypecli link --- docs/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.html b/docs/index.html index a460951..0ac42d3 100644 --- a/docs/index.html +++ b/docs/index.html @@ -11891,7 +11891,7 @@ <div class="pure-g domain-table-container color04"> <a class="subject-link pure-u-1-4" target="_blank" href="notebooks/advanced_create_interfaces.html">Create Interfaces</a> <a class="subject-link pure-u-1-4" target="_blank" href="notebooks/advanced_interfaces_caching.html">Interfaces Caching</a> - <a class="subject-link pure-u-1-4" target="_blank" href="notebooks/advanced_nipypecli.html">Nipype Command Line Interface</a> + <a class="subject-link pure-u-1-4" target="_blank" href="notebooks/advanced_command_line_interface.html">Nipype Command Line Interface</a> <a class="subject-link pure-u-1-4" target="_blank" href="notebooks/advanced_aws.html">Amazon Web Services (AWS)</a> <a class="subject-link pure-u-1-4" target="_blank" href="notebooks/advanced_sphinx_ext.html">Sphinx extensions</a> <a class="subject-link pure-u-1-4" target="_blank" href="notebooks/advanced_spmmcr.html">SPM with MATLAB Common Runtime (MCR)</a> @@ -12062,7 +12062,7 @@

Advanced Concepts<
Create Interfaces Interfaces Caching - Nipype Command Line Interface + Nipype Command Line Interface Amazon Web Services (AWS) Sphinx extensions SPM with MATLAB Common Runtime (MCR) From 27df96a803870c707c7a3a21768084c906b14e0f Mon Sep 17 00:00:00 2001 From: Michael Notter Date: Tue, 23 Jul 2019 22:14:06 +0200 Subject: [PATCH 11/41] [FIX]: Update broken links --- index.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/index.ipynb b/index.ipynb index edb048d..99e977c 100644 --- a/index.ipynb +++ b/index.ipynb @@ -111,7 +111,7 @@ "
\n", " Create Interfaces\n", " Interfaces Caching\n", - " Nipype Command Line Interface\n", + " Nipype Command Line Interface\n", " Amazon Web Services (AWS)\n", " Sphinx extensions\n", " SPM with MATLAB Common Runtime (MCR)\n", @@ -134,7 +134,7 @@ " BIDS Apps\n", " fmriprep\n", " MRIQC\n", - " Mindboggle\n", + " Mindboggle\n", " PORcupine\n", "
\n", "

This section will give you helpful links and resources so that you always know where to go to learn more.

\n", @@ -280,7 +280,7 @@ "
\n", " Create Interfaces\n", " Interfaces Caching\n", - " Nipype Command Line Interface\n", + " Nipype Command Line Interface\n", " Amazon Web Services (AWS)\n", " Sphinx extensions\n", " SPM with MATLAB Common Runtime (MCR)\n", From 86e45ac522d98235996701adab416e32bee04479 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 10 Sep 2019 15:46:45 -0400 Subject: [PATCH 12/41] [DATALAD RUNCMD] git-sedi felicitate facilitate === Do not change lines below === { "chain": [], "cmd": "git-sedi felicitate facilitate", "exit": 0, "extra_inputs": [], "inputs": [], "outputs": [], "pwd": "." } ^^^ Do not change lines above ^^^ --- docs/notebooks/basic_data_input.html | 2 +- notebooks/basic_data_input.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/notebooks/basic_data_input.html b/docs/notebooks/basic_data_input.html index 4bc10f8..4d1d3d9 100644 --- a/docs/notebooks/basic_data_input.html +++ b/docs/notebooks/basic_data_input.html @@ -12816,7 +12816,7 @@

Exercise 2
-

FreeSurferSource

FreeSurferSource is a specific case of a file grabber that felicitates the data import of outputs from the FreeSurfer recon-all algorithm. This, of course, requires that you've already run recon-all on your subject.

+

FreeSurferSource

FreeSurferSource is a specific case of a file grabber that facilitates the data import of outputs from the FreeSurfer recon-all algorithm. This, of course, requires that you've already run recon-all on your subject.

diff --git a/notebooks/basic_data_input.ipynb b/notebooks/basic_data_input.ipynb index e8636cf..e40f7ec 100644 --- a/notebooks/basic_data_input.ipynb +++ b/notebooks/basic_data_input.ipynb @@ -635,7 +635,7 @@ "source": [ "## FreeSurferSource\n", "\n", - "`FreeSurferSource` is a specific case of a file grabber that felicitates the data import of outputs from the FreeSurfer recon-all algorithm. This, of course, requires that you've already run `recon-all` on your subject." + "`FreeSurferSource` is a specific case of a file grabber that facilitates the data import of outputs from the FreeSurfer recon-all algorithm. This, of course, requires that you've already run `recon-all` on your subject." ] }, { From 0473abef50332320bc4053c243bad242b03e446c Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 27 Jan 2020 14:45:46 -0500 Subject: [PATCH 13/41] FIX: Datalad command --- notebooks/basic_data_input.ipynb | 2 +- notebooks/example_normalize.ipynb | 2 +- notebooks/example_preprocessing.ipynb | 5 +++-- notebooks/handson_preprocessing.ipynb | 5 +++-- notebooks/introduction_dataset.ipynb | 8 ++++---- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/notebooks/basic_data_input.ipynb b/notebooks/basic_data_input.ipynb index e8636cf..a9bd700 100644 --- a/notebooks/basic_data_input.ipynb +++ b/notebooks/basic_data_input.ipynb @@ -651,7 +651,7 @@ "metadata": {}, "outputs": [], "source": [ - "!datalad get -r -J 4 /data/ds000114/derivatives/freesurfer/sub-01" + "!datalad get -r -J 4 -d /data/ds000114 /data/ds000114/derivatives/freesurfer/sub-01" ] }, { diff --git a/notebooks/example_normalize.ipynb b/notebooks/example_normalize.ipynb index 7b5a492..dcf85ca 100644 --- a/notebooks/example_normalize.ipynb +++ b/notebooks/example_normalize.ipynb @@ -27,7 +27,7 @@ "outputs": [], "source": [ "%%bash\n", - "datalad get -J 4 /data/ds000114/derivatives/fmriprep/sub-0[2345789]/anat/*h5" + "datalad get -J 4 -d /data/ds000114 /data/ds000114/derivatives/fmriprep/sub-0[2345789]/anat/*h5" ] }, { diff --git a/notebooks/example_preprocessing.ipynb b/notebooks/example_preprocessing.ipynb index 072aeb7..9be7a98 100644 --- a/notebooks/example_preprocessing.ipynb +++ b/notebooks/example_preprocessing.ipynb @@ -32,8 +32,9 @@ "outputs": [], "source": [ "%%bash\n", - "datalad get -J 4 /data/ds000114/derivatives/fmriprep/sub-*/anat/*preproc.nii.gz \\\n", - " /data/ds000114/sub-*/ses-test/func/*fingerfootlips*" + "datalad get -J 4 -d /data/ds000114 \\\n", + " /data/ds000114/derivatives/fmriprep/sub-*/anat/*preproc.nii.gz \\\n", + " /data/ds000114/sub-*/ses-test/func/*fingerfootlips*" ] }, { diff --git a/notebooks/handson_preprocessing.ipynb b/notebooks/handson_preprocessing.ipynb index c1a5915..5f37b8e 100644 --- a/notebooks/handson_preprocessing.ipynb +++ b/notebooks/handson_preprocessing.ipynb @@ -29,8 +29,9 @@ "outputs": [], "source": [ "%%bash\n", - "datalad get -J 4 /data/ds000114/sub-0[234789]/ses-test/anat/sub-0[234789]_ses-test_T1w.nii.gz \\\n", - " /data/ds000114/sub-0[234789]/ses-test/func/*fingerfootlips*" + "datalad get -J 4 -d /data/ds000114 \\\n", + " /data/ds000114/sub-0[234789]/ses-test/anat/sub-0[234789]_ses-test_T1w.nii.gz \\\n", + " /data/ds000114/sub-0[234789]/ses-test/func/*fingerfootlips*" ] }, { diff --git a/notebooks/introduction_dataset.ipynb b/notebooks/introduction_dataset.ipynb index c951b97..d075c06 100644 --- a/notebooks/introduction_dataset.ipynb +++ b/notebooks/introduction_dataset.ipynb @@ -68,9 +68,9 @@ "source": [ "%%bash\n", "cd /data/ds000114\n", - "datalad get -J 4 /data/ds000114/derivatives/fmriprep/sub-*/anat/*preproc.nii.gz \\\n", - " /data/ds000114/sub-01/ses-test/anat \\\n", - " /data/ds000114/sub-*/ses-test/func/*fingerfootlips*" + "datalad get -J 4 derivatives/fmriprep/sub-*/anat/*preproc.nii.gz \\\n", + " sub-01/ses-test/anat \\\n", + " sub-*/ses-test/func/*fingerfootlips*" ] }, { @@ -121,7 +121,7 @@ "source": [ "%%bash\n", "cd /data/ds000114\n", - "datalad get /data/ds000114/sub-01/ses-test/func/sub-01_ses-test_task-linebisection_events.tsv" + "datalad get sub-01/ses-test/func/sub-01_ses-test_task-linebisection_events.tsv" ] }, { From 4fe35372ee611b2f028bc76d50ab9d2a0d0e17e3 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 27 Jan 2020 15:41:56 -0500 Subject: [PATCH 14/41] MNT: Install niflow-nipype1-workflows --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 35270c8..bd39263 100644 --- a/Dockerfile +++ b/Dockerfile @@ -138,6 +138,7 @@ RUN export PATH="/opt/miniconda-latest/bin:$PATH" \ && pip install --no-cache-dir \ https://github.com/nipy/nipype/tarball/master \ https://github.com/INCF/pybids/tarball/0.7.1 \ + niflow-nipype1-workflows \ nilearn \ datalad[full] \ nipy \ From 6478732bbc6c4c0440b96816101e27bc89b0efdf Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 9 Mar 2020 16:56:30 -0700 Subject: [PATCH 15/41] fix: do not use Circle's premium layer caching --- .circleci/config.yml | 129 +++++++++++++++++++++++++++++++++---------- 1 file changed, 100 insertions(+), 29 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e80375b..c2de5ef 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,61 +1,132 @@ version: 2 jobs: build: - docker: - - image: docker:18.05.0-ce-git + machine: + image: circleci/classic:201808-01 steps: + - restore_cache: + keys: + - build-v1-{{ .Branch }}-{{ epoch }} + - build-v1-{{ .Branch }}- + - build-v1-master- + - build-v1- + paths: + - /tmp/docker + - run: + name: Set-up a Docker registry + command: | + docker run -d -p 5000:5000 --restart=always --name=registry \ + -v /tmp/docker:/var/lib/registry registry:2 + - run: + name: Pull existing images + command: | + set +e + docker pull localhost:5000/neurodebian + success=$? + set -e + if [[ "$success" = "0" ]]; then + echo "Pulling from local registry" + docker tag localhost:5000/neurodebian neurodebian:stretch-non-free + docker pull localhost:5000/nipype_tutorial + docker tag localhost:5000/nipype_tutorial miykael/nipype_tutorial:latest + else + echo "Pulling from Docker Hub" + docker pull neurodebian:stretch-non-free + docker tag neurodebian:stretch-non-free localhost:5000/neurodebian + docker push localhost:5000/neurodebian + fi + - checkout + - run: + name: Build Docker image & push to registry + no_output_timeout: 60m + command: | + docker build --rm --cache-from=miykael/nipype_tutorial:latest \ + -t miykael/nipype_tutorial:latest . + docker tag miykael/nipype_tutorial:latest localhost:5000/nipype_tutorial + docker push localhost:5000/nipype_tutorial + - run: + name: Docker registry garbage collection + command: | + docker exec -it registry /bin/registry garbage-collect --delete-untagged \ + /etc/docker/registry/config.yml + - save_cache: + key: build-v1-{{ .Branch }}-{{ epoch }} + paths: + - /tmp/docker + test_1: - docker: - - image: docker:18.05.0-ce-git + machine: + image: circleci/classic:201808-01 steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true + - restore_cache: + keys: + - build-v1-{{ .Branch }}-{{ epoch }} + - build-v1-{{ .Branch }}- + - build-v1-master- + - build-v1- + paths: + - /tmp/docker - run: - name: docker build 1 - no_output_timeout: 60m + name: Set-up a Docker registry & pull command: | - docker build -t miykael/nipype_tutorial:$(echo $CIRCLE_BRANCH | sed 's|/|_|g') . + docker run -d -p 5000:5000 --restart=always --name=registry \ + -v /tmp/docker:/var/lib/registry registry:2 + docker pull localhost:5000/nipype_tutorial + docker tag localhost:5000/nipype_tutorial miykael/nipype_tutorial:latest - run: name: run tests 1 no_output_timeout: 120m command: | - docker run -it --rm miykael/nipype_tutorial:$(echo $CIRCLE_BRANCH | sed 's|/|_|g') python /home/neuro/nipype_tutorial/test_notebooks.py 1 + docker run -it --rm miykael/nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 1 test_2: - docker: - - image: docker:18.05.0-ce-git + machine: + image: circleci/classic:201808-01 steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true + - restore_cache: + keys: + - build-v1-{{ .Branch }}-{{ epoch }} + - build-v1-{{ .Branch }}- + - build-v1-master- + - build-v1- + paths: + - /tmp/docker - run: - name: docker build 2 - no_output_timeout: 60m + name: Set-up a Docker registry & pull command: | - docker build -t miykael/nipype_tutorial:$(echo $CIRCLE_BRANCH | sed 's|/|_|g') . + docker run -d -p 5000:5000 --restart=always --name=registry \ + -v /tmp/docker:/var/lib/registry registry:2 + docker pull localhost:5000/nipype_tutorial + docker tag localhost:5000/nipype_tutorial miykael/nipype_tutorial:latest - run: name: run tests 2 no_output_timeout: 120m command: | - docker run -it --rm miykael/nipype_tutorial:$(echo $CIRCLE_BRANCH | sed 's|/|_|g') python /home/neuro/nipype_tutorial/test_notebooks.py 2 + docker run -it --rm miykael/nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 2 test_3: - docker: - - image: docker:18.05.0-ce-git + machine: + image: circleci/classic:201808-01 steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true + - restore_cache: + keys: + - build-v1-{{ .Branch }}-{{ epoch }} + - build-v1-{{ .Branch }}- + - build-v1-master- + - build-v1- + paths: + - /tmp/docker - run: - name: docker build 3 - no_output_timeout: 60m + name: Set-up a Docker registry & pull command: | - docker build -t miykael/nipype_tutorial:$(echo $CIRCLE_BRANCH | sed 's|/|_|g') . + docker run -d -p 5000:5000 --restart=always --name=registry \ + -v /tmp/docker:/var/lib/registry registry:2 + docker pull localhost:5000/nipype_tutorial + docker tag localhost:5000/nipype_tutorial miykael/nipype_tutorial:latest - run: name: run tests 3 no_output_timeout: 120m command: | - docker run -it --rm miykael/nipype_tutorial:$(echo $CIRCLE_BRANCH | sed 's|/|_|g') python /home/neuro/nipype_tutorial/test_notebooks.py 3 + docker run -it --rm miykael/nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 3 workflows: version: 2 From 26edbb2914e564aee161a54366e70e2b4fe671d8 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 9 Mar 2020 18:05:39 -0700 Subject: [PATCH 16/41] enh: add a nighly build --- .circleci/config.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index c2de5ef..b08ad56 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -142,3 +142,23 @@ workflows: - test_3: requires: - build + + nightly: + triggers: + - schedule: + cron: "0 10 * * *" + filters: + branches: + only: + - master + jobs: + - build + - test_1: + requires: + - build + - test_2: + requires: + - build + - test_3: + requires: + - build From a7b85b77ad6ca6c05269b767a29ad5ffb5f80584 Mon Sep 17 00:00:00 2001 From: "Andrew J. Worth" Date: Wed, 20 May 2020 07:39:12 -0400 Subject: [PATCH 17/41] One character change! --- notebooks/introduction_docker.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/introduction_docker.ipynb b/notebooks/introduction_docker.ipynb index b4c1343..8dac7af 100644 --- a/notebooks/introduction_docker.ipynb +++ b/notebooks/introduction_docker.ipynb @@ -68,7 +68,7 @@ "\n", " docker run -it --rm -p 8888:8888 miykael/nipype_tutorial jupyter notebook\n", " \n", - "However, if you want to use your version of notebooks, safe notebook outputs locally or use you local data, you can also mount your local directories, e.g.: \n", + "However, if you want to use your version of notebooks, save notebook outputs locally or use you local data, you can also mount your local directories, e.g.: \n", "\n", " docker run -it --rm -v /path/to/nipype_tutorial/:/home/neuro/nipype_tutorial -v /path/to/data/:/data -v /path/to/output/:/output -p 8888:8888 miykael/nipype_tutorial jupyter notebook\n", "\n", From 2d148d0db5965a53faeee8a4458d780bdfb6445b Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 4 Aug 2020 23:55:00 -0400 Subject: [PATCH 18/41] chaniging imports from nipype.workflows to niflow.nipype1.workflows in the notebooks --- notebooks/basic_graph_visualization.ipynb | 6 +++--- notebooks/basic_import_workflows.ipynb | 6 +++--- notebooks/basic_workflow.ipynb | 6 +++--- notebooks/handson_preprocessing.ipynb | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/notebooks/basic_graph_visualization.ipynb b/notebooks/basic_graph_visualization.ipynb index 7b0c89a..6522844 100644 --- a/notebooks/basic_graph_visualization.ipynb +++ b/notebooks/basic_graph_visualization.ipynb @@ -41,7 +41,7 @@ "outputs": [], "source": [ "# Import the function to create an spm fmri preprocessing workflow\n", - "from nipype.workflows.fmri.spm import create_spm_preproc\n", + "from niflow.nipype1.workflows.fmri.spm import create_spm_preproc\n", "\n", "# Create the workflow object\n", "spmflow = create_spm_preproc()" @@ -274,7 +274,7 @@ "metadata": { "anaconda-cloud": {}, "kernelspec": { - "display_name": "Python [default]", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -288,7 +288,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.11" } }, "nbformat": 4, diff --git a/notebooks/basic_import_workflows.ipynb b/notebooks/basic_import_workflows.ipynb index 6b163a5..4151ffc 100644 --- a/notebooks/basic_import_workflows.ipynb +++ b/notebooks/basic_import_workflows.ipynb @@ -40,7 +40,7 @@ "metadata": {}, "outputs": [], "source": [ - "from nipype.workflows.fmri.fsl.preprocess import create_susan_smooth\n", + "from niflow.nipype1.workflows.fmri.fsl.preprocess import create_susan_smooth\n", "smoothwf = create_susan_smooth()" ] }, @@ -323,7 +323,7 @@ "metadata": { "anaconda-cloud": {}, "kernelspec": { - "display_name": "Python [default]", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -337,7 +337,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.11" } }, "nbformat": 4, diff --git a/notebooks/basic_workflow.ipynb b/notebooks/basic_workflow.ipynb index 60f3b93..09bde6f 100644 --- a/notebooks/basic_workflow.ipynb +++ b/notebooks/basic_workflow.ipynb @@ -484,7 +484,7 @@ "metadata": {}, "outputs": [], "source": [ - "from nipype.workflows.fmri.fsl import create_susan_smooth" + "from niflow.nipype1.workflows.fmri.fsl import create_susan_smooth" ] }, { @@ -946,7 +946,7 @@ "metadata": { "anaconda-cloud": {}, "kernelspec": { - "display_name": "Python [default]", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -960,7 +960,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.11" } }, "nbformat": 4, diff --git a/notebooks/handson_preprocessing.ipynb b/notebooks/handson_preprocessing.ipynb index 5f37b8e..444c171 100644 --- a/notebooks/handson_preprocessing.ipynb +++ b/notebooks/handson_preprocessing.ipynb @@ -765,7 +765,7 @@ "metadata": {}, "outputs": [], "source": [ - "from nipype.workflows.fmri.fsl.preprocess import create_susan_smooth" + "from niflow.nipype1.workflows.fmri.fsl.preprocess import create_susan_smooth" ] }, { @@ -1753,7 +1753,7 @@ "metadata": { "anaconda-cloud": {}, "kernelspec": { - "display_name": "Python [default]", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -1767,7 +1767,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.11" } }, "nbformat": 4, From 3ce8c10af318e4eac1a5c58694dda1976ec877ad Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 5 Aug 2020 00:03:02 -0400 Subject: [PATCH 19/41] updating python and adding niflow.nipype1 to the requirements --- Dockerfile | 63 ++++++++++++++++++++++++++------------------------ Singularity | 66 ++++++++++++++++++++++++++++++----------------------- generate.sh | 12 +++++----- 3 files changed, 76 insertions(+), 65 deletions(-) diff --git a/Dockerfile b/Dockerfile index bd39263..920d774 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ -# Generated by Neurodocker version 0.5.0 -# Timestamp: 2019-07-14 08:54:07 UTC +# Your version: 0.6.0+5.g74cb187.dirty Latest version: 0.7.0 +# Generated by Neurodocker version 0.6.0+5.g74cb187.dirty +# Timestamp: 2020-08-05 04:01:42 UTC # # Thank you for using Neurodocker. If you discover any issues # or ways to improve this software, please submit an issue or @@ -9,6 +10,8 @@ FROM neurodebian:stretch-non-free +USER root + ARG DEBIAN_FRONTEND="noninteractive" ENV LANG="en_US.UTF-8" \ @@ -91,12 +94,12 @@ RUN export TMPDIR="$(mktemp -d)" \ && rm -rf "$TMPDIR" \ && unset TMPDIR \ && echo "Downloading standalone SPM ..." \ - && curl -fsSL --retry 5 -o /tmp/spm12.zip http://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/previous/spm12_r7219_R2010a.zip \ + && curl -fsSL --retry 5 -o /tmp/spm12.zip https://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/previous/spm12_r7219_R2010a.zip \ && unzip -q /tmp/spm12.zip -d /tmp \ && mkdir -p /opt/spm12-r7219 \ && mv /tmp/spm12/* /opt/spm12-r7219/ \ && chmod -R 777 /opt/spm12-r7219 \ - && rm -rf /tmp/* \ + && rm -rf /tmp/spm* \ && /opt/spm12-r7219/run_spm12.sh /opt/matlabmcr-2010a/v713 quit \ && sed -i '$iexport SPMMCRCMD=\"/opt/spm12-r7219/run_spm12.sh /opt/matlabmcr-2010a/v713 script\"' $ND_ENTRYPOINT @@ -120,35 +123,35 @@ RUN export PATH="/opt/miniconda-latest/bin:$PATH" \ && sync && conda clean --all && sync \ && conda create -y -q --name neuro \ && conda install -y -q --name neuro \ - 'python=3.6' \ - 'pytest' \ - 'jupyter' \ - 'jupyterlab' \ - 'jupyter_contrib_nbextensions' \ - 'traits' \ - 'pandas' \ - 'matplotlib' \ - 'scikit-learn' \ - 'scikit-image' \ - 'seaborn' \ - 'nbformat' \ - 'nb_conda' \ + "python=3.7" \ + "pytest" \ + "jupyter" \ + "jupyterlab" \ + "jupyter_contrib_nbextensions" \ + "traits" \ + "pandas" \ + "matplotlib" \ + "scikit-learn" \ + "scikit-image" \ + "seaborn" \ + "nbformat" \ + "nb_conda" \ && sync && conda clean --all && sync \ && bash -c "source activate neuro \ && pip install --no-cache-dir \ - https://github.com/nipy/nipype/tarball/master \ - https://github.com/INCF/pybids/tarball/0.7.1 \ - niflow-nipype1-workflows \ - nilearn \ - datalad[full] \ - nipy \ - duecredit \ - nbval" \ + "https://github.com/nipy/nipype/tarball/master" \ + "https://github.com/INCF/pybids/tarball/0.7.1" \ + "nilearn" \ + "datalad[full]" \ + "nipy" \ + "duecredit" \ + "nbval" \ + "niflow-nipype1-workflows"" \ && rm -rf ~/.cache/pip/* \ && sync \ && sed -i '$isource activate neuro' $ND_ENTRYPOINT -ENV LD_LIBRARY_PATH="/opt/miniconda-latest/envs/neuro:/usr/lib/fsl/5.0" +ENV LD_LIBRARY_PATH="/opt/miniconda-latest/envs/neuro:" RUN bash -c 'source activate neuro && jupyter nbextension enable exercise2/main && jupyter nbextension enable spellchecker/main' @@ -232,9 +235,8 @@ RUN echo '{ \ \n [ \ \n "miniconda", \ \n { \ - \n "miniconda_version": "4.3.31", \ \n "conda_install": [ \ - \n "python=3.6", \ + \n "python=3.7", \ \n "pytest", \ \n "jupyter", \ \n "jupyterlab", \ @@ -255,7 +257,8 @@ RUN echo '{ \ \n "datalad[full]", \ \n "nipy", \ \n "duecredit", \ - \n "nbval" \ + \n "nbval", \ + \n "niflow-nipype1-workflows" \ \n ], \ \n "create_env": "neuro", \ \n "activate": true \ @@ -264,7 +267,7 @@ RUN echo '{ \ \n [ \ \n "env", \ \n { \ - \n "LD_LIBRARY_PATH": "/opt/miniconda-latest/envs/neuro:/usr/lib/fsl/5.0" \ + \n "LD_LIBRARY_PATH": "/opt/miniconda-latest/envs/neuro:" \ \n } \ \n ], \ \n [ \ diff --git a/Singularity b/Singularity index bce0d3c..fcde735 100644 --- a/Singularity +++ b/Singularity @@ -1,5 +1,6 @@ -# Generated by Neurodocker version 0.5.0 -# Timestamp: 2019-07-14 08:54:09 UTC +# Your version: 0.6.0+5.g74cb187.dirty Latest version: 0.7.0 +# Generated by Neurodocker version 0.6.0+5.g74cb187.dirty +# Timestamp: 2020-08-05 04:01:43 UTC # # Thank you for using Neurodocker. If you discover any issues # or ways to improve this software, please submit an issue or @@ -11,6 +12,8 @@ Bootstrap: docker From: neurodebian:stretch-non-free %post +su - root + export ND_ENTRYPOINT="/neurodocker/startup.sh" apt-get update -qq apt-get install -y -q --no-install-recommends \ @@ -83,12 +86,12 @@ chmod +x "$TMPDIR/MCRInstaller.bin" rm -rf "$TMPDIR" unset TMPDIR echo "Downloading standalone SPM ..." -curl -fsSL --retry 5 -o /tmp/spm12.zip http://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/previous/spm12_r7219_R2010a.zip +curl -fsSL --retry 5 -o /tmp/spm12.zip https://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/previous/spm12_r7219_R2010a.zip unzip -q /tmp/spm12.zip -d /tmp mkdir -p /opt/spm12-r7219 mv /tmp/spm12/* /opt/spm12-r7219/ chmod -R 777 /opt/spm12-r7219 -rm -rf /tmp/* +rm -rf /tmp/spm* /opt/spm12-r7219/run_spm12.sh /opt/matlabmcr-2010a/v713 quit sed -i '$iexport SPMMCRCMD=\"/opt/spm12-r7219/run_spm12.sh /opt/matlabmcr-2010a/v713 script\"' $ND_ENTRYPOINT @@ -110,29 +113,30 @@ conda config --system --set show_channel_urls true sync && conda clean --all && sync conda create -y -q --name neuro conda install -y -q --name neuro \ - 'python=3.6' \ - 'pytest' \ - 'jupyter' \ - 'jupyterlab' \ - 'jupyter_contrib_nbextensions' \ - 'traits' \ - 'pandas' \ - 'matplotlib' \ - 'scikit-learn' \ - 'scikit-image' \ - 'seaborn' \ - 'nbformat' \ - 'nb_conda' + "python=3.7" \ + "pytest" \ + "jupyter" \ + "jupyterlab" \ + "jupyter_contrib_nbextensions" \ + "traits" \ + "pandas" \ + "matplotlib" \ + "scikit-learn" \ + "scikit-image" \ + "seaborn" \ + "nbformat" \ + "nb_conda" sync && conda clean --all && sync bash -c "source activate neuro pip install --no-cache-dir \ - https://github.com/nipy/nipype/tarball/master \ - https://github.com/INCF/pybids/tarball/0.7.1 \ - nilearn \ - datalad[full] \ - nipy \ - duecredit \ - nbval" + "https://github.com/nipy/nipype/tarball/master" \ + "https://github.com/INCF/pybids/tarball/0.7.1" \ + "nilearn" \ + "datalad[full]" \ + "nipy" \ + "duecredit" \ + "nbval" \ + "niflow-nipype1-workflows"" rm -rf ~/.cache/pip/* sync sed -i '$isource activate neuro' $ND_ENTRYPOINT @@ -174,6 +178,10 @@ echo '{ \n "neurodebian:stretch-non-free" \n ], \n [ +\n "user", +\n "root" +\n ], +\n [ \n "_header", \n { \n "version": "generic", @@ -223,9 +231,8 @@ echo '{ \n [ \n "miniconda", \n { -\n "miniconda_version": "4.3.31", \n "conda_install": [ -\n "python=3.6", +\n "python=3.7", \n "pytest", \n "jupyter", \n "jupyterlab", @@ -246,7 +253,8 @@ echo '{ \n "datalad[full]", \n "nipy", \n "duecredit", -\n "nbval" +\n "nbval", +\n "niflow-nipype1-workflows" \n ], \n "create_env": "neuro", \n "activate": true @@ -255,7 +263,7 @@ echo '{ \n [ \n "env", \n { -\n "LD_LIBRARY_PATH": "/opt/miniconda-latest/envs/neuro:/usr/lib/fsl/5.0" +\n "LD_LIBRARY_PATH": "/opt/miniconda-latest/envs/neuro:" \n } \n ], \n [ @@ -333,7 +341,7 @@ export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu:/opt/matlabmc export MATLABCMD="/opt/matlabmcr-2010a/v713/toolbox/matlab" export CONDA_DIR="/opt/miniconda-latest" export PATH="/opt/miniconda-latest/bin:$PATH" -export LD_LIBRARY_PATH="/opt/miniconda-latest/envs/neuro:/usr/lib/fsl/5.0" +export LD_LIBRARY_PATH="/opt/miniconda-latest/envs/neuro:" %files . /home/neuro/nipype_tutorial diff --git a/generate.sh b/generate.sh index a8a3e94..a22ace7 100644 --- a/generate.sh +++ b/generate.sh @@ -14,12 +14,12 @@ generate_docker() { --spm12 version=r7219 \ --user=neuro \ --workdir /home/neuro \ - --miniconda miniconda_version="4.3.31" \ - conda_install="python=3.6 pytest jupyter jupyterlab jupyter_contrib_nbextensions + --miniconda \ + conda_install="python=3.7 pytest jupyter jupyterlab jupyter_contrib_nbextensions traits pandas matplotlib scikit-learn scikit-image seaborn nbformat nb_conda" \ pip_install="https://github.com/nipy/nipype/tarball/master%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20https://github.com/INCF/pybids/tarball/0.7.1-%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20nilearn%20datalad[full]%20nipy%20duecredit%20nbval" \ + nilearn datalad[full] nipy duecredit nbval niflow-nipype1-workflows" \ create_env="neuro" \ activate=True \ --env LD_LIBRARY_PATH="/opt/miniconda-latest/envs/neuro:$LD_LIBRARY_PATH" \ @@ -53,12 +53,12 @@ generate_singularity() { --spm12 version=r7219 \ --user=neuro \ --workdir /home/neuro \ - --miniconda miniconda_version="4.3.31" \ - conda_install="python=3.6 pytest jupyter jupyterlab jupyter_contrib_nbextensions + --miniconda \ + conda_install="python=3.7 pytest jupyter jupyterlab jupyter_contrib_nbextensions traits pandas matplotlib scikit-learn scikit-image seaborn nbformat nb_conda" \ pip_install="https://github.com/nipy/nipype/tarball/master%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20https://github.com/INCF/pybids/tarball/0.7.1-%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20nilearn%20datalad[full]%20nipy%20duecredit%20nbval" \ + nilearn datalad[full] nipy duecredit nbval niflow-nipype1-workflows" \ create_env="neuro" \ activate=True \ --env LD_LIBRARY_PATH="/opt/miniconda-latest/envs/neuro:$LD_LIBRARY_PATH" \ From 95944ed48b7b4d6587c29e5c256fda8e9b79aa5f Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 7 Aug 2020 02:14:07 -0400 Subject: [PATCH 20/41] adding testing using github actions --- .github/workflows/testing.yml | 50 +++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/testing.yml diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 0000000..d44161e --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,50 @@ +name: CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + # In this step, this action saves a list of existing images, + # the cache is created without them in the post run. + # It also restores the cache if it exists. + - uses: satackey/action-docker-layer-caching@v0.0.5 + - name: build the image + run: docker build . --file Dockerfile -t nipype_tutorial:latest + + test_1: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: satackey/action-docker-layer-caching@v0.0.5 + - name: run test 1 + run: docker run -it --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 1 + + test_2: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: satackey/action-docker-layer-caching@v0.0.5 + - name: run test 2 + run: docker run -it --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 2 + + test_3: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: satackey/action-docker-layer-caching@v0.0.5 + - name: run test 3 + run: docker run -it --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 3 + From 0df59672a674fc17008a1a0812b88a20c545c735 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 7 Aug 2020 11:26:31 -0400 Subject: [PATCH 21/41] Update .github/workflows/testing.yml Co-authored-by: Mathias Goncalves --- .github/workflows/testing.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d44161e..3ec3a56 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@v2 - uses: satackey/action-docker-layer-caching@v0.0.5 - name: run test 1 - run: docker run -it --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 1 + run: docker run --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 1 test_2: needs: build @@ -47,4 +47,3 @@ jobs: - uses: satackey/action-docker-layer-caching@v0.0.5 - name: run test 3 run: docker run -it --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 3 - From c1f0e284bdf10e5770fee1021ca6cd05b3f30d5f Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 7 Aug 2020 11:26:39 -0400 Subject: [PATCH 22/41] Update .github/workflows/testing.yml Co-authored-by: Mathias Goncalves --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 3ec3a56..91a272d 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -46,4 +46,4 @@ jobs: - uses: actions/checkout@v2 - uses: satackey/action-docker-layer-caching@v0.0.5 - name: run test 3 - run: docker run -it --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 3 + run: docker run --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 3 From dff13040ea9cc3ab9da13753f7c83e3b26eb4c89 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 7 Aug 2020 11:27:01 -0400 Subject: [PATCH 23/41] Update .github/workflows/testing.yml Co-authored-by: Mathias Goncalves --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 91a272d..b6fe46b 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -37,7 +37,7 @@ jobs: - uses: actions/checkout@v2 - uses: satackey/action-docker-layer-caching@v0.0.5 - name: run test 2 - run: docker run -it --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 2 + run: docker run --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 2 test_3: needs: build From 197aa73c22f7e6c29e385184e065a91c2759a107 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 7 Aug 2020 11:54:00 -0400 Subject: [PATCH 24/41] adding dockerfile generation to the ga --- .github/workflows/testing.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index b6fe46b..4a04daf 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -13,7 +13,8 @@ jobs: steps: - uses: actions/checkout@v2 - + - name: generate the Dockerfile from generate.sh + run: bash generate.sh # In this step, this action saves a list of existing images, # the cache is created without them in the post run. # It also restores the cache if it exists. From 90a7de7e869e05d87b83908759b1635cd4ded94b Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 7 Aug 2020 12:01:33 -0400 Subject: [PATCH 25/41] removing Dockerfile/Singularity - should be generated by generate.sh in the GA testing workflow --- Dockerfile | 343 -------------------------------------------------- Singularity | 350 ---------------------------------------------------- 2 files changed, 693 deletions(-) delete mode 100644 Dockerfile delete mode 100644 Singularity diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 920d774..0000000 --- a/Dockerfile +++ /dev/null @@ -1,343 +0,0 @@ -# Your version: 0.6.0+5.g74cb187.dirty Latest version: 0.7.0 -# Generated by Neurodocker version 0.6.0+5.g74cb187.dirty -# Timestamp: 2020-08-05 04:01:42 UTC -# -# Thank you for using Neurodocker. If you discover any issues -# or ways to improve this software, please submit an issue or -# pull request on our GitHub repository: -# -# https://github.com/kaczmarj/neurodocker - -FROM neurodebian:stretch-non-free - -USER root - -ARG DEBIAN_FRONTEND="noninteractive" - -ENV LANG="en_US.UTF-8" \ - LC_ALL="en_US.UTF-8" \ - ND_ENTRYPOINT="/neurodocker/startup.sh" -RUN export ND_ENTRYPOINT="/neurodocker/startup.sh" \ - && apt-get update -qq \ - && apt-get install -y -q --no-install-recommends \ - apt-utils \ - bzip2 \ - ca-certificates \ - curl \ - locales \ - unzip \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ - && dpkg-reconfigure --frontend=noninteractive locales \ - && update-locale LANG="en_US.UTF-8" \ - && chmod 777 /opt && chmod a+s /opt \ - && mkdir -p /neurodocker \ - && if [ ! -f "$ND_ENTRYPOINT" ]; then \ - echo '#!/usr/bin/env bash' >> "$ND_ENTRYPOINT" \ - && echo 'set -e' >> "$ND_ENTRYPOINT" \ - && echo 'export USER="${USER:=`whoami`}"' >> "$ND_ENTRYPOINT" \ - && echo 'if [ -n "$1" ]; then "$@"; else /usr/bin/env bash; fi' >> "$ND_ENTRYPOINT"; \ - fi \ - && chmod -R 777 /neurodocker && chmod a+s /neurodocker - -ENTRYPOINT ["/neurodocker/startup.sh"] - -RUN apt-get update -qq \ - && apt-get install -y -q --no-install-recommends \ - convert3d \ - ants \ - fsl \ - gcc \ - g++ \ - graphviz \ - tree \ - git-annex-standalone \ - vim \ - emacs-nox \ - nano \ - less \ - ncdu \ - tig \ - git-annex-remote-rclone \ - octave \ - netbase \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -RUN sed -i '$isource /etc/fsl/fsl.sh' $ND_ENTRYPOINT - -ENV FORCE_SPMMCR="1" \ - LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu:/opt/matlabmcr-2010a/v713/runtime/glnxa64:/opt/matlabmcr-2010a/v713/bin/glnxa64:/opt/matlabmcr-2010a/v713/sys/os/glnxa64:/opt/matlabmcr-2010a/v713/extern/bin/glnxa64" \ - MATLABCMD="/opt/matlabmcr-2010a/v713/toolbox/matlab" -RUN export TMPDIR="$(mktemp -d)" \ - && apt-get update -qq \ - && apt-get install -y -q --no-install-recommends \ - bc \ - libncurses5 \ - libxext6 \ - libxmu6 \ - libxpm-dev \ - libxt6 \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && echo "Downloading MATLAB Compiler Runtime ..." \ - && curl -sSL --retry 5 -o /tmp/toinstall.deb http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb \ - && dpkg -i /tmp/toinstall.deb \ - && rm /tmp/toinstall.deb \ - && apt-get install -f \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && curl -fsSL --retry 5 -o "$TMPDIR/MCRInstaller.bin" https://dl.dropbox.com/s/zz6me0c3v4yq5fd/MCR_R2010a_glnxa64_installer.bin \ - && chmod +x "$TMPDIR/MCRInstaller.bin" \ - && "$TMPDIR/MCRInstaller.bin" -silent -P installLocation="/opt/matlabmcr-2010a" \ - && rm -rf "$TMPDIR" \ - && unset TMPDIR \ - && echo "Downloading standalone SPM ..." \ - && curl -fsSL --retry 5 -o /tmp/spm12.zip https://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/previous/spm12_r7219_R2010a.zip \ - && unzip -q /tmp/spm12.zip -d /tmp \ - && mkdir -p /opt/spm12-r7219 \ - && mv /tmp/spm12/* /opt/spm12-r7219/ \ - && chmod -R 777 /opt/spm12-r7219 \ - && rm -rf /tmp/spm* \ - && /opt/spm12-r7219/run_spm12.sh /opt/matlabmcr-2010a/v713 quit \ - && sed -i '$iexport SPMMCRCMD=\"/opt/spm12-r7219/run_spm12.sh /opt/matlabmcr-2010a/v713 script\"' $ND_ENTRYPOINT - -RUN test "$(getent passwd neuro)" || useradd --no-user-group --create-home --shell /bin/bash neuro -USER neuro - -WORKDIR /home/neuro - -ENV CONDA_DIR="/opt/miniconda-latest" \ - PATH="/opt/miniconda-latest/bin:$PATH" -RUN export PATH="/opt/miniconda-latest/bin:$PATH" \ - && echo "Downloading Miniconda installer ..." \ - && conda_installer="/tmp/miniconda.sh" \ - && curl -fsSL --retry 5 -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - && bash "$conda_installer" -b -p /opt/miniconda-latest \ - && rm -f "$conda_installer" \ - && conda update -yq -nbase conda \ - && conda config --system --prepend channels conda-forge \ - && conda config --system --set auto_update_conda false \ - && conda config --system --set show_channel_urls true \ - && sync && conda clean --all && sync \ - && conda create -y -q --name neuro \ - && conda install -y -q --name neuro \ - "python=3.7" \ - "pytest" \ - "jupyter" \ - "jupyterlab" \ - "jupyter_contrib_nbextensions" \ - "traits" \ - "pandas" \ - "matplotlib" \ - "scikit-learn" \ - "scikit-image" \ - "seaborn" \ - "nbformat" \ - "nb_conda" \ - && sync && conda clean --all && sync \ - && bash -c "source activate neuro \ - && pip install --no-cache-dir \ - "https://github.com/nipy/nipype/tarball/master" \ - "https://github.com/INCF/pybids/tarball/0.7.1" \ - "nilearn" \ - "datalad[full]" \ - "nipy" \ - "duecredit" \ - "nbval" \ - "niflow-nipype1-workflows"" \ - && rm -rf ~/.cache/pip/* \ - && sync \ - && sed -i '$isource activate neuro' $ND_ENTRYPOINT - -ENV LD_LIBRARY_PATH="/opt/miniconda-latest/envs/neuro:" - -RUN bash -c 'source activate neuro && jupyter nbextension enable exercise2/main && jupyter nbextension enable spellchecker/main' - -USER root - -RUN mkdir /data && chmod 777 /data && chmod a+s /data - -RUN mkdir /output && chmod 777 /output && chmod a+s /output - -USER neuro - -RUN printf "[user]\n\tname = miykael\n\temail = michaelnotter@hotmail.com\n" > ~/.gitconfig - -RUN bash -c 'source activate neuro && cd /data && datalad install -r ///workshops/nih-2017/ds000114 && cd ds000114 && datalad update -r && datalad get -r sub-01/ses-test/anat sub-01/ses-test/func/*fingerfootlips*' - -RUN curl -L https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/580705089ad5a101f17944a9 -o /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz && tar xf /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz -C /data/ds000114/derivatives/fmriprep/. && rm /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz && find /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c -type f -not -name ?mm_T1.nii.gz -not -name ?mm_brainmask.nii.gz -not -name ?mm_tpm*.nii.gz -delete - -COPY [".", "/home/neuro/nipype_tutorial"] - -USER root - -RUN chown -R neuro /home/neuro/nipype_tutorial - -RUN rm -rf /opt/conda/pkgs/* - -USER neuro - -RUN mkdir -p ~/.jupyter && echo c.NotebookApp.ip = \"0.0.0.0\" > ~/.jupyter/jupyter_notebook_config.py - -WORKDIR /home/neuro/nipype_tutorial - -CMD ["jupyter-notebook"] - -RUN echo '{ \ - \n "pkg_manager": "apt", \ - \n "instructions": [ \ - \n [ \ - \n "base", \ - \n "neurodebian:stretch-non-free" \ - \n ], \ - \n [ \ - \n "install", \ - \n [ \ - \n "convert3d", \ - \n "ants", \ - \n "fsl", \ - \n "gcc", \ - \n "g++", \ - \n "graphviz", \ - \n "tree", \ - \n "git-annex-standalone", \ - \n "vim", \ - \n "emacs-nox", \ - \n "nano", \ - \n "less", \ - \n "ncdu", \ - \n "tig", \ - \n "git-annex-remote-rclone", \ - \n "octave", \ - \n "netbase" \ - \n ] \ - \n ], \ - \n [ \ - \n "add_to_entrypoint", \ - \n "source /etc/fsl/fsl.sh" \ - \n ], \ - \n [ \ - \n "spm12", \ - \n { \ - \n "version": "r7219" \ - \n } \ - \n ], \ - \n [ \ - \n "user", \ - \n "neuro" \ - \n ], \ - \n [ \ - \n "workdir", \ - \n "/home/neuro" \ - \n ], \ - \n [ \ - \n "miniconda", \ - \n { \ - \n "conda_install": [ \ - \n "python=3.7", \ - \n "pytest", \ - \n "jupyter", \ - \n "jupyterlab", \ - \n "jupyter_contrib_nbextensions", \ - \n "traits", \ - \n "pandas", \ - \n "matplotlib", \ - \n "scikit-learn", \ - \n "scikit-image", \ - \n "seaborn", \ - \n "nbformat", \ - \n "nb_conda" \ - \n ], \ - \n "pip_install": [ \ - \n "https://github.com/nipy/nipype/tarball/master", \ - \n "https://github.com/INCF/pybids/tarball/0.7.1", \ - \n "nilearn", \ - \n "datalad[full]", \ - \n "nipy", \ - \n "duecredit", \ - \n "nbval", \ - \n "niflow-nipype1-workflows" \ - \n ], \ - \n "create_env": "neuro", \ - \n "activate": true \ - \n } \ - \n ], \ - \n [ \ - \n "env", \ - \n { \ - \n "LD_LIBRARY_PATH": "/opt/miniconda-latest/envs/neuro:" \ - \n } \ - \n ], \ - \n [ \ - \n "run_bash", \ - \n "source activate neuro && jupyter nbextension enable exercise2/main && jupyter nbextension enable spellchecker/main" \ - \n ], \ - \n [ \ - \n "user", \ - \n "root" \ - \n ], \ - \n [ \ - \n "run", \ - \n "mkdir /data && chmod 777 /data && chmod a+s /data" \ - \n ], \ - \n [ \ - \n "run", \ - \n "mkdir /output && chmod 777 /output && chmod a+s /output" \ - \n ], \ - \n [ \ - \n "user", \ - \n "neuro" \ - \n ], \ - \n [ \ - \n "run", \ - \n "printf \"[user]\\\n\\tname = miykael\\\n\\temail = michaelnotter@hotmail.com\\\n\" > ~/.gitconfig" \ - \n ], \ - \n [ \ - \n "run_bash", \ - \n "source activate neuro && cd /data && datalad install -r ///workshops/nih-2017/ds000114 && cd ds000114 && datalad update -r && datalad get -r sub-01/ses-test/anat sub-01/ses-test/func/*fingerfootlips*" \ - \n ], \ - \n [ \ - \n "run", \ - \n "curl -L https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/580705089ad5a101f17944a9 -o /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz && tar xf /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz -C /data/ds000114/derivatives/fmriprep/. && rm /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz && find /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c -type f -not -name ?mm_T1.nii.gz -not -name ?mm_brainmask.nii.gz -not -name ?mm_tpm*.nii.gz -delete" \ - \n ], \ - \n [ \ - \n "copy", \ - \n [ \ - \n ".", \ - \n "/home/neuro/nipype_tutorial" \ - \n ] \ - \n ], \ - \n [ \ - \n "user", \ - \n "root" \ - \n ], \ - \n [ \ - \n "run", \ - \n "chown -R neuro /home/neuro/nipype_tutorial" \ - \n ], \ - \n [ \ - \n "run", \ - \n "rm -rf /opt/conda/pkgs/*" \ - \n ], \ - \n [ \ - \n "user", \ - \n "neuro" \ - \n ], \ - \n [ \ - \n "run", \ - \n "mkdir -p ~/.jupyter && echo c.NotebookApp.ip = \\\"0.0.0.0\\\" > ~/.jupyter/jupyter_notebook_config.py" \ - \n ], \ - \n [ \ - \n "workdir", \ - \n "/home/neuro/nipype_tutorial" \ - \n ], \ - \n [ \ - \n "cmd", \ - \n [ \ - \n "jupyter-notebook" \ - \n ] \ - \n ] \ - \n ] \ - \n}' > /neurodocker/neurodocker_specs.json diff --git a/Singularity b/Singularity deleted file mode 100644 index fcde735..0000000 --- a/Singularity +++ /dev/null @@ -1,350 +0,0 @@ -# Your version: 0.6.0+5.g74cb187.dirty Latest version: 0.7.0 -# Generated by Neurodocker version 0.6.0+5.g74cb187.dirty -# Timestamp: 2020-08-05 04:01:43 UTC -# -# Thank you for using Neurodocker. If you discover any issues -# or ways to improve this software, please submit an issue or -# pull request on our GitHub repository: -# -# https://github.com/kaczmarj/neurodocker - -Bootstrap: docker -From: neurodebian:stretch-non-free - -%post -su - root - -export ND_ENTRYPOINT="/neurodocker/startup.sh" -apt-get update -qq -apt-get install -y -q --no-install-recommends \ - apt-utils \ - bzip2 \ - ca-certificates \ - curl \ - locales \ - unzip -apt-get clean -rm -rf /var/lib/apt/lists/* -sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen -dpkg-reconfigure --frontend=noninteractive locales -update-locale LANG="en_US.UTF-8" -chmod 777 /opt && chmod a+s /opt -mkdir -p /neurodocker -if [ ! -f "$ND_ENTRYPOINT" ]; then - echo '#!/usr/bin/env bash' >> "$ND_ENTRYPOINT" - echo 'set -e' >> "$ND_ENTRYPOINT" - echo 'export USER="${USER:=`whoami`}"' >> "$ND_ENTRYPOINT" - echo 'if [ -n "$1" ]; then "$@"; else /usr/bin/env bash; fi' >> "$ND_ENTRYPOINT"; -fi -chmod -R 777 /neurodocker && chmod a+s /neurodocker - -apt-get update -qq -apt-get install -y -q --no-install-recommends \ - convert3d \ - ants \ - fsl \ - gcc \ - g++ \ - graphviz \ - tree \ - git-annex-standalone \ - vim \ - emacs-nox \ - nano \ - less \ - ncdu \ - tig \ - git-annex-remote-rclone \ - octave \ - netbase -apt-get clean -rm -rf /var/lib/apt/lists/* - -sed -i '$isource /etc/fsl/fsl.sh' $ND_ENTRYPOINT - -export TMPDIR="$(mktemp -d)" -apt-get update -qq -apt-get install -y -q --no-install-recommends \ - bc \ - libncurses5 \ - libxext6 \ - libxmu6 \ - libxpm-dev \ - libxt6 -apt-get clean -rm -rf /var/lib/apt/lists/* -echo "Downloading MATLAB Compiler Runtime ..." -curl -sSL --retry 5 -o /tmp/toinstall.deb http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb -dpkg -i /tmp/toinstall.deb -rm /tmp/toinstall.deb -apt-get install -f -apt-get clean -rm -rf /var/lib/apt/lists/* -curl -fsSL --retry 5 -o "$TMPDIR/MCRInstaller.bin" https://dl.dropbox.com/s/zz6me0c3v4yq5fd/MCR_R2010a_glnxa64_installer.bin -chmod +x "$TMPDIR/MCRInstaller.bin" -"$TMPDIR/MCRInstaller.bin" -silent -P installLocation="/opt/matlabmcr-2010a" -rm -rf "$TMPDIR" -unset TMPDIR -echo "Downloading standalone SPM ..." -curl -fsSL --retry 5 -o /tmp/spm12.zip https://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/previous/spm12_r7219_R2010a.zip -unzip -q /tmp/spm12.zip -d /tmp -mkdir -p /opt/spm12-r7219 -mv /tmp/spm12/* /opt/spm12-r7219/ -chmod -R 777 /opt/spm12-r7219 -rm -rf /tmp/spm* -/opt/spm12-r7219/run_spm12.sh /opt/matlabmcr-2010a/v713 quit -sed -i '$iexport SPMMCRCMD=\"/opt/spm12-r7219/run_spm12.sh /opt/matlabmcr-2010a/v713 script\"' $ND_ENTRYPOINT - -test "$(getent passwd neuro)" || useradd --no-user-group --create-home --shell /bin/bash neuro -su - neuro - -cd /home/neuro - -export PATH="/opt/miniconda-latest/bin:$PATH" -echo "Downloading Miniconda installer ..." -conda_installer="/tmp/miniconda.sh" -curl -fsSL --retry 5 -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -bash "$conda_installer" -b -p /opt/miniconda-latest -rm -f "$conda_installer" -conda update -yq -nbase conda -conda config --system --prepend channels conda-forge -conda config --system --set auto_update_conda false -conda config --system --set show_channel_urls true -sync && conda clean --all && sync -conda create -y -q --name neuro -conda install -y -q --name neuro \ - "python=3.7" \ - "pytest" \ - "jupyter" \ - "jupyterlab" \ - "jupyter_contrib_nbextensions" \ - "traits" \ - "pandas" \ - "matplotlib" \ - "scikit-learn" \ - "scikit-image" \ - "seaborn" \ - "nbformat" \ - "nb_conda" -sync && conda clean --all && sync -bash -c "source activate neuro - pip install --no-cache-dir \ - "https://github.com/nipy/nipype/tarball/master" \ - "https://github.com/INCF/pybids/tarball/0.7.1" \ - "nilearn" \ - "datalad[full]" \ - "nipy" \ - "duecredit" \ - "nbval" \ - "niflow-nipype1-workflows"" -rm -rf ~/.cache/pip/* -sync -sed -i '$isource activate neuro' $ND_ENTRYPOINT - - -bash -c 'source activate neuro && jupyter nbextension enable exercise2/main && jupyter nbextension enable spellchecker/main' - -su - root - -mkdir /data && chmod 777 /data && chmod a+s /data - -mkdir /output && chmod 777 /output && chmod a+s /output - -su - neuro - -printf "[user]\n\tname = miykael\n\temail = michaelnotter@hotmail.com\n" > ~/.gitconfig - -bash -c 'source activate neuro && cd /data && datalad install -r ///workshops/nih-2017/ds000114 && cd ds000114 && datalad update -r && datalad get -r sub-01/ses-test/anat sub-01/ses-test/func/*fingerfootlips*' - -curl -L https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/580705089ad5a101f17944a9 -o /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz && tar xf /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz -C /data/ds000114/derivatives/fmriprep/. && rm /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz && find /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c -type f -not -name ?mm_T1.nii.gz -not -name ?mm_brainmask.nii.gz -not -name ?mm_tpm*.nii.gz -delete - -su - root - -chown -R neuro /home/neuro/nipype_tutorial - -rm -rf /opt/conda/pkgs/* - -su - neuro - -mkdir -p ~/.jupyter && echo c.NotebookApp.ip = \"0.0.0.0\" > ~/.jupyter/jupyter_notebook_config.py - -cd /home/neuro/nipype_tutorial - -echo '{ -\n "pkg_manager": "apt", -\n "instructions": [ -\n [ -\n "base", -\n "neurodebian:stretch-non-free" -\n ], -\n [ -\n "user", -\n "root" -\n ], -\n [ -\n "_header", -\n { -\n "version": "generic", -\n "method": "custom" -\n } -\n ], -\n [ -\n "install", -\n [ -\n "convert3d", -\n "ants", -\n "fsl", -\n "gcc", -\n "g++", -\n "graphviz", -\n "tree", -\n "git-annex-standalone", -\n "vim", -\n "emacs-nox", -\n "nano", -\n "less", -\n "ncdu", -\n "tig", -\n "git-annex-remote-rclone", -\n "octave", -\n "netbase" -\n ] -\n ], -\n [ -\n "add_to_entrypoint", -\n "source /etc/fsl/fsl.sh" -\n ], -\n [ -\n "spm12", -\n { -\n "version": "r7219" -\n } -\n ], -\n [ -\n "user", -\n "neuro" -\n ], -\n [ -\n "workdir", -\n "/home/neuro" -\n ], -\n [ -\n "miniconda", -\n { -\n "conda_install": [ -\n "python=3.7", -\n "pytest", -\n "jupyter", -\n "jupyterlab", -\n "jupyter_contrib_nbextensions", -\n "traits", -\n "pandas", -\n "matplotlib", -\n "scikit-learn", -\n "scikit-image", -\n "seaborn", -\n "nbformat", -\n "nb_conda" -\n ], -\n "pip_install": [ -\n "https://github.com/nipy/nipype/tarball/master", -\n "https://github.com/INCF/pybids/tarball/0.7.1", -\n "nilearn", -\n "datalad[full]", -\n "nipy", -\n "duecredit", -\n "nbval", -\n "niflow-nipype1-workflows" -\n ], -\n "create_env": "neuro", -\n "activate": true -\n } -\n ], -\n [ -\n "env", -\n { -\n "LD_LIBRARY_PATH": "/opt/miniconda-latest/envs/neuro:" -\n } -\n ], -\n [ -\n "run_bash", -\n "source activate neuro && jupyter nbextension enable exercise2/main && jupyter nbextension enable spellchecker/main" -\n ], -\n [ -\n "user", -\n "root" -\n ], -\n [ -\n "run", -\n "mkdir /data && chmod 777 /data && chmod a+s /data" -\n ], -\n [ -\n "run", -\n "mkdir /output && chmod 777 /output && chmod a+s /output" -\n ], -\n [ -\n "user", -\n "neuro" -\n ], -\n [ -\n "run", -\n "printf \"[user]\\\n\\tname = miykael\\\n\\temail = michaelnotter@hotmail.com\\\n\" > ~/.gitconfig" -\n ], -\n [ -\n "run_bash", -\n "source activate neuro && cd /data && datalad install -r ///workshops/nih-2017/ds000114 && cd ds000114 && datalad update -r && datalad get -r sub-01/ses-test/anat sub-01/ses-test/func/*fingerfootlips*" -\n ], -\n [ -\n "run", -\n "curl -L https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/580705089ad5a101f17944a9 -o /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz && tar xf /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz -C /data/ds000114/derivatives/fmriprep/. && rm /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz && find /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c -type f -not -name ?mm_T1.nii.gz -not -name ?mm_brainmask.nii.gz -not -name ?mm_tpm*.nii.gz -delete" -\n ], -\n [ -\n "copy", -\n [ -\n ".", -\n "/home/neuro/nipype_tutorial" -\n ] -\n ], -\n [ -\n "user", -\n "root" -\n ], -\n [ -\n "run", -\n "chown -R neuro /home/neuro/nipype_tutorial" -\n ], -\n [ -\n "run", -\n "rm -rf /opt/conda/pkgs/*" -\n ], -\n [ -\n "user", -\n "neuro" -\n ], -\n [ -\n "run", -\n "mkdir -p ~/.jupyter && echo c.NotebookApp.ip = \\\"0.0.0.0\\\" > ~/.jupyter/jupyter_notebook_config.py" -\n ], -\n [ -\n "workdir", -\n "/home/neuro/nipype_tutorial" -\n ] -\n ] -\n}' > /neurodocker/neurodocker_specs.json - -%environment -export LANG="en_US.UTF-8" -export LC_ALL="en_US.UTF-8" -export ND_ENTRYPOINT="/neurodocker/startup.sh" -export FORCE_SPMMCR="1" -export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu:/opt/matlabmcr-2010a/v713/runtime/glnxa64:/opt/matlabmcr-2010a/v713/bin/glnxa64:/opt/matlabmcr-2010a/v713/sys/os/glnxa64:/opt/matlabmcr-2010a/v713/extern/bin/glnxa64" -export MATLABCMD="/opt/matlabmcr-2010a/v713/toolbox/matlab" -export CONDA_DIR="/opt/miniconda-latest" -export PATH="/opt/miniconda-latest/bin:$PATH" -export LD_LIBRARY_PATH="/opt/miniconda-latest/envs/neuro:" - -%files -. /home/neuro/nipype_tutorial - -%runscript -/neurodocker/startup.sh "$@" From dd5773c807de701a9e7ee2674e20a14ea9080dbe Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 7 Aug 2020 13:34:50 -0400 Subject: [PATCH 26/41] removing circleCI --- .circleci/config.yml | 164 ------------------------------------------- README.md | 1 - 2 files changed, 165 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index b08ad56..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,164 +0,0 @@ -version: 2 -jobs: - build: - machine: - image: circleci/classic:201808-01 - steps: - - restore_cache: - keys: - - build-v1-{{ .Branch }}-{{ epoch }} - - build-v1-{{ .Branch }}- - - build-v1-master- - - build-v1- - paths: - - /tmp/docker - - run: - name: Set-up a Docker registry - command: | - docker run -d -p 5000:5000 --restart=always --name=registry \ - -v /tmp/docker:/var/lib/registry registry:2 - - run: - name: Pull existing images - command: | - set +e - docker pull localhost:5000/neurodebian - success=$? - set -e - if [[ "$success" = "0" ]]; then - echo "Pulling from local registry" - docker tag localhost:5000/neurodebian neurodebian:stretch-non-free - docker pull localhost:5000/nipype_tutorial - docker tag localhost:5000/nipype_tutorial miykael/nipype_tutorial:latest - else - echo "Pulling from Docker Hub" - docker pull neurodebian:stretch-non-free - docker tag neurodebian:stretch-non-free localhost:5000/neurodebian - docker push localhost:5000/neurodebian - fi - - - checkout - - run: - name: Build Docker image & push to registry - no_output_timeout: 60m - command: | - docker build --rm --cache-from=miykael/nipype_tutorial:latest \ - -t miykael/nipype_tutorial:latest . - docker tag miykael/nipype_tutorial:latest localhost:5000/nipype_tutorial - docker push localhost:5000/nipype_tutorial - - run: - name: Docker registry garbage collection - command: | - docker exec -it registry /bin/registry garbage-collect --delete-untagged \ - /etc/docker/registry/config.yml - - save_cache: - key: build-v1-{{ .Branch }}-{{ epoch }} - paths: - - /tmp/docker - - test_1: - machine: - image: circleci/classic:201808-01 - steps: - - restore_cache: - keys: - - build-v1-{{ .Branch }}-{{ epoch }} - - build-v1-{{ .Branch }}- - - build-v1-master- - - build-v1- - paths: - - /tmp/docker - - run: - name: Set-up a Docker registry & pull - command: | - docker run -d -p 5000:5000 --restart=always --name=registry \ - -v /tmp/docker:/var/lib/registry registry:2 - docker pull localhost:5000/nipype_tutorial - docker tag localhost:5000/nipype_tutorial miykael/nipype_tutorial:latest - - run: - name: run tests 1 - no_output_timeout: 120m - command: | - docker run -it --rm miykael/nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 1 - test_2: - machine: - image: circleci/classic:201808-01 - steps: - - restore_cache: - keys: - - build-v1-{{ .Branch }}-{{ epoch }} - - build-v1-{{ .Branch }}- - - build-v1-master- - - build-v1- - paths: - - /tmp/docker - - run: - name: Set-up a Docker registry & pull - command: | - docker run -d -p 5000:5000 --restart=always --name=registry \ - -v /tmp/docker:/var/lib/registry registry:2 - docker pull localhost:5000/nipype_tutorial - docker tag localhost:5000/nipype_tutorial miykael/nipype_tutorial:latest - - run: - name: run tests 2 - no_output_timeout: 120m - command: | - docker run -it --rm miykael/nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 2 - test_3: - machine: - image: circleci/classic:201808-01 - steps: - - restore_cache: - keys: - - build-v1-{{ .Branch }}-{{ epoch }} - - build-v1-{{ .Branch }}- - - build-v1-master- - - build-v1- - paths: - - /tmp/docker - - run: - name: Set-up a Docker registry & pull - command: | - docker run -d -p 5000:5000 --restart=always --name=registry \ - -v /tmp/docker:/var/lib/registry registry:2 - docker pull localhost:5000/nipype_tutorial - docker tag localhost:5000/nipype_tutorial miykael/nipype_tutorial:latest - - run: - name: run tests 3 - no_output_timeout: 120m - command: | - docker run -it --rm miykael/nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 3 - -workflows: - version: 2 - build_and_test: - jobs: - - build - - test_1: - requires: - - build - - test_2: - requires: - - build - - test_3: - requires: - - build - - nightly: - triggers: - - schedule: - cron: "0 10 * * *" - filters: - branches: - only: - - master - jobs: - - build - - test_1: - requires: - - build - - test_2: - requires: - - build - - test_3: - requires: - - build diff --git a/README.md b/README.md index 9a93c5e..c9740fc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # Nipype Tutorial Notebooks -[![CircleCi](https://circleci.com/gh/miykael/nipype_tutorial.svg?style=shield)](https://circleci.com/gh/miykael/nipype_tutorial/tree/master) [![GitHub issues](https://img.shields.io/github/issues/miykael/nipype_tutorial.svg)](https://github.com/miykael/nipype_tutorial/issues/) [![GitHub pull-requests](https://img.shields.io/github/issues-pr/miykael/nipype_tutorial.svg)](https://github.com/miykael/nipype_tutorial/pulls/) [![GitHub contributors](https://img.shields.io/github/contributors/miykael/nipype_tutorial.svg)](https://GitHub.com/miykael/nipype_tutorial/graphs/contributors/) From 3633d357b60b8ea6d0c0fe220fc51a7afd45106a Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 7 Aug 2020 14:57:29 -0400 Subject: [PATCH 27/41] adding Github Action CI badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c9740fc..4a35427 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Nipype Tutorial Notebooks - +[![Github Action CI](https://github.com/miykael/nipype_tutorial/workflows/CI/badge.svg?branch=master)](https://github.com/miykael/nipype_tutorial/actions?query=workflow%3ACI) [![GitHub issues](https://img.shields.io/github/issues/miykael/nipype_tutorial.svg)](https://github.com/miykael/nipype_tutorial/issues/) [![GitHub pull-requests](https://img.shields.io/github/issues-pr/miykael/nipype_tutorial.svg)](https://github.com/miykael/nipype_tutorial/pulls/) [![GitHub contributors](https://img.shields.io/github/contributors/miykael/nipype_tutorial.svg)](https://GitHub.com/miykael/nipype_tutorial/graphs/contributors/) From d0e0906dfccf3b72c9f0f8e43ea01cb1b4956cd5 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 11 Aug 2020 14:41:12 -0400 Subject: [PATCH 28/41] fixing the test script - using pytest.main --- test_notebooks.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/test_notebooks.py b/test_notebooks.py index 8a11c4f..20de7c5 100644 --- a/test_notebooks.py +++ b/test_notebooks.py @@ -1,7 +1,6 @@ -import os import sys -import time from glob import glob +import pytest def test_version(): import nipype @@ -90,7 +89,6 @@ def reduce_notebook_load(path): print('Reducing: %s' % n) notebooks.append(reduce_notebook_load(n)) - for test in notebooks: - pytest_cmd = 'pytest --nbval-lax --nbval-cell-timeout 7200 -v -s %s' % test - print(pytest_cmd) - os.system(pytest_cmd) + # testing all tests from the notebooks list + pytest.main(["--nbval-lax", "--nbval-cell-timeout", "7200", "-vs"] + notebooks) + From 92c49161180e3acae0943c41f06693eed599faec Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 11 Aug 2020 16:14:28 -0400 Subject: [PATCH 29/41] adding pytest exit code --- test_notebooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_notebooks.py b/test_notebooks.py index 20de7c5..2d2db6d 100644 --- a/test_notebooks.py +++ b/test_notebooks.py @@ -90,5 +90,5 @@ def reduce_notebook_load(path): notebooks.append(reduce_notebook_load(n)) # testing all tests from the notebooks list - pytest.main(["--nbval-lax", "--nbval-cell-timeout", "7200", "-vs"] + notebooks) - + pytest_exit_code = pytest.main(["--nbval-lax", "--nbval-cell-timeout", "7200", "-vs"] + notebooks) + sys.exit(pytest_exit_code) From 1fafc34a817c743159d0e5d1c972e0b4f0225a4d Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sat, 15 Aug 2020 11:12:27 -0400 Subject: [PATCH 30/41] changing action-docker-layer-caching to v0.0.7 --- .github/workflows/testing.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 4a04daf..ccdc31d 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -18,7 +18,7 @@ jobs: # In this step, this action saves a list of existing images, # the cache is created without them in the post run. # It also restores the cache if it exists. - - uses: satackey/action-docker-layer-caching@v0.0.5 + - uses: satackey/action-docker-layer-caching@v0.0.7 - name: build the image run: docker build . --file Dockerfile -t nipype_tutorial:latest @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: satackey/action-docker-layer-caching@v0.0.5 + - uses: satackey/action-docker-layer-caching@v0.0.7 - name: run test 1 run: docker run --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 1 @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: satackey/action-docker-layer-caching@v0.0.5 + - uses: satackey/action-docker-layer-caching@v0.0.7 - name: run test 2 run: docker run --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 2 @@ -45,6 +45,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: satackey/action-docker-layer-caching@v0.0.5 + - uses: satackey/action-docker-layer-caching@v0.0.7 - name: run test 3 run: docker run --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 3 From c8522b313fc3f550c60a4037db5675cb951525bd Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 19 Aug 2020 21:25:58 -0400 Subject: [PATCH 31/41] fixing the matlab example - adding extension to the image filename --- notebooks/advanced_create_interfaces.ipynb | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/notebooks/advanced_create_interfaces.ipynb b/notebooks/advanced_create_interfaces.ipynb index ce1052e..0ad6bd3 100644 --- a/notebooks/advanced_create_interfaces.ipynb +++ b/notebooks/advanced_create_interfaces.ipynb @@ -1214,7 +1214,7 @@ "from scipy.io import savemat\n", "\n", "# 1. save the image in matlab format as tmp_image.mat\n", - "tmp_image = 'tmp_image'\n", + "tmp_image = 'tmp_image.mat'\n", "data = nb.load(in_file).get_data()\n", "savemat(tmp_image, {b'data': data}, do_compression=False)" ] @@ -1298,7 +1298,7 @@ "\n", " def _run_interface(self, runtime): \n", " # Save the image in matlab format as tmp_image.mat\n", - " tmp_image = 'tmp_image'\n", + " tmp_image = 'tmp_image.mat'\n", " data = nb.load(self.inputs.in_file).get_data()\n", " savemat(tmp_image, {b'data': data}, do_compression=False)\n", " \n", @@ -1374,7 +1374,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "solution2": "hidden", + "solution2": "shown", "solution2_first": true }, "outputs": [], @@ -1386,7 +1386,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "solution2": "hidden" + "solution2": "shown" }, "outputs": [], "source": [ @@ -1411,7 +1411,7 @@ "\n", " def _run_interface(self, runtime): \n", " # Save the image in matlab format as tmp_image.mat\n", - " tmp_image = 'tmp_image'\n", + " tmp_image = 'tmp_image.mat'\n", " data = nb.load(self.inputs.in_file).get_data()\n", " savemat(tmp_image, {b'data': data}, do_compression=False)\n", " \n", @@ -1495,11 +1495,18 @@ "source": [ "!cat volume.txt" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python [default]", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -1513,7 +1520,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.7.8" } }, "nbformat": 4, From e0c7152b4d4880ac15909c910d6bd27f8dfda580 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 19 Aug 2020 21:27:07 -0400 Subject: [PATCH 32/41] fixing handson_analysis - saving output from plot_glass_brain to a png file to avoid issues with size limit --- notebooks/handson_analysis.ipynb | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/notebooks/handson_analysis.ipynb b/notebooks/handson_analysis.ipynb index 988fe88..dd97805 100644 --- a/notebooks/handson_analysis.ipynb +++ b/notebooks/handson_analysis.ipynb @@ -998,8 +998,11 @@ "outputs": [], "source": [ "plot_glass_brain('/output/datasink_handson/normalized/sub-07/wess_0008.nii',\n", + " output_file=\"/output/datasink_handson/normalized/sub-07/f-contr_activation.png\",\n", " colorbar=True, display_mode='lyrz', black_bg=True, threshold=25,\n", - " title='subject 7 - F-contrast: Activation');" + " title='subject 7 - F-contrast: Activation');\n", + "\n", + "Image(filename='/output/datasink_handson/normalized/sub-07/f-contr_activation.png')" ] }, { @@ -1009,8 +1012,11 @@ "outputs": [], "source": [ "plot_glass_brain('/output/datasink_handson/normalized/sub-07/wess_0009.nii',\n", + " output_file=\"/output/datasink_handson/normalized/sub-07/f-contr_differences.png\",\n", " colorbar=True, display_mode='lyrz', black_bg=True, threshold=25,\n", - " title='subject 7 - F-contrast: Differences');" + " title='subject 7 - F-contrast: Differences');\n", + "\n", + "Image(filename='/output/datasink_handson/normalized/sub-07/f-contr_differences.png')" ] }, { @@ -1648,7 +1654,7 @@ "metadata": { "anaconda-cloud": {}, "kernelspec": { - "display_name": "Python [default]", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -1662,7 +1668,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.7.8" } }, "nbformat": 4, From bfee0e760cb4c18e55b718b2800e13a02f1cc9e3 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 15 Jun 2021 17:18:07 -0400 Subject: [PATCH 33/41] FIX: Path location is updated when symlinked --- notebooks/basic_nodes.ipynb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/notebooks/basic_nodes.ipynb b/notebooks/basic_nodes.ipynb index b8e2627..20cdbfe 100644 --- a/notebooks/basic_nodes.ipynb +++ b/notebooks/basic_nodes.ipynb @@ -170,8 +170,10 @@ "metadata": {}, "outputs": [], "source": [ + "in_file = '/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz'\n", + "\n", "# Specify node inputs\n", - "bet.inputs.in_file = '/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz'\n", + "bet.inputs.in_file = in_file\n", "bet.inputs.out_file = '/output/node_T1w_bet.nii.gz'" ] }, @@ -200,7 +202,7 @@ "from nilearn.plotting import plot_anat\n", "%matplotlib inline\n", "import matplotlib.pyplot as plt\n", - "plot_anat(bet.inputs.in_file, title='BET input', cut_coords=(10,10,10),\n", + "plot_anat(in_file, title='BET input', cut_coords=(10,10,10),\n", " display_mode='ortho', dim=-1, draw_cross=False, annotate=False);\n", "plot_anat(res.outputs.out_file, title='BET output', cut_coords=(10,10,10),\n", " display_mode='ortho', dim=-1, draw_cross=False, annotate=False);" From c8d1989885317bf72ef4badafb232bbbb90d9398 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 30 Jun 2021 20:36:07 -0400 Subject: [PATCH 34/41] FIX: Save .mat files with unicode keys --- notebooks/advanced_create_interfaces.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/notebooks/advanced_create_interfaces.ipynb b/notebooks/advanced_create_interfaces.ipynb index 0ad6bd3..33c47db 100644 --- a/notebooks/advanced_create_interfaces.ipynb +++ b/notebooks/advanced_create_interfaces.ipynb @@ -1216,7 +1216,7 @@ "# 1. save the image in matlab format as tmp_image.mat\n", "tmp_image = 'tmp_image.mat'\n", "data = nb.load(in_file).get_data()\n", - "savemat(tmp_image, {b'data': data}, do_compression=False)" + "savemat(tmp_image, {'data': data}, do_compression=False)" ] }, { @@ -1300,7 +1300,7 @@ " # Save the image in matlab format as tmp_image.mat\n", " tmp_image = 'tmp_image.mat'\n", " data = nb.load(self.inputs.in_file).get_data()\n", - " savemat(tmp_image, {b'data': data}, do_compression=False)\n", + " savemat(tmp_image, {'data': data}, do_compression=False)\n", " \n", " # Load script\n", " with open(self.inputs.script_file) as script_file:\n", @@ -1413,7 +1413,7 @@ " # Save the image in matlab format as tmp_image.mat\n", " tmp_image = 'tmp_image.mat'\n", " data = nb.load(self.inputs.in_file).get_data()\n", - " savemat(tmp_image, {b'data': data}, do_compression=False)\n", + " savemat(tmp_image, {'data': data}, do_compression=False)\n", " \n", " # Load script\n", " with open(self.inputs.script_file) as script_file:\n", From 831c5972ca26137fbf965edc9fe7480365bde392 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 18 Oct 2021 11:50:27 -0400 Subject: [PATCH 35/41] FIX: Be more forgiving of the exception type Nipype changed how it reports exceptions raised in nodes in 1.7.0 --- notebooks/basic_function_interface.ipynb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/notebooks/basic_function_interface.ipynb b/notebooks/basic_function_interface.ipynb index f4b4076..5dc2b89 100644 --- a/notebooks/basic_function_interface.ipynb +++ b/notebooks/basic_function_interface.ipynb @@ -247,8 +247,8 @@ "# Run node\n", "try:\n", " rndArray.run()\n", - "except(NameError) as err:\n", - " print(\"NameError:\", err)\n", + "except Exception as err:\n", + " print(err)\n", "else:\n", " raise" ] @@ -259,8 +259,12 @@ "source": [ "As you can see, if we don't import `random` inside the scope of the function, we receive the following error:\n", "\n", - " NameError: global name 'random' is not defined\n", - " Interface Function failed to run. " + " Exception raised while executing Node rndArray_node.\n", + "\n", + " Traceback (most recent call last):\n", + " [...]\n", + " File \"\", line 3, in get_random_array\n", + " NameError: name 'random' is not defined" ] } ], From 48460056378b47cd4537bc162ae13e2fb4c2bd31 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 18 Oct 2021 14:29:44 -0400 Subject: [PATCH 36/41] ENH: Update to more recent PyBIDS --- generate.sh | 4 ++-- notebooks/basic_data_input_bids.ipynb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/generate.sh b/generate.sh index a22ace7..950990f 100644 --- a/generate.sh +++ b/generate.sh @@ -18,7 +18,7 @@ generate_docker() { conda_install="python=3.7 pytest jupyter jupyterlab jupyter_contrib_nbextensions traits pandas matplotlib scikit-learn scikit-image seaborn nbformat nb_conda" \ pip_install="https://github.com/nipy/nipype/tarball/master-%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20https://github.com/INCF/pybids/tarball/0.7.1+%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20pybids==0.13.1%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20nilearn%20datalad[full]%20nipy%20duecredit%20nbval%20niflow-nipype1-workflows" \ create_env="neuro" \ activate=True \ @@ -57,7 +57,7 @@ generate_singularity() { conda_install="python=3.7 pytest jupyter jupyterlab jupyter_contrib_nbextensions traits pandas matplotlib scikit-learn scikit-image seaborn nbformat nb_conda" \ pip_install="https://github.com/nipy/nipype/tarball/master-%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20https://github.com/INCF/pybids/tarball/0.7.1+%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20pybids==0.13.1%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20nilearn%20datalad[full]%20nipy%20duecredit%20nbval%20niflow-nipype1-workflows" \ create_env="neuro" \ activate=True \ diff --git a/notebooks/basic_data_input_bids.ipynb b/notebooks/basic_data_input_bids.ipynb index 3275f9d..e87d70a 100644 --- a/notebooks/basic_data_input_bids.ipynb +++ b/notebooks/basic_data_input_bids.ipynb @@ -140,7 +140,7 @@ "metadata": {}, "outputs": [], "source": [ - "layout.get(subject='01', suffix='bold', extensions=['nii', 'nii.gz'])" + "layout.get(subject='01', suffix='bold', extension=['.nii', '.nii.gz'])" ] }, { @@ -156,7 +156,7 @@ "metadata": {}, "outputs": [], "source": [ - "layout.get(subject='01', suffix='bold', extensions=['nii', 'nii.gz'], return_type='file')" + "layout.get(subject='01', suffix='bold', extension=['.nii', '.nii.gz'], return_type='file')" ] }, { From 0ca3dd83b48affc9274d0f61124aabbdf0f3723b Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 4 Apr 2022 13:09:51 -0400 Subject: [PATCH 37/41] MNT: Update Python to 3.8 --- generate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate.sh b/generate.sh index 950990f..3c2f44d 100644 --- a/generate.sh +++ b/generate.sh @@ -15,7 +15,7 @@ generate_docker() { --user=neuro \ --workdir /home/neuro \ --miniconda \ - conda_install="python=3.7 pytest jupyter jupyterlab jupyter_contrib_nbextensions + conda_install="python=3.8 pytest jupyter jupyterlab jupyter_contrib_nbextensions traits pandas matplotlib scikit-learn scikit-image seaborn nbformat nb_conda" \ pip_install="https://github.com/nipy/nipype/tarball/master%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20pybids==0.13.1From%20da92a08bfbc734ea3ca22d249a64ab9a1766ae2e%20Mon%20Sep%2017%2000:00:00%202001From:%20mathiasg%20%3Cmathiasg@stanford.edu%3EDate:%20Thu,%201%20Sep%202022%2015:36:31%20-0400Subject:%20[PATCH%2038/41]%20ENH:%20Allow%20passing%20in%20specific%20nipype%20branch---%20generate.sh%20|%2014%20++++++++++++--%201%20file%20changed,%2012%20insertions(+),%202%20deletions(-)diff%20--git%20a/generate.sh%20b/generate.shindex%203c2f44d..635192a%20100644---%20a/generate.sh+++%20b/generate.sh@@%20-2,6%20+2,16%20@@%20%20set%20-e%20+NIPYPE_BRANCH=$%7B1:-"master"} +case $NIPYPE_BRANCH in + master) + NIPYPE_URL="https://github.com/nipy/nipype/tarball/master" + ;; + *) + NIPYPE_URL="git+https://github.com/nipy/nipype.git@${NIPYPE_BRANCH}" + ;; +esac + # Generate Dockerfile generate_docker() { docker run --rm kaczmarj/neurodocker:master generate docker \ @@ -17,7 +27,7 @@ generate_docker() { --miniconda \ conda_install="python=3.8 pytest jupyter jupyterlab jupyter_contrib_nbextensions traits pandas matplotlib scikit-learn scikit-image seaborn nbformat nb_conda" \ - pip_install="https://github.com/nipy/nipype/tarball/master+%20%20%20%20%20%20%20%20%20%20%20%20%20pip_install="$NIPYPE_URL pybids==0.13.1 nilearn datalad[full] nipy duecredit nbval niflow-nipype1-workflows" \ create_env="neuro" \ @@ -56,7 +66,7 @@ generate_singularity() { --miniconda \ conda_install="python=3.7 pytest jupyter jupyterlab jupyter_contrib_nbextensions traits pandas matplotlib scikit-learn scikit-image seaborn nbformat nb_conda" \ - pip_install="https://github.com/nipy/nipype/tarball/master+%20%20%20%20%20%20%20%20%20%20%20%20%20pip_install="$NIPYPE_URL pybids==0.13.1 nilearn datalad[full] nipy duecredit nbval niflow-nipype1-workflows" \ create_env="neuro" \ From 1836933d07a1cdb129ab8bcc4a7890114a3c5511 Mon Sep 17 00:00:00 2001 From: mathiasg Date: Thu, 1 Sep 2022 15:37:02 -0400 Subject: [PATCH 39/41] MAINT: Bump singularity python version --- generate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate.sh b/generate.sh index 635192a..81ec0f1 100644 --- a/generate.sh +++ b/generate.sh @@ -64,7 +64,7 @@ generate_singularity() { --user=neuro \ --workdir /home/neuro \ --miniconda \ - conda_install="python=3.7 pytest jupyter jupyterlab jupyter_contrib_nbextensions + conda_install="python=3.8 pytest jupyter jupyterlab jupyter_contrib_nbextensions traits pandas matplotlib scikit-learn scikit-image seaborn nbformat nb_conda" \ pip_install="$NIPYPE_URL pybids==0.13.1 From b574d85b8fa65e6e46b7bf0b9e6bb4c6f9755414 Mon Sep 17 00:00:00 2001 From: mathiasg Date: Thu, 1 Sep 2022 15:44:33 -0400 Subject: [PATCH 40/41] TST: Add manual trigger option to GHA --- .github/workflows/testing.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index ccdc31d..b265c03 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -5,6 +5,12 @@ on: branches: [ master ] pull_request: branches: [ master ] + workflow_dispatch: + inputs: + nipype_branch: + description: 'Build specific Nipype branch' + required: true + default: 'master' jobs: @@ -14,7 +20,10 @@ jobs: steps: - uses: actions/checkout@v2 - name: generate the Dockerfile from generate.sh - run: bash generate.sh + run: | + BRANCH=${{ github.event.inputs.nipype_branch }} + BRANCH=${BRANCH:-"master"} + bash generate.sh $BRANCH # In this step, this action saves a list of existing images, # the cache is created without them in the post run. # It also restores the cache if it exists. From 9cc50bdf8151cca19bdea0398b2b63336161094d Mon Sep 17 00:00:00 2001 From: mathiasg Date: Wed, 21 Sep 2022 10:39:40 -0400 Subject: [PATCH 41/41] CI: Reset docker layer caching --- .github/workflows/testing.yml | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index b265c03..5ac3bc4 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -1,4 +1,4 @@ -name: CI +name: Build & run notebooks on: push: @@ -27,7 +27,12 @@ jobs: # In this step, this action saves a list of existing images, # the cache is created without them in the post run. # It also restores the cache if it exists. - - uses: satackey/action-docker-layer-caching@v0.0.7 + - uses: satackey/action-docker-layer-caching@v0.0.11 + with: + key: tutorial-docker-cache-{hash} + restore-keys: | + tutorial-docker-cache- + layer-tutorial-docker-cache- - name: build the image run: docker build . --file Dockerfile -t nipype_tutorial:latest @@ -35,8 +40,12 @@ jobs: needs: build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: satackey/action-docker-layer-caching@v0.0.7 + - uses: satackey/action-docker-layer-caching@v0.0.11 + with: + key: tutorial-docker-cache-{hash} + restore-keys: | + tutorial-docker-cache- + layer-tutorial-docker-cache- - name: run test 1 run: docker run --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 1 @@ -44,8 +53,12 @@ jobs: needs: build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: satackey/action-docker-layer-caching@v0.0.7 + - uses: satackey/action-docker-layer-caching@v0.0.11 + with: + key: tutorial-docker-cache-{hash} + restore-keys: | + tutorial-docker-cache- + layer-tutorial-docker-cache- - name: run test 2 run: docker run --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 2 @@ -53,7 +66,11 @@ jobs: needs: build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: satackey/action-docker-layer-caching@v0.0.7 + - uses: satackey/action-docker-layer-caching@v0.0.11 + with: + key: tutorial-docker-cache-{hash} + restore-keys: | + tutorial-docker-cache- + layer-tutorial-docker-cache- - name: run test 3 run: docker run --rm nipype_tutorial:latest python /home/neuro/nipype_tutorial/test_notebooks.py 3