From 59e04d6e901be821e4a57061062165e7bd5077d7 Mon Sep 17 00:00:00 2001 From: bpinsard Date: Fri, 27 Oct 2023 13:54:58 -0400 Subject: [PATCH] refactor initial commit --- .gitlab-ci.yml | 81 +++++++++++++++++++++++++++++ README.md | 28 ++++++++++ docker/datalad-apptainer/Dockerfile | 43 +++++++++++++++ docker/datalad-docker/Dockerfile | 4 ++ docker/deface/Dockerfile | 13 +++++ docker/heudiconv/Dockerfile | 14 +++++ docker/pydeface/Dockerfile | 32 ++++++++++++ 7 files changed, 215 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 README.md create mode 100644 docker/datalad-apptainer/Dockerfile create mode 100644 docker/datalad-docker/Dockerfile create mode 100644 docker/deface/Dockerfile create mode 100644 docker/heudiconv/Dockerfile create mode 100644 docker/pydeface/Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..d960585 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,81 @@ +image: docker:20.10.16 + +variables: + # When you use the dind service, you must instruct Docker to talk with + # the daemon started inside of the service. The daemon is available + # with a network connection instead of the default + # /var/run/docker.sock socket. Docker 19.03 does this automatically + # by setting the DOCKER_HOST in + # https://github.com/docker-library/docker/blob/d45051476babc297257df490d22cbd806f1b11e4/19.03/docker-entrypoint.sh#L23-L29 + # + # The 'docker' hostname is the alias of the service container as described at + # https://docs.gitlab.com/ee/ci/services/#accessing-the-services. + # + # Specify to Docker where to create the certificates. Docker + # creates them automatically on boot, and creates + # `/certs/client` to share between the service and job + # container, thanks to volume mount from config.toml + DOCKER_TLS_CERTDIR: "/certs" + +services: + - name: docker:20.10.16-dind +# command: ["--registry-mirror", "http://maple.criugm.qc.ca:5000" ] # Specify the registry mirror to use + + +stages: + - build + +.build_tpl: + stage: build + script: + - export IMAGE_TAG=$IMAGE:$CI_COMMIT_REF_SLUG + - echo $CI_REGISTRY_USER $CI_REGISTRY $IMAGE_TAG + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker pull $IMAGE:latest || true + - docker build --cache-from $IMAGE:latest -t $IMAGE:$CI_COMMIT_SHA -t $IMAGE:$CI_COMMIT_SHORT_SHA -t $IMAGE_TAG -t $IMAGE:latest ./docker/datalad-docker + - docker push --all-tags $IMAGE + +build_docker-datalad: + extends: .build_tpl + variables: + IMAGE: $CI_REGISTRY_IMAGE/docker-datalad + rules: + - changes: + - docker/datalad-docker/**/* + - .gitlab-ci.yml + +build_datalad-apptainer: + extends: .build_tpl + variables: + IMAGE: $CI_REGISTRY_IMAGE/datalad-apptainer + rules: + - changes: + - docker/datalad-apptainer/**/* + - .gitlab-ci.yml + +build_heudiconv: + extends: .build_tpl + variables: + IMAGE: $CI_REGISTRY_IMAGE/heudiconv + rules: + - changes: + - docker/heudiconv/**/* + - .gitlab-ci.yml + +build_deface: + extends: .build_tpl + variables: + IMAGE: $CI_REGISTRY_IMAGE/deface + rules: + - changes: + - docker/deface/**/* + - .gitlab-ci.yml + +build_pydeface: + extends: .build_tpl + variables: + IMAGE: $CI_REGISTRY_IMAGE/pydeface + rules: + - changes: + - docker/pydeface/**/* + - .gitlab-ci.yml diff --git a/README.md b/README.md new file mode 100644 index 0000000..5b91b17 --- /dev/null +++ b/README.md @@ -0,0 +1,28 @@ +# ni-dataops containers + +This repo contains builds of containers used in the ci-pipelines +To build these, it requires a docker-in-docker runner. +Any changes to Dockerfile or folder of a container should trigger a rebuild and push to gitlab registry. + +## datalad-apptainer + +A simple lightweight container (alpine-based) that contains datalad, datalad-containers and apptainer. +It's an apptainer-in-docker setup, which avoids caveats of using docker-in-docker (DinD), and allow to use community or custom containers made for rootless HPC usage. +That simple env allows notably to run ReproNim bids-app on bids repos but could also run other apptainer images. + +## datalad-docker + +Not currently used, replaced by datalad-apptainer. + + +## heudiconv + +Container used for the dicom to BIDS conversion. + +## deface + +Custom defacing + +## pydeface + +Not currently used. diff --git a/docker/datalad-apptainer/Dockerfile b/docker/datalad-apptainer/Dockerfile new file mode 100644 index 0000000..d885229 --- /dev/null +++ b/docker/datalad-apptainer/Dockerfile @@ -0,0 +1,43 @@ +FROM golang:1.20.1-alpine3.17 as builder + +RUN apk add --no-cache \ + # Required for apptainer to find min go version + bash \ + cryptsetup \ + gawk \ + gcc \ + git \ + libc-dev \ + linux-headers \ + libressl-dev \ + libuuid \ + libseccomp-dev \ + make \ + util-linux-dev + +ARG APPTAINER_COMMITISH="main" +ARG MCONFIG_OPTIONS="--with-suid" +WORKDIR $GOPATH/src/github.com/apptainer +RUN git clone https://github.com/apptainer/apptainer.git \ + && cd apptainer \ + && git checkout "$APPTAINER_COMMITISH" \ + && ./mconfig $MCONFIG_OPTIONS -p /usr/local/apptainer \ + && cd builddir \ + && make \ + && make install + +FROM alpine:3.17.2 +COPY --from=builder /usr/local/apptainer /usr/local/apptainer +ENV PATH="/usr/local/apptainer/bin:$PATH" \ + APPTAINER_TMPDIR="/tmp-apptainer" +RUN apk add --no-cache ca-certificates libseccomp squashfs-tools tzdata fuse2fs fuse-overlayfs \ + python3 py3-pip git openssh-client git-annex curl bzip2 bash glab\ + && mkdir -p $APPTAINER_TMPDIR \ + && cp /usr/share/zoneinfo/UTC /etc/localtime \ + && apk del tzdata \ + && rm -rf /tmp/* /var/cache/apk/* +RUN apk add --no-cache squashfuse --repository=https://dl-cdn.alpinelinux.org/alpine/edge/testing + +RUN pip install --no-cache-dir datalad datalad-container pytest ssh_agent_setup python-gitlab + +WORKDIR /work diff --git a/docker/datalad-docker/Dockerfile b/docker/datalad-docker/Dockerfile new file mode 100644 index 0000000..f0be992 --- /dev/null +++ b/docker/datalad-docker/Dockerfile @@ -0,0 +1,4 @@ +FROM docker:20.10.16 + +RUN apk add git git-annex python3 py3-pip p7zip glab +RUN pip install --no-cache-dir datalad datalad-container python-gitlab diff --git a/docker/deface/Dockerfile b/docker/deface/Dockerfile new file mode 100644 index 0000000..ecdf634 --- /dev/null +++ b/docker/deface/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.8-slim + +RUN apt-get update \ + && apt-get -y install git git-annex build-essential\ + && apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* +RUN git clone https://github.com/courtois-neuromod/ds_prep.git && \ + cd ds_prep && git checkout gitlab && \ + pip install --no-cache-dir packaging datalad && \ + pip install --no-cache-dir -r requirements.txt +RUN git config --global user.name docker_bot && \ + git config --global user.email docket_bot@dummy.net +RUN cd /ds_prep && \ + datalad get global/templates/*.nii.gz diff --git a/docker/heudiconv/Dockerfile b/docker/heudiconv/Dockerfile new file mode 100644 index 0000000..eec3a38 --- /dev/null +++ b/docker/heudiconv/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.10-slim + +RUN apt-get update \ + && apt-get -y install 7zip curl git git-annex dcm2niix \ + && apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* + +ARG GLAB_VERSION=1.25.3 +RUN curl -O -L https://gitlab.com/gitlab-org/cli/-/releases/v${GLAB_VERSION}/downloads/glab_${GLAB_VERSION}_Linux_x86_64.deb \ + && dpkg -i glab_${GLAB_VERSION}_Linux_x86_64.deb \ + && rm -f glab_${GLAB_VERSION}_Linux_x86_64.deb +RUN ln -s /usr/bin/7zz /usr/bin/7z + +RUN pip install --no-cache-dir frozendict bids datalad +RUN pip install --no-cache-dir git+https://github.com/UNFmontreal/heudiconv@enh-custom-seqinfo diff --git a/docker/pydeface/Dockerfile b/docker/pydeface/Dockerfile new file mode 100644 index 0000000..df64c95 --- /dev/null +++ b/docker/pydeface/Dockerfile @@ -0,0 +1,32 @@ +FROM nipreps/fmriprep:unstable as fsl_source + +FROM python:3.10-slim + +COPY --from=fsl_source /opt/conda/envs/fmriprep/bin/flirt /opt/fsl/bin/ + +COPY --from=fsl_source /opt/conda/envs/fmriprep/lib/libquadmath.so.0.0.0 /opt/fsl/lib/ +COPY --from=fsl_source /opt/conda/envs/fmriprep/lib/libgfortran.so.5.0.0 /opt/fsl/lib/ +COPY --from=fsl_source /opt/conda/envs/fmriprep/lib/libfsl-NewNifti.so /opt/fsl/lib/ +COPY --from=fsl_source /opt/conda/envs/fmriprep/lib/libfsl-cprob.so /opt/fsl/lib/ +COPY --from=fsl_source /opt/conda/envs/fmriprep/lib/libfsl-miscmaths.so /opt/fsl/lib/ +COPY --from=fsl_source /opt/conda/envs/fmriprep/lib/libfsl-newimage.so /opt/fsl/lib/ +COPY --from=fsl_source /opt/conda/envs/fmriprep/lib/libfsl-utils.so /opt/fsl/lib/ +COPY --from=fsl_source /opt/conda/envs/fmriprep/lib/libfsl-znz.so /opt/fsl/lib/ +COPY --from=fsl_source /opt/conda/envs/fmriprep/lib/libgcc_s.so.1 /opt/fsl/lib/ +COPY --from=fsl_source /opt/conda/envs/fmriprep/lib/libopenblasp-r0.3.23.so /opt/fsl/lib/ +COPY --from=fsl_source /opt/conda/envs/fmriprep/lib/libstdc++.so.6.0.31 /opt/fsl/lib/ +COPY --from=fsl_source /opt/conda/envs/fmriprep/lib/libz.so.1.2.13 /opt/fsl/lib/ + +RUN ln -s libopenblasp-r0.3.23.so /opt/fsl/lib/liblapack.so.3 +RUN ln -s libopenblasp-r0.3.23.so /opt/fsl/lib/libblas.so.3 +RUN ln -s libquadmath.so.0.0.0 /opt/fsl/lib/libquadmath.so.0 +RUN ln -s libgfortran.so.5.0.0 /opt/fsl/lib/libgfortran.so.5 +RUN ln -s libstdc++.so.6.0.31 /opt/fsl/lib/libstdc++.so.6 +RUN ln -s libz.so.1.2.13 /opt/fsl/lib/libz.so.1 + +ENV LD_LIBRARY_PATH=/opt/fsl/lib + +RUN pip --no-cache-dir install pydeface +ENV PATH=$PATH:/opt/fsl/bin +ENV FSLDIR=/opt/fsl/bin +RUN sed -i "s/shutil.which('fsl')/shutil.which('flirt')/g" /usr/local/lib/python3.10/site-packages/pydeface/utils.py