From ad5eb5bbb626f63fe2be897dc4cfb3f4cac50ead Mon Sep 17 00:00:00 2001 From: bpinsard Date: Tue, 30 Jan 2024 13:43:32 -0500 Subject: [PATCH] add repo templates, fix s3 export --- docker/dicom_indexer/indexer/index_dicom.py | 12 +++++++----- .../indexer/repo_templates/bids/.all-contributorsrc | 5 +++++ .../indexer/repo_templates/bids/.bidsignore | 3 +++ .../indexer/repo_templates/bids/.derivatives | 1 + .../indexer/repo_templates/bids/.gitattributes | 13 +++++++++++++ .../indexer/repo_templates/bids/.gitignore | 2 ++ .../indexer/repo_templates/bids/.gitlab-ci.yml | 4 ++++ .../indexer/repo_templates/bids/README | 6 ++++++ .../repo_templates/dicom_study/.gitattributes | 12 ++++++++++++ .../repo_templates/dicom_study/.gitlab-ci.yml | 6 ++++++ 10 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 docker/dicom_indexer/indexer/repo_templates/bids/.all-contributorsrc create mode 100644 docker/dicom_indexer/indexer/repo_templates/bids/.bidsignore create mode 100644 docker/dicom_indexer/indexer/repo_templates/bids/.derivatives create mode 100644 docker/dicom_indexer/indexer/repo_templates/bids/.gitattributes create mode 100644 docker/dicom_indexer/indexer/repo_templates/bids/.gitignore create mode 100644 docker/dicom_indexer/indexer/repo_templates/bids/.gitlab-ci.yml create mode 100644 docker/dicom_indexer/indexer/repo_templates/bids/README create mode 100644 docker/dicom_indexer/indexer/repo_templates/dicom_study/.gitattributes create mode 100644 docker/dicom_indexer/indexer/repo_templates/dicom_study/.gitlab-ci.yml diff --git a/docker/dicom_indexer/indexer/index_dicom.py b/docker/dicom_indexer/indexer/index_dicom.py index d1e872a..f495a85 100644 --- a/docker/dicom_indexer/indexer/index_dicom.py +++ b/docker/dicom_indexer/indexer/index_dicom.py @@ -426,19 +426,21 @@ def export_to_s3( # TODO: check if we can reuse a single bucket (or per study) with fileprefix # git-annex initremote remotename ... remote_name = s3_url.hostname - bucket_name, path = pathlib.Path(s3_url.path).parts - ds.repo.initremote( + _, bucket_name, *fileprefix = pathlib.Path(s3_url.path).parts + fileprefix.append(session_metas['StudyInstanceUID']+'/') + ds.repo.init_remote( remote_name, S3_REMOTE_DEFAULT_PARAMETERS + [ f"host={s3_url.hostname}", f"bucket={bucket_name}", - f"fileprefix={'/'.join(path)}", + f"fileprefix={'/'.join(fileprefix)}", ], ) ds.repo.set_preferred_content( - remote_name, - "include=**.{7z,tar.gz,zip}", + "wanted", + "include=*7z or include=*.tar.gz or include=*zip", + remote=remote_name, ) ds.push(to=remote_name) diff --git a/docker/dicom_indexer/indexer/repo_templates/bids/.all-contributorsrc b/docker/dicom_indexer/indexer/repo_templates/bids/.all-contributorsrc new file mode 100644 index 0000000..ead2a2a --- /dev/null +++ b/docker/dicom_indexer/indexer/repo_templates/bids/.all-contributorsrc @@ -0,0 +1,5 @@ +{ + "files": [ + "README" + ] +} diff --git a/docker/dicom_indexer/indexer/repo_templates/bids/.bidsignore b/docker/dicom_indexer/indexer/repo_templates/bids/.bidsignore new file mode 100644 index 0000000..e6dbe00 --- /dev/null +++ b/docker/dicom_indexer/indexer/repo_templates/bids/.bidsignore @@ -0,0 +1,3 @@ +**/anat/*localizer* +**/anat/*scout* +**/*__dup* diff --git a/docker/dicom_indexer/indexer/repo_templates/bids/.derivatives b/docker/dicom_indexer/indexer/repo_templates/bids/.derivatives new file mode 100644 index 0000000..c8a15dd --- /dev/null +++ b/docker/dicom_indexer/indexer/repo_templates/bids/.derivatives @@ -0,0 +1 @@ +qc/mriqc diff --git a/docker/dicom_indexer/indexer/repo_templates/bids/.gitattributes b/docker/dicom_indexer/indexer/repo_templates/bids/.gitattributes new file mode 100644 index 0000000..b63f3e1 --- /dev/null +++ b/docker/dicom_indexer/indexer/repo_templates/bids/.gitattributes @@ -0,0 +1,13 @@ +**/.git* annex.largefiles=nothing +* annex.largefiles=(largerthan=100kb) +*.yml annex.largefiles=nothing +*.json annex.largefiles=nothing +*.txt annex.largefiles=nothing +*.tsv annex.largefiles=nothing +*.nii.gz annex.largefiles=anything +*.tgz annex.largefiles=anything +*_scans.tsv annex.largefiles=anything +# annex event files as they contain subjects behavioral responses +sub-*/**/*_events.tsv annex.largefiles=anything +*.bk2 annex.largefiles=anything +.bidsignore annex.largefiles=nothing diff --git a/docker/dicom_indexer/indexer/repo_templates/bids/.gitignore b/docker/dicom_indexer/indexer/repo_templates/bids/.gitignore new file mode 100644 index 0000000..899093a --- /dev/null +++ b/docker/dicom_indexer/indexer/repo_templates/bids/.gitignore @@ -0,0 +1,2 @@ +.pybids_cache/** +.heudiconv diff --git a/docker/dicom_indexer/indexer/repo_templates/bids/.gitlab-ci.yml b/docker/dicom_indexer/indexer/repo_templates/bids/.gitlab-ci.yml new file mode 100644 index 0000000..51dba8d --- /dev/null +++ b/docker/dicom_indexer/indexer/repo_templates/bids/.gitlab-ci.yml @@ -0,0 +1,4 @@ +include: + - project: 'unf/ni-dataops' + file: + - 'ci-pipelines/bids/bids_repo.yml' diff --git a/docker/dicom_indexer/indexer/repo_templates/bids/README b/docker/dicom_indexer/indexer/repo_templates/bids/README new file mode 100644 index 0000000..eb7bcdd --- /dev/null +++ b/docker/dicom_indexer/indexer/repo_templates/bids/README @@ -0,0 +1,6 @@ +# xyz dataset + +## Contributors + + + diff --git a/docker/dicom_indexer/indexer/repo_templates/dicom_study/.gitattributes b/docker/dicom_indexer/indexer/repo_templates/dicom_study/.gitattributes new file mode 100644 index 0000000..637ed86 --- /dev/null +++ b/docker/dicom_indexer/indexer/repo_templates/dicom_study/.gitattributes @@ -0,0 +1,12 @@ +**/.git* annex.largefiles=nothing +* annex.largefiles=(largerthan=100kb) +*.yml annex.largefiles=nothing +*.json annex.largefiles=nothing +*.txt annex.largefiles=nothing +*.tsv annex.largefiles=nothing +*.nii.gz annex.largefiles=anything +*.tgz annex.largefiles=anything +*_scans.tsv annex.largefiles=anything +# annex event files as they contain subjects behavioral responses +sub-*/**/*_events.tsv annex.largefiles=anything +*.bk2 annex.largefiles=anything diff --git a/docker/dicom_indexer/indexer/repo_templates/dicom_study/.gitlab-ci.yml b/docker/dicom_indexer/indexer/repo_templates/dicom_study/.gitlab-ci.yml new file mode 100644 index 0000000..78031e8 --- /dev/null +++ b/docker/dicom_indexer/indexer/repo_templates/dicom_study/.gitlab-ci.yml @@ -0,0 +1,6 @@ + +include: + - local: /ci-env.yml + - project: 'unf/ni-dataops' + file: + - 'ci-pipelines/sources/dicoms_study.yml'