From 6385f4f72f7af103fbfe10373d2fdb62e65a8e5c Mon Sep 17 00:00:00 2001 From: bpinsard Date: Fri, 9 Feb 2024 10:22:49 -0500 Subject: [PATCH] fix bids init with hidden file --- docker/dicom_indexer/indexer/index_dicom.py | 53 +++++++++++++++---- .../indexer/repo_templates/bids/.bids_init | 1 + 2 files changed, 43 insertions(+), 11 deletions(-) create mode 100644 docker/dicom_indexer/indexer/repo_templates/bids/.bids_init diff --git a/docker/dicom_indexer/indexer/index_dicom.py b/docker/dicom_indexer/indexer/index_dicom.py index bfe8fe8..ea1396b 100644 --- a/docker/dicom_indexer/indexer/index_dicom.py +++ b/docker/dicom_indexer/indexer/index_dicom.py @@ -4,6 +4,7 @@ import argparse import pathlib import urllib.parse import datalad.api as dlad +import datalad.config import shutil import gitlab import tempfile @@ -12,6 +13,10 @@ import subprocess import yaml from contextlib import contextmanager +REPO_TEMPLATES_PATH = ( + pathlib.Path(os.path.dirname(os.path.realpath(__file__))) / "repo_templates" +) + DEBUG = bool(os.environ.get("DEBUG", False)) if DEBUG: logging.basicConfig(level=logging.DEBUG) @@ -19,6 +24,7 @@ if DEBUG: GITLAB_REMOTE_NAME = os.environ.get("GITLAB_REMOTE_NAME", "origin") GITLAB_TOKEN = os.environ.get("GITLAB_TOKEN", None) GITLAB_BOT_USERNAME = os.environ.get("GITLAB_BOT_USERNAME", None) +GITLAB_BOT_EMAIL = os.environ.get("GITLAB_BOT_EMAIL", None) BIDS_DEV_BRANCH = os.environ.get("BIDS_DEV_BRANCH", "dev") NI_DATAOPS_GITLAB_ROOT = os.environ.get("NI_DATAOPS_GITLAB_ROOT", "ni-dataops") @@ -33,6 +39,23 @@ S3_REMOTE_DEFAULT_PARAMETERS = [ ] +def git_global_setup( + storage_remote_url: urllib.parse.ParseResult, scope="global" +) -> None: + git_config = datalad.config.ConfigManager() + git_config.add("user.name", GITLAB_BOT_USERNAME, scope=scope) + git_config.add("user.email", GITLAB_BOT_EMAIL, scope=scope) + if storage_remote_url.scheme == "s3": + import socket + + s3_ip = socket.gethostbyname(storage_remote_url.hostname) + git_config.add( + "annex.security.allowed-ip-addresses", + s3_ip, + scope=scope, + ) + + # TODO: rewrite for pathlib.Path input def sort_series(path: pathlib.Path) -> None: """Sort series in separate folder @@ -82,7 +105,7 @@ def _build_arg_parser() -> argparse.ArgumentParser: p.add_argument("--storage-remote", help="url to the datalad remote") p.add_argument( "--sort-series", - type=bool, + action="store_true", default=True, help="sort dicom series in separate folders", ) @@ -115,6 +138,8 @@ def main() -> None: output_remote = urllib.parse.urlparse(args.storage_remote) gitlab_url = urllib.parse.urlparse(args.gitlab_url) + git_global_setup(output_remote) + with index_dicoms( input, sort_series=args.sort_series, @@ -177,7 +202,7 @@ def index_dicoms( # cannot pass message above so commit now dicom_session_ds.save(message=f"index dicoms from archive {archive}") # # optimize git index after large import - dicom_session_ds.repo.gc() # aggressive by default + #dicom_session_ds.repo.gc() # aggressive by default yield dicom_session_ds @@ -198,7 +223,9 @@ def export_data( export_to_s3(dicom_session_ds, output_remote, session_metas) -def set_bot_privileges(gitlab_conn: gitlab.Gitlab, gitlab_group_path: pathlib.Path) -> None: +def set_bot_privileges( + gitlab_conn: gitlab.Gitlab, gitlab_group_path: pathlib.Path +) -> None: # add maint permissions for the dicom bot user on the study repos study_group = get_or_create_gitlab_group(gitlab_conn, gitlab_group_path) bot_user = gitlab_conn.users.list(username=GITLAB_BOT_USERNAME) @@ -295,9 +322,11 @@ def init_bids( source=bids_project_repo._attrs["ssh_url_to_repo"], path=tmpdir, ) - bids_project_ds.create(force=True) - shutil.copytree("repo_templates/bids", bids_project_ds.path, dirs_exist_ok=True) + shutil.copytree( + REPO_TEMPLATES_PATH / "bids", bids_project_ds.path, dirs_exist_ok=True + ) write_ci_env(bids_project_ds, gitlab_group_path) + bids_project_ds.create(force=True) bids_project_ds.save(path=".", message="init structure and pipelines") bids_project_ds.install( path="sourcedata/dicoms", @@ -318,7 +347,7 @@ def init_dicom_study( gitlab_group_path: pathlib.Path, ) -> None: shutil.copytree( - "repo_templates/dicom_study", dicom_study_ds.path, dirs_exist_ok=True + REPO_TEMPLATES_PATH / "dicom_study", dicom_study_ds.path, dirs_exist_ok=True ) write_ci_env(dicom_study_ds, gitlab_group_path) dicom_study_ds.save(path=".", message="init structure and pipelines") @@ -326,14 +355,13 @@ def init_dicom_study( def write_ci_env( - ds: dlad.Dataset - gitlab_group_path: pathlib.Path -): + ds: dlad.Dataset, + gitlab_group_path: pathlib.Path, +) -> None: env = { "variables": { "STUDY_PATH": str(gitlab_group_path), "BIDS_PATH": str(gitlab_group_path / "bids"), - "NI_DATAOPS_GITLAB_ROOT": NI_DATAOPS_GITLAB_ROOT, } } with (pathlib.Path(ds.path) / ".ci-env.yml").open("w") as outfile: @@ -435,7 +463,10 @@ def export_to_s3( # TODO: check if we can reuse a single bucket (or per study) with fileprefix # git-annex initremote remotename ... remote_name = s3_url.hostname - _, bucket_name, *fileprefix = pathlib.Path(s3_url.path).parts + s3_path = s3_url.path + if '{' in s3_path: + s3_path = s3_path.format(**session_metas) + _, bucket_name, *fileprefix = pathlib.Path(s3_path).parts fileprefix.append(session_metas["StudyInstanceUID"] + "/") ds.repo.init_remote( remote_name, diff --git a/docker/dicom_indexer/indexer/repo_templates/bids/.bids_init b/docker/dicom_indexer/indexer/repo_templates/bids/.bids_init new file mode 100644 index 0000000..a2f7595 --- /dev/null +++ b/docker/dicom_indexer/indexer/repo_templates/bids/.bids_init @@ -0,0 +1 @@ +dummy file to trigger ci to init BIDS branches and remotes