dataLoaderOpenMind.py

import csv
import json
from collections import defaultdict
from pathlib import Path

def load_metadata(csv_path):
    metadata = {}
    with open(csv_path, newline='', encoding='utf-8') as csvfile:
        reader = csv.DictReader(csvfile)
        for row in reader:
            uid = row["unique_id"].strip()
            metadata[uid] = {k: v.strip() for k, v in row.items() if v and k != "unique_id"}
    print(f"[DEBUG] Loaded {len(metadata)} metadata entries.")
    return metadata


def find_metadata_entry(metadata, dataset, subject, session):
    prefix = f"{dataset}__{subject}__{session}"
    for uid, meta in metadata.items():
        if uid.startswith(prefix):
            return meta
    return {}


def extract_modality(nii_name):
    """
    Extracts modality (T1w, T2w, FLAIR, etc.) from file name.
    Examples:
      sub-16_preproc-quasiraw_T1w_mask.nii.gz         -> T1w
      sub-0076_ses-02_preproc-quasiraw-sitk_T2w.nii.gz -> T2w
      sub-0076_ses-02_TSE_run-02_preproc-quasiraw-sitk_T2w_mask.nii.gz -> T2w
    """
    base = nii_name.replace(".nii.gz", "")
    parts = base.split("_")
    for p in parts:
        if p.upper() in {"T1W","MP2RAGE"}:#, "T2W", "FLAIR"}:
            return p.upper()
    return "unknown"


def convert_structure(base_dir, csv_path, output_json):
    base_dir = Path(base_dir)
    metadata = load_metadata(csv_path)
    grouped = defaultdict(list)
    print(f"[INFO] Scanning base directory: {base_dir}")

    nii_files = [
        f for f in base_dir.rglob("*preproc-quasiraw*.nii.gz")
        if "_mask" not in f.name
    ]
    print(f"[DEBUG] Found {len(nii_files)} NIfTI files matching 'preproc-quasiraw' pattern.")

    for nii_path in nii_files:
        modality = extract_modality(nii_path.name)
        if modality not in {"T1W","MP2RAGE"}:#, "T2W", "FLAIR"}:
            print(f"[SKIP] Ignoring {nii_path.name} (modality={modality})")
            continue

        print(f"[DEBUG] Processing {nii_path.name} (modality={modality})")

        run = next((p for p in nii_path.stem.split("_") if p.startswith("run-")), "run-1")
        subject = next((p for p in nii_path.parts if p.startswith("sub-")), None)
        dataset = next((p for p in nii_path.parts if p.startswith("ds")), None)
        session = next((p for p in nii_path.parts if p.startswith("ses-")), "ses-DEFAULT")

        meta = find_metadata_entry(metadata, dataset, subject, session)
        if not meta:
            print(f"[WARNING] No metadata found for prefix: {dataset}__{subject}__{session}")

        full_path = str(nii_path)
        rel_path = full_path.split("OpenMind/", 1)[-1]
        entry = {
            "image_path": "/scratch/OpenMind/" + rel_path,
            "modality": modality
        }

        for k, v in meta.items():
            if k not in entry:
                try:
                    entry[k] = float(v) if "." in v else int(v)
                except ValueError:
                    entry[k] = v

        grouped[f"{dataset}__{subject}"].append(entry)

    final_output = []
    for subj_id, entries in grouped.items():
        print(f"[DEBUG] Finalizing subject: {subj_id} with {len(entries)} valid entries.")
        final_output.append({
            "subject": subj_id,
            "data": entries
        })

    with open(output_json, "w", encoding="utf-8") as f:
        json.dump(final_output, f, indent=2)
    print(f"✅ JSON saved to: {output_json}")


def main():
    base_dir = Path.home() / "scratch" / "OpenMind"
    metadata_csv = Path("openneuro_metadata.csv")
    output_json = Path("qrawopenneuro_filtered.json")

    print("[INFO] Starting conversion process...")
    convert_structure(base_dir, metadata_csv, output_json)
    print("[INFO] Conversion completed.")


if __name__ == "__main__":
    main()