Added job for training and prediction

atlas_server/src/static/training/predict.job.sh

@@ -14,11 +14,12 @@ eval $(atlas_bind_dev)
 export CUDA_VISIBLE_DEVICES=0,1,2,3
 
 # Environment variables
+export BRAIN="__BRAIN__"
+export LABELS="__LABELS__"
 export SCRATCH_USER=/p/scratch/cjinm16/schiffer1/
 export OUTPUT_SPACING=16
 
 # Set annotation directory (assume we are in the experiment folder)
-export BRAIN=__BRAIN__
 export ATLAS_ANNOTATION_DIR=`pwd`/annotations/
 export ATLAS_ANNOTATION_FILE="B${BRAIN}_{slice:04d}_annotations.hdf5"
 

atlas_server/src/static/training/train_predict.job.sh

+#!/bin/bash -ex
+
+# Load modules
+echo Loading modules...
+# Note: Purge important, surrounding atlas_controller might have a different environment. 
+module purge
+source ${HOME}/galaxy/init.sh
+export ATLAS_MODULE=atlas
+module load ${ATLAS_MODULE}
+echo Modules loaded
+# Use the dev packages
+eval $(atlas_bind_dev)
+
+export CUDA_VISIBLE_DEVICES=0,1,2,3
+
+# Environment variables
+export BRAIN="__BRAIN__"
+export LABELS="__LABELS__"
+export SCRATCH_USER=/p/scratch/cjinm16/schiffer1/
+
+# Set annotation directory (assume we are in the experiment folder)
+export ATLAS_ANNOTATION_DIR=`pwd`/annotations/
+export ATLAS_ANNOTATION_FILE="B${BRAIN}_{slice:04d}_annotations.hdf5"
+
+# Some more variables
+INPUT_SPACING=1
+OUTPUT_SPACING=16
+PREFIX="mask"
+
+which python
+ml atlas
+which python
+
+# Get path to TIFF images to determine shape
+IMAGE_PATTERN=`python -c "from atlas import constants; print(constants.original_path.format(brain=int('${BRAIN}'), slice=1234))"`
+echo "Image pattern: ${IMAGE_PATTERN}"
+
+ANNOTATION_PATTERN=`python -c "from atlas import constants; print(constants.annotation_path.format(brain=int('${BRAIN}'), slice=1234))"`
+echo "Annotation pattern: ${ANNOTATION_PATTERN}"
+
+# Setup the annotation data
+atlas img ann2hdf5 ${ATLAS_ANNOTATION_DIR}/*.json \
+                   ${IMAGE_PATTERN} \
+                   ${ANNOTATION_PATTERN} \
+                   --input-spacing ${INPUT_SPACING} \
+                   --output-spacing ${OUTPUT_SPACING} \
+                   --prefix ${PREFIX} \
+                   --overwrite \
+                   --labels "${LABELS}" \
+                   --chunk-size 64 \
+                   --compression gzip
+
+# Setup the experiment data
+atlas exp run new `pwd`
+
+# Train network
+srun --cpus-per-task 4 atlas dl train `pwd` --verbose --distribute
+#t 
+# Make prediction and run evaluation
+srun --cpus-per-task 4 atlas dl predict `pwd` --verbose --output-spacing ${OUTPUT_SPACING} --overwrite --save-argmax
+srun --cpus-per-task 4 atlas dl evaluate `pwd` --verbose --embedded-pred-to-hdf5 --output-spacing ${OUTPUT_SPACING}