main.py added, argparse added

code/data.py

@@ -31,26 +31,28 @@ pli_path = '/p/fastdata/pli/Private/oberstrass1/datasets/vervet1818/vervet1818-s
 cyto_path = '/p/fastdata/pli/Private/oberstrass1/datasets/vervet1818/vervet1818-stained/data/aligned/stained'
 
 pli_files_list = [file for file in os.listdir(pli_path) if file.endswith(('.h5', '.hdf', '.h4', '.hdf4', '.he2', '.hdf5', '.he5'))]
+pli_files_list.sort()
 cyto_files_list = [file for file in os.listdir(cyto_path) if file.endswith(('.h5', '.hdf', '.h4', '.hdf4', '.he2', '.hdf5', '.he5'))]
+cyto_files_list.sort()
 
 
 # print(len(pli_files_list))
-
 # print(pli_files_list)
 # print(cyto_files_list)
 
+
 num_images = len(pli_files_list)
 
 class TestSampler(Dataset):
     # Gives you a random crop and a random image at each request
 
-    def __init__(self, pli_files_list, cyto_files_list, transforms, crop_size, dataset_size):
+    def __init__(self, pli_list, cyto_list, transforms, crop_size, dataset_size):
         # crop_size is the size before the rotation and center crop. So the patch_size * sqrt(2)
         # dataset_size defines the number of drawn patches per epoch. As we are drawing (arbitrary many) random patches we have to set is manually
         super().__init__()
         # list of pli has to be in the same order as list of cyto. So index i in pli should correspond to the same index in cyto
-        self.list_of_pli = pli_files_list
-        self.list_of_cyto = cyto_files_list
+        self.list_of_pli = pli_list
+        self.list_of_cyto = cyto_list
         self.n_images = num_images
         self.transforms = transforms
         self.crop_size = crop_size
@@ -59,19 +61,12 @@ class TestSampler(Dataset):
     def __getitem__(self, ix):
         # Get a random image
         i = random.randint(0, self.n_images-1)
-
-        '''
-        pli_temp = h5py.File(os.path.join(pli_path, self.list_of_pli[i]), 'r')
-        pli_image = pli_temp.get('Image').value
-        #pli_image = self.list_of_pli[i]
-        cyto_temp = h5py.File(os.path.join(pli_path, self.list_of_cyto[i]), 'r')
-        cyto_image = cyto_temp.get('Image').value
-        #cyto_image = self.list_of_cyto[i]
-        '''
+        pli_image = self.list_of_pli[i]
+        cyto_image = self.list_of_cyto[i]
 
         # Generate a random patch location from the image
-        x = random.randint(image.shape[1] - self.crop_size)
-        y = random.randint(image.shape[0] - self.crop_size)
+        x = random.randint(pli_image.shape[1] - self.crop_size)
+        y = random.randint(pli_image.shape[0] - self.crop_size)
 
         # Get crops at the x, y location with size crop_size x crop_size
         random_crop_pli = pli_image[y:y + self.crop_size, x:x + self.crop_size]
@@ -129,8 +124,22 @@ class TestDataModule(pl.LightningDataModule):
 
             # TODO: Load the PLI and Cytp train data here as lists of numpy arrays: List[np.ndarray]
             # Load the pyramid/00 per file
-            self.pli_train = h5py.File(os.path.join(pli_path, pli_files_list[:3]), 'r')
-            self.cyto_train = h5py.File(os.path.join(cyto_path, cyto_files_list[:3]), 'r')
+
+            self.pli_train = []
+            self.cyto_train = []
+
+            for i in range(0,3):
+                pli_train_file = h5py.File(os.path.join(pli_path, pli_files_list[i]), 'r')
+                pli_train_file = pli_train_file['pyramid/00']
+                pli_train_file = np.asarray(pli_train_file)
+                self.pli_train.append(pli_train_file)
+
+            for i in range(0,3):
+                cyto_train_file = h5py.File(os.path.join(pli_path, cyto_files_list[i]), 'r')
+                cyto_train_file = cyto_train_file['pyramid/00']
+                cyto_train_file = np.asarray(cyto_train_file)
+                self.cyto_train.append(cyto_train_file)
+
         else:
             print(f"Train data for rank {rank}/{size} already prepared")
 
@@ -140,8 +149,19 @@ class TestDataModule(pl.LightningDataModule):
             # TODO: Load the PLI and Cytp val data here as lists of numpy arrays: List[np.ndarray]
             # This should contain only unseen images
             # Load the pyramid/00 per file
-            self.pli_val = h5py.File(os.path.join(pli_path, pli_files_list[4]), 'r')
-            self.cyto_val = h5py.File(os.path.join(cyto_path, cyto_files_list[4]), 'r')
+            pli_val = []
+            cyto_val = []
+
+            pli_val_file = h5py.File(os.path.join(pli_path, pli_files_list[4]), 'r')
+            pli_val_file = pli_val_file['pyramid/00']
+            pli_val_file = np.asarray(pli_val_file)
+            pli_val.append(pli_val_file)
+
+            cyto_val_file = h5py.File(os.path.join(pli_path, cyto_files_list[4]), 'r')
+            cyto_val_file = cyto_val_file['pyramid/00']
+            cyto_val_file = np.asarray(cyto_val_file)
+            cyto_val.append(cyto_val_file)
+
         else:
             print(f"Validation data for rank {rank}/{size} already prepared")
 

doc/tensorboard/Ragib_UNet/version_10/hparams.yaml

+accelerator: null
+accumulate_grad_batches: null
+amp_backend: native
+amp_level: null
+auto_lr_find: false
+auto_scale_batch_size: false
+auto_select_gpus: false
+batch_size: 8
+benchmark: null
+check_val_every_n_epoch: 1
+checkpoint_callback: null
+ckpt_dir: tmp/ckpt/
+crop_size: 362
+default_root_dir: null
+depth: 3
+detect_anomaly: false
+deterministic: false
+devices: null
+enable_checkpointing: true
+enable_model_summary: true
+enable_progress_bar: true
+fast_dev_run: false
+flush_logs_every_n_steps: null
+gpus: 1
+gradient_clip_algorithm: null
+gradient_clip_val: null
+ipus: null
+learning_rate: 0.001
+limit_predict_batches: null
+limit_test_batches: null
+limit_train_batches: null
+limit_val_batches: null
+log_dir: doc/tensorboard/
+log_every_n_steps: 50
+log_gpu_memory: null
+logger: true
+max_epochs: 10
+max_steps: -1
+max_time: null
+min_epochs: null
+min_steps: null
+move_metrics_to_cpu: false
+multiple_trainloader_mode: max_size_cycle
+name: Ragib_UNet
+num_nodes: 1
+num_processes: null
+num_sanity_val_steps: 2
+num_workers: 4
+overfit_batches: 0.0
+patch_size: 256
+plugins: null
+precision: 32
+prepare_data_per_node: null
+process_position: 0
+profiler: null
+progress_bar_refresh_rate: null
+reload_dataloaders_every_n_epochs: 0
+replace_sampler_ddp: true
+resume_from_checkpoint: null
+save_every_n_epochs: null
+stochastic_weight_avg: false
+strategy: null
+sync_batchnorm: true
+terminate_on_nan: null
+tpu_cores: null
+track_grad_norm: -1
+train_size: 1024
+val_check_interval: null
+val_size: 64
+weights_save_path: null
+weights_summary: top

doc/tensorboard/Ragib_UNet/version_11/hparams.yaml

+accelerator: null
+accumulate_grad_batches: null
+amp_backend: native
+amp_level: null
+auto_lr_find: false
+auto_scale_batch_size: false
+auto_select_gpus: false
+batch_size: 8
+benchmark: null
+check_val_every_n_epoch: 1
+checkpoint_callback: null
+ckpt_dir: tmp/ckpt/
+crop_size: 362
+default_root_dir: null
+depth: 3
+detect_anomaly: false
+deterministic: false
+devices: null
+enable_checkpointing: true
+enable_model_summary: true
+enable_progress_bar: true
+fast_dev_run: false
+flush_logs_every_n_steps: null
+gpus: 1
+gradient_clip_algorithm: null
+gradient_clip_val: null
+ipus: null
+learning_rate: 0.001
+limit_predict_batches: null
+limit_test_batches: null
+limit_train_batches: null
+limit_val_batches: null
+log_dir: doc/tensorboard/
+log_every_n_steps: 50
+log_gpu_memory: null
+logger: true
+max_epochs: 10
+max_steps: -1
+max_time: null
+min_epochs: null
+min_steps: null
+move_metrics_to_cpu: false
+multiple_trainloader_mode: max_size_cycle
+name: Ragib_UNet
+num_nodes: 1
+num_processes: null
+num_sanity_val_steps: 2
+num_workers: 4
+overfit_batches: 0.0
+patch_size: 256
+plugins: null
+precision: 32
+prepare_data_per_node: null
+process_position: 0
+profiler: null
+progress_bar_refresh_rate: null
+reload_dataloaders_every_n_epochs: 0
+replace_sampler_ddp: true
+resume_from_checkpoint: null
+save_every_n_epochs: null
+stochastic_weight_avg: false
+strategy: null
+sync_batchnorm: true
+terminate_on_nan: null
+tpu_cores: null
+track_grad_norm: -1
+train_size: 1024
+val_check_interval: null
+val_size: 64
+weights_save_path: null
+weights_summary: top

doc/tensorboard/Ragib_UNet/version_12/hparams.yaml

+accelerator: null
+accumulate_grad_batches: null
+amp_backend: native
+amp_level: null
+auto_lr_find: false
+auto_scale_batch_size: false
+auto_select_gpus: false
+batch_size: 8
+benchmark: null
+check_val_every_n_epoch: 1
+checkpoint_callback: null
+ckpt_dir: tmp/ckpt/
+crop_size: 362
+default_root_dir: null
+depth: 3
+detect_anomaly: false
+deterministic: false
+devices: null
+enable_checkpointing: true
+enable_model_summary: true
+enable_progress_bar: true
+fast_dev_run: false
+flush_logs_every_n_steps: null
+gpus: 1
+gradient_clip_algorithm: null
+gradient_clip_val: null
+ipus: null
+learning_rate: 0.001
+limit_predict_batches: null
+limit_test_batches: null
+limit_train_batches: null
+limit_val_batches: null
+log_dir: doc/tensorboard/
+log_every_n_steps: 50
+log_gpu_memory: null
+logger: true
+max_epochs: 10
+max_steps: -1
+max_time: null
+min_epochs: null
+min_steps: null
+move_metrics_to_cpu: false
+multiple_trainloader_mode: max_size_cycle
+name: Ragib_UNet
+num_nodes: 1
+num_processes: null
+num_sanity_val_steps: 2
+num_workers: 4
+overfit_batches: 0.0
+patch_size: 256
+plugins: null
+precision: 32
+prepare_data_per_node: null
+process_position: 0
+profiler: null
+progress_bar_refresh_rate: null
+reload_dataloaders_every_n_epochs: 0
+replace_sampler_ddp: true
+resume_from_checkpoint: null
+save_every_n_epochs: null
+stochastic_weight_avg: false
+strategy: null
+sync_batchnorm: true
+terminate_on_nan: null
+tpu_cores: null
+track_grad_norm: -1
+train_size: 1024
+val_check_interval: null
+val_size: 64
+weights_save_path: null
+weights_summary: top

doc/tensorboard/Ragib_UNet/version_13/hparams.yaml

+accelerator: null
+accumulate_grad_batches: null
+amp_backend: native
+amp_level: null
+auto_lr_find: false
+auto_scale_batch_size: false
+auto_select_gpus: false
+batch_size: 8
+benchmark: null
+check_val_every_n_epoch: 1
+checkpoint_callback: null
+ckpt_dir: tmp/ckpt/
+crop_size: 362
+default_root_dir: null
+depth: 3
+detect_anomaly: false
+deterministic: false
+devices: null
+enable_checkpointing: true
+enable_model_summary: true
+enable_progress_bar: true
+fast_dev_run: false
+flush_logs_every_n_steps: null
+gpus: 1
+gradient_clip_algorithm: null
+gradient_clip_val: null
+ipus: null
+learning_rate: 0.001
+limit_predict_batches: null
+limit_test_batches: null
+limit_train_batches: null
+limit_val_batches: null
+log_dir: doc/tensorboard/
+log_every_n_steps: 50
+log_gpu_memory: null
+logger: true
+max_epochs: 10
+max_steps: -1
+max_time: null
+min_epochs: null
+min_steps: null
+move_metrics_to_cpu: false
+multiple_trainloader_mode: max_size_cycle
+name: Ragib_UNet
+num_nodes: 1
+num_processes: null
+num_sanity_val_steps: 2
+num_workers: 4
+overfit_batches: 0.0
+patch_size: 256
+plugins: null
+precision: 32
+prepare_data_per_node: null
+process_position: 0
+profiler: null
+progress_bar_refresh_rate: null
+reload_dataloaders_every_n_epochs: 0
+replace_sampler_ddp: true
+resume_from_checkpoint: null
+save_every_n_epochs: null
+stochastic_weight_avg: false
+strategy: null
+sync_batchnorm: true
+terminate_on_nan: null
+tpu_cores: null
+track_grad_norm: -1
+train_size: 1024
+val_check_interval: null
+val_size: 64
+weights_save_path: null
+weights_summary: top

doc/tensorboard/Ragib_UNet/version_14/hparams.yaml

+accelerator: null
+accumulate_grad_batches: null
+amp_backend: native
+amp_level: null
+auto_lr_find: false
+auto_scale_batch_size: false
+auto_select_gpus: false
+batch_size: 8
+benchmark: null
+check_val_every_n_epoch: 1
+checkpoint_callback: null
+ckpt_dir: tmp/ckpt/
+crop_size: 362
+default_root_dir: null
+depth: 3
+detect_anomaly: false
+deterministic: false
+devices: null
+enable_checkpointing: true
+enable_model_summary: true
+enable_progress_bar: true
+fast_dev_run: false
+flush_logs_every_n_steps: null
+gpus: 1
+gradient_clip_algorithm: null
+gradient_clip_val: null
+ipus: null
+learning_rate: 0.001
+limit_predict_batches: null
+limit_test_batches: null
+limit_train_batches: null
+limit_val_batches: null
+log_dir: doc/tensorboard/
+log_every_n_steps: 50
+log_gpu_memory: null
+logger: true
+max_epochs: 10
+max_steps: -1
+max_time: null
+min_epochs: null
+min_steps: null
+move_metrics_to_cpu: false
+multiple_trainloader_mode: max_size_cycle
+name: Ragib_UNet
+num_nodes: 1
+num_processes: null
+num_sanity_val_steps: 2
+num_workers: 4
+overfit_batches: 0.0
+patch_size: 256
+plugins: null
+precision: 32
+prepare_data_per_node: null
+process_position: 0
+profiler: null
+progress_bar_refresh_rate: null
+reload_dataloaders_every_n_epochs: 0
+replace_sampler_ddp: true
+resume_from_checkpoint: null
+save_every_n_epochs: null
+stochastic_weight_avg: false
+strategy: null
+sync_batchnorm: true
+terminate_on_nan: null
+tpu_cores: null
+track_grad_norm: -1
+train_size: 1024
+val_check_interval: null
+val_size: 64
+weights_save_path: null
+weights_summary: top

doc/tensorboard/Ragib_UNet/version_15/hparams.yaml

+accelerator: null
+accumulate_grad_batches: null
+amp_backend: native
+amp_level: null
+auto_lr_find: false
+auto_scale_batch_size: false
+auto_select_gpus: false
+batch_size: 8
+benchmark: null
+check_val_every_n_epoch: 1
+checkpoint_callback: null
+ckpt_dir: tmp/ckpt/
+crop_size: 362
+default_root_dir: null
+depth: 3
+detect_anomaly: false
+deterministic: false
+devices: null
+enable_checkpointing: true
+enable_model_summary: true
+enable_progress_bar: true
+fast_dev_run: false
+flush_logs_every_n_steps: null
+gpus: 1
+gradient_clip_algorithm: null
+gradient_clip_val: null
+ipus: null
+learning_rate: 0.001
+limit_predict_batches: null
+limit_test_batches: null
+limit_train_batches: null
+limit_val_batches: null
+log_dir: doc/tensorboard/
+log_every_n_steps: 50
+log_gpu_memory: null
+logger: true
+max_epochs: 10
+max_steps: -1
+max_time: null
+min_epochs: null
+min_steps: null
+move_metrics_to_cpu: false
+multiple_trainloader_mode: max_size_cycle
+name: Ragib_UNet
+num_nodes: 1
+num_processes: null
+num_sanity_val_steps: 2
+num_workers: 4
+overfit_batches: 0.0
+patch_size: 256
+plugins: null
+precision: 32
+prepare_data_per_node: null
+process_position: 0
+profiler: null
+progress_bar_refresh_rate: null
+reload_dataloaders_every_n_epochs: 0
+replace_sampler_ddp: true
+resume_from_checkpoint: null
+save_every_n_epochs: null
+stochastic_weight_avg: false
+strategy: null
+sync_batchnorm: true
+terminate_on_nan: null
+tpu_cores: null
+track_grad_norm: -1
+train_size: 1024
+val_check_interval: null
+val_size: 64
+weights_save_path: null
+weights_summary: top

doc/tensorboard/Ragib_UNet/version_16/hparams.yaml

+accelerator: null
+accumulate_grad_batches: null
+amp_backend: native
+amp_level: null
+auto_lr_find: false
+auto_scale_batch_size: false
+auto_select_gpus: false
+batch_size: 8
+benchmark: null
+check_val_every_n_epoch: 1
+checkpoint_callback: null
+ckpt_dir: tmp/ckpt/
+crop_size: 362
+default_root_dir: null
+depth: 3
+detect_anomaly: false
+deterministic: false
+devices: null
+enable_checkpointing: true
+enable_model_summary: true
+enable_progress_bar: true
+fast_dev_run: false
+flush_logs_every_n_steps: null
+gpus: 1
+gradient_clip_algorithm: null
+gradient_clip_val: null
+ipus: null
+learning_rate: 0.001
+limit_predict_batches: null
+limit_test_batches: null
+limit_train_batches: null
+limit_val_batches: null
+log_dir: doc/tensorboard/
+log_every_n_steps: 50
+log_gpu_memory: null
+logger: true
+max_epochs: 10
+max_steps: -1
+max_time: null
+min_epochs: null
+min_steps: null
+move_metrics_to_cpu: false
+multiple_trainloader_mode: max_size_cycle
+name: Ragib_UNet
+num_nodes: 1
+num_processes: null
+num_sanity_val_steps: 2
+num_workers: 4
+overfit_batches: 0.0
+patch_size: 256
+plugins: null
+precision: 32
+prepare_data_per_node: null
+process_position: 0
+profiler: null
+progress_bar_refresh_rate: null
+reload_dataloaders_every_n_epochs: 0
+replace_sampler_ddp: true
+resume_from_checkpoint: null
+save_every_n_epochs: null
+stochastic_weight_avg: false
+strategy: null
+sync_batchnorm: true
+terminate_on_nan: null
+tpu_cores: null
+track_grad_norm: -1
+train_size: 1024
+val_check_interval: null
+val_size: 64
+weights_save_path: null
+weights_summary: top

doc/tensorboard/Ragib_UNet/version_17/hparams.yaml

+accelerator: null
+accumulate_grad_batches: null
+amp_backend: native
+amp_level: null
+auto_lr_find: false
+auto_scale_batch_size: false
+auto_select_gpus: false
+batch_size: 8
+benchmark: null
+check_val_every_n_epoch: 1
+checkpoint_callback: null
+ckpt_dir: tmp/ckpt/
+crop_size: 362
+default_root_dir: null
+depth: 3
+detect_anomaly: false
+deterministic: false
+devices: null
+enable_checkpointing: true
+enable_model_summary: true
+enable_progress_bar: true
+fast_dev_run: false
+flush_logs_every_n_steps: null
+gpus: 1
+gradient_clip_algorithm: null
+gradient_clip_val: null
+ipus: null
+learning_rate: 0.001
+limit_predict_batches: null
+limit_test_batches: null
+limit_train_batches: null
+limit_val_batches: null
+log_dir: doc/tensorboard/
+log_every_n_steps: 50
+log_gpu_memory: null
+logger: true
+max_epochs: 10
+max_steps: -1
+max_time: null
+min_epochs: null
+min_steps: null
+move_metrics_to_cpu: false
+multiple_trainloader_mode: max_size_cycle
+name: Ragib_UNet
+num_nodes: 1
+num_processes: null
+num_sanity_val_steps: 2
+num_workers: 4
+overfit_batches: 0.0
+patch_size: 256
+plugins: null
+precision: 32
+prepare_data_per_node: null
+process_position: 0
+profiler: null
+progress_bar_refresh_rate: null
+reload_dataloaders_every_n_epochs: 0
+replace_sampler_ddp: true
+resume_from_checkpoint: null
+save_every_n_epochs: null
+stochastic_weight_avg: false
+strategy: null
+sync_batchnorm: true
+terminate_on_nan: null
+tpu_cores: null
+track_grad_norm: -1
+train_size: 1024
+val_check_interval: null
+val_size: 64
+weights_save_path: null
+weights_summary: top

doc/tensorboard/Ragib_UNet/version_18/hparams.yaml

+accelerator: null
+accumulate_grad_batches: null
+amp_backend: native
+amp_level: null
+auto_lr_find: false
+auto_scale_batch_size: false
+auto_select_gpus: false
+batch_size: 8
+benchmark: null
+check_val_every_n_epoch: 1
+checkpoint_callback: null
+ckpt_dir: tmp/ckpt/
+crop_size: 362
+default_root_dir: null
+depth: 3
+detect_anomaly: false
+deterministic: false
+devices: null
+enable_checkpointing: true
+enable_model_summary: true
+enable_progress_bar: true
+fast_dev_run: false
+flush_logs_every_n_steps: null
+gpus: 1
+gradient_clip_algorithm: null
+gradient_clip_val: null
+ipus: null
+learning_rate: 0.001
+limit_predict_batches: null
+limit_test_batches: null
+limit_train_batches: null
+limit_val_batches: null
+log_dir: doc/tensorboard/
+log_every_n_steps: 50
+log_gpu_memory: null
+logger: true
+max_epochs: 10
+max_steps: -1
+max_time: null
+min_epochs: null
+min_steps: null
+move_metrics_to_cpu: false
+multiple_trainloader_mode: max_size_cycle
+name: Ragib_UNet
+num_nodes: 1
+num_processes: null
+num_sanity_val_steps: 2
+num_workers: 4
+overfit_batches: 0.0
+patch_size: 256
+plugins: null
+precision: 32
+prepare_data_per_node: null
+process_position: 0
+profiler: null
+progress_bar_refresh_rate: null
+reload_dataloaders_every_n_epochs: 0
+replace_sampler_ddp: true
+resume_from_checkpoint: null
+save_every_n_epochs: null
+stochastic_weight_avg: false
+strategy: null
+sync_batchnorm: true
+terminate_on_nan: null
+tpu_cores: null
+track_grad_norm: -1
+train_size: 1024
+val_check_interval: null
+val_size: 64
+weights_save_path: null
+weights_summary: top