Move dumping of python env into RESULTS commit instead of PROJECT commit

cadetrdm/__init__.py

-from .utils import ProjectRepo
+from .repositories import ProjectRepo
 from .initialize_repo import initialize_git_repo
 from .conda_env_utils import prepare_conda_env

cadetrdm/initialize_repo.py

@@ -2,7 +2,7 @@ import os
 
 import click
 
-from cadetrdm.utils import ProjectRepo, ResultsRepo
+from cadetrdm.repositories import ProjectRepo, ResultsRepo
 
 
 def add_linebreaks(input_list):
@@ -127,7 +127,7 @@ def initialize_git_repo(path_to_repo: str, output_repo_name: (str | bool) = "out
     write_lines_to_file(path=".gitignore", lines=gitignore)
 
     if output_repo_kwargs is None:
-        output_repo_kwargs = {"gitattributes": ["log.csv merge=union"]}
+        output_repo_kwargs = {"gitattributes": ["logs/log.csv merge=union"]}
 
     if output_repo_name:
         # This means we are in the project repo and should now initialize the output_repo

cadetrdm/utils.py → cadetrdm/repositories.py

@@ -144,21 +144,24 @@ class BaseRepo:
     def exist_uncomitted_changes(self):
         return len(self._git.status("--porcelain")) > 0
 
-    def update_package_list(self):
+    def dump_package_list(self, target_folder):
         """
         Use "conda env export" and "pip freeze" to create environment.yml and pip_requirements.txt files.
         """
-        repo_path = self.working_dir
+        if target_folder is not None:
+            dump_path = target_folder
+        else:
+            dump_path = self.working_dir
         print("Dumping conda environment.yml, this might take a moment.")
-        os.system(f"conda env export > {repo_path}/conda_environment.yml")
+        os.system(f"conda env export > {dump_path}/conda_environment.yml")
         print("Dumping conda independent environment.yml, this might take a moment.")
-        os.system(f"conda env export --from-history > {repo_path}/conda_independent_environment.yml")
+        os.system(f"conda env export --from-history > {dump_path}/conda_independent_environment.yml")
         print("Dumping pip requirements.txt.")
-        os.system(f"pip freeze > {repo_path}/pip_requirements.txt")
+        os.system(f"pip freeze > {dump_path}/pip_requirements.txt")
         print("Dumping pip independent requirements.txt.")
-        os.system(f"pip list --not-required --format freeze > {repo_path}/pip_independent_requirements.txt")
+        os.system(f"pip list --not-required --format freeze > {dump_path}/pip_independent_requirements.txt")
 
-    def commit(self, message: str, add_all=True, update_packages=False):
+    def commit(self, message: str, add_all=True):
         """
         Commit current state of the repository.
 
@@ -166,16 +169,12 @@ class BaseRepo:
             Commit message
         :param add_all:
             Option to add all changed and new files to git automatically.
-        :param update_packages:
-            Option to automatically dump the python environment information into environment.yml files.
         """
         if not self.exist_uncomitted_changes:
             print(f"No changes to commit in repo {self.working_dir}")
             return
 
         print(f"Commiting changes to repo {self.working_dir}")
-        if update_packages:
-            self.update_package_list()
         if add_all:
             self.add(".")
         commit_return = self._git.commit("-m", message)
@@ -331,10 +330,14 @@ class ProjectRepo(BaseRepo):
 
         self._output_repo._git.checkout("master")
 
-        json_filepath = os.path.join(self.working_dir, self._output_folder, f"{output_branch_name}.json")
+        logs_folderpath = os.path.join(self.working_dir, self._output_folder, "logs")
+        if not os.path.exists(logs_folderpath):
+            os.makedirs(logs_folderpath)
+
+        json_filepath = os.path.join(logs_folderpath, f"{output_branch_name}.json")
         # note: if filename of "log.csv" is changed,
         #  this also has to be changed in the gitattributes of the init repo func
-        csv_filepath = os.path.join(self.working_dir, self._output_folder, "log.csv")
+        csv_filepath = os.path.join(logs_folderpath, "log.csv")
 
         meta_info_dict = {"Output repo branch": output_branch_name,
                           "Output repo commit hash": output_repo_hash,
@@ -361,6 +364,8 @@ class ProjectRepo(BaseRepo):
         with open(csv_filepath, "a") as f:
             f.write(csv_data + "\n")
 
+        self.dump_package_list(logs_folderpath)
+
         self._output_repo.add(".")
         self._output_repo._git.commit("-m", output_branch_name)
 
@@ -440,28 +445,6 @@ class ProjectRepo(BaseRepo):
 
         return target_filepath
 
-    @contextlib.contextmanager
-    def load_previous_result_file(self, branch_name, file_path, *args, **kwargs):
-        """
-        Context manager around load_previous_result_file that directly opens a handle to the loaded file.
-        :param branch_name:
-            Name of the branch of the output repository in which the results are stored
-        :param file_path:
-            Relative path within the output repository to the file you wish to load.
-        :param args:
-            Args to be handed to the open() function
-        :param kwargs:
-            kwargs to be handed to the open() function
-        :return:
-            Handle to the copied file.
-        """
-        cached_filepath = self.load_previous_result_file(branch_name, file_path)
-        file_handle = open(cached_filepath, *args, **kwargs)
-        try:
-            yield file_handle
-        finally:
-            file_handle.close()
-
     def remove_cached_files(self):
         """
         Delete all previously cached results.

tests/test_git_adapter.py

@@ -80,7 +80,7 @@ def try_commit_code(path_to_repo):
     current_commit_number = count_commit_number(repo)
 
     modify_code(path_to_repo)
-    repo.commit("add code to print random number", add_all=True, update_packages=False)
+    repo.commit("add code to print random number", add_all=True)
 
     updated_commit_number = count_commit_number(repo)
     assert current_commit_number + 1 == updated_commit_number
@@ -91,6 +91,7 @@ def try_add_submodule(path_to_repo):
     submodule_path = repo.load_external_repository("https://jugit.fz-juelich.de/IBG-1/ModSim/cadet/git_lfs_data_1")
     assert os.path.exists(submodule_path)
 
+
 def try_commit_code_without_code_changes(path_to_repo):
     repo = ProjectRepo(path_to_repo)
     current_commit_number = count_commit_number(repo)
@@ -115,7 +116,7 @@ def try_commit_results_with_uncommitted_code_changes(path_to_repo):
     with pytest.raises(Exception):
         with repo.track_results(results_commit_message="Add array"):
             example_generate_results_array(path_to_repo, output_folder=repo._output_folder)
-    repo.commit("add code to print random number", add_all=True, update_packages=False)
+    repo.commit("add code to print random number", add_all=True)
 
 
 def try_load_previous_result(path_to_repo, branch_name):