Add ability to push and to initialize from an existing remote

cadetrdm/__init__.py

 from .repositories import ProjectRepo
-from .initialize_repo import initialize_git_repo
+from .initialize_repo import initialize_git_repo, initialize_from_remote
 from .conda_env_utils import prepare_conda_env

cadetrdm/initialize_repo.py

 import os
+import json
 
 import click
 
+try:
+    import git
+except ImportError:
+    # Adding this hint to save users the confusion of trying $pip install git
+    raise ImportError("No module named git, please install the gitpython package")
+
 from cadetrdm.repositories import ProjectRepo, ResultsRepo
+from cadetrdm.utils import ssh_url_to_http_url
 
 
 def add_linebreaks(input_list):
@@ -131,13 +139,63 @@ def initialize_git_repo(path_to_repo: str, output_repo_name: (str | bool) = "out
 
     if output_repo_name:
         # This means we are in the project repo and should now initialize the output_repo
+        create_readme()
         initialize_git_repo(output_repo_name, output_repo_name=False, **output_repo_kwargs)
         # This instance of ProjectRepo is therefore the project repo
         repo = ProjectRepo(".", output_folder=output_repo_name)
     else:
+        create_output_readme()
         # If output_repo_name is False we are in the output_repo and should finish by committing the changes
         repo = ResultsRepo(".")
 
     repo.commit("initial commit")
 
     os.chdir(starting_directory)
+
+
+def create_readme():
+    readme_lines = ["# Project repo", "Your code goes in this repo.", "Please add a description here including: ",
+                    "- authors", "- project", "- things we will find interesting later", "", "",
+                    "The output repository can be found at:",
+                    "[output_repo]() (not actually set yet because no remote has been configured at this moment"]
+    write_lines_to_file("README.md", readme_lines, open_type="w")
+
+
+def create_output_readme():
+    readme_lines = ["# Output repo", "Your results will be stored here.", "Please add a description here including: ",
+                    "- authors", "- project", "- things we will find interesting later", "", "",
+                    "The project repository can be found at:",
+                    "[project_repo]() (not actually set yet because no remote has been configured at this moment"]
+    write_lines_to_file("README.md", readme_lines, open_type="w")
+
+
+@click.command()
+@click.option('--path_to_repo', default=None,
+              help='Path to folder for the repository. Optional.')
+@click.argument('project_url')
+def initialize_from_remote_cli(project_url, path_to_repo: str = None):
+    initialize_from_remote(project_url, path_to_repo)
+
+
+def initialize_from_remote(project_url, path_to_repo: str = None):
+    if path_to_repo is None:
+        path_to_repo = project_url.split("/")[-1]
+    print(f"Cloning {project_url} into {path_to_repo}")
+    git.Repo.clone_from(project_url, path_to_repo)
+
+    # Clone output repo from remotes
+    json_path = os.path.join(path_to_repo, "output_remotes.json")
+    with open(json_path, "r") as file_handle:
+        meta_dict = json.load(file_handle)
+
+    output_folder_name = os.path.join(path_to_repo, meta_dict["output_foldername"])
+    ssh_remotes = list(meta_dict["output_remotes"].values())
+    http_remotes = [ssh_url_to_http_url(url) for url in ssh_remotes]
+    for output_remote in ssh_remotes + http_remotes:
+        try:
+            print(f"Attempting to clone {output_remote} into {output_folder_name}")
+            git.Repo.clone_from(output_remote, output_folder_name)
+        except Exception as e:
+            print(e)
+        else:
+            break

cadetrdm/repositories.py

@@ -3,7 +3,6 @@ import json
 from datetime import datetime
 import shutil
 import contextlib
-import click
 
 try:
     import git
@@ -11,6 +10,8 @@ except ImportError:
     # Adding this hint to save users the confusion of trying $pip install git
     raise ImportError("No module named git, please install the gitpython package")
 
+from cadetrdm.utils import ssh_url_to_http_url
+
 
 class BaseRepo:
     def __init__(self, repository_path=None, search_parent_directories=False, *args, **kwargs):
@@ -255,6 +256,32 @@ class BaseRepo:
         if self.exist_uncomitted_changes:
             raise RuntimeError(f"Found uncommitted changes in the repository {self.working_dir}.")
 
+    def add_list_of_remotes_in_readme_file(self, repo_identifier: str, remotes_url_list: list):
+        if len(remotes_url_list) > 0:
+            remotes_url_list_http = [ssh_url_to_http_url(remote)
+                                     for remote in remotes_url_list]
+            output_link_line = " and ".join(f"[{repo_identifier}]({output_repo_remote})"
+                                            for output_repo_remote in remotes_url_list_http) + "\n"
+
+            readme_filepath = os.path.join(self.working_dir, "README.md")
+            with open(readme_filepath, "r") as file_handle:
+                filelines = file_handle.readlines()
+                filelines_giving_output_repo = [i for i in range(len(filelines))
+                                                if filelines[i].startswith(f"[{repo_identifier}](")]
+                if len(filelines_giving_output_repo) == 1:
+                    line_to_be_modified = filelines_giving_output_repo[0]
+                    filelines[line_to_be_modified] = output_link_line
+                elif len(filelines_giving_output_repo) == 0:
+                    filelines.append("The output repo can be found at:\n")
+                    filelines.append(output_link_line)
+                else:
+                    raise RuntimeError(f"Multiple lines in the README.md at {readme_filepath}"
+                                       f" link to the {repo_identifier}. "
+                                       "Can't automatically update the link.")
+
+            with open(readme_filepath, "w") as file_handle:
+                file_handle.writelines(filelines)
+
 
 class ProjectRepo(BaseRepo):
     def __init__(self, repository_path=None, output_folder=None,
@@ -390,6 +417,27 @@ class ProjectRepo(BaseRepo):
                 target_file_path
             )
 
+    def commit(self, message: str, add_all=True):
+        """
+        Commit current state of the repository.
+
+        :param message:
+            Commit message
+        :param add_all:
+            Option to add all changed and new files to git automatically.
+        """
+
+        output_repo_remotes = [str(remote.url) for remote in self.output_repo.remotes]
+        self.add_list_of_remotes_in_readme_file("output_repo", output_repo_remotes)
+
+        output_json_filepath = os.path.join(self.working_dir, "output_remotes.json")
+        with open(output_json_filepath, "w") as file_handle:
+            remotes_dict = {remote.name: str(remote.url) for remote in self.output_repo.remotes}
+            json_dict = {"output_folder_name": self.output_folder, "output_remotes": remotes_dict}
+            json.dump(json_dict, file_handle, indent=2)
+
+        super().commit(message=message, add_all=add_all)
+
     def load_external_repository(self, url, branch=None, commit=None, name=None, path=None, ):
         """
         Load an external git repository as a git submodule into this repository.
@@ -492,6 +540,13 @@ class ProjectRepo(BaseRepo):
         output_repo.delete_active_branch_if_branch_is_empty()
 
         new_branch_name = self.get_new_output_branch_name()
+
+        # update urls in master branch of output_repo
+        output_repo._git.checkout("master")
+        project_repo_remotes = [str(remote.url) for remote in self.remotes]
+        output_repo.add_list_of_remotes_in_readme_file("project_repo", project_repo_remotes)
+        output_repo.commit("Update urls")
+
         output_repo.prepare_new_branch(new_branch_name)
         return new_branch_name
 
@@ -511,14 +566,17 @@ class ProjectRepo(BaseRepo):
 
         print("Completed computations, commiting results")
         self.output_repo.add(".")
-        commit_return = self.output_repo._git.commit("-m", message)
-
-        print("\n" + commit_return + "\n")
-
-        self.update_output_master_logs()
-        self.remove_cached_files()
-        self._is_in_context_manager = False
-        self._on_context_enter_commit_hash = None
+        try:
+            # This has to be from ._git.commit because this way it raises an error if no results have been written.
+            commit_return = self.output_repo._git.commit("-m", message)
+            self.update_output_master_logs()
+            print("\n" + commit_return + "\n")
+        except git.exc.GitCommandError as e:
+            raise e
+        finally:
+            self.remove_cached_files()
+            self._is_in_context_manager = False
+            self._on_context_enter_commit_hash = None
 
     @contextlib.contextmanager
     def track_results(self, results_commit_message: str, debug=False):

cadetrdm/utils.py

+def ssh_url_to_http_url(url):
+    url.replace(":", "/").replace("git@", "https://").replace(".git", "")
+    return url

setup.cfg

@@ -27,6 +27,7 @@ include_package_data = True
 [options.entry_points]
 console_scripts =
     initialize_git_repo = cadetrdm.initialize_repo:initialize_git_repo_cli
+    initialize_from_remote = cadetrdm.initialize_repo:initialize_from_remote_cli
     prepare_conda_env = cadetrdm.conda_env_utils:prepare_conda_env_cli
 
 [options.extras_require]

tests/test_git_adapter.py

@@ -7,7 +7,7 @@ import pytest
 import git
 import numpy as np
 
-from cadetrdm import initialize_git_repo, ProjectRepo
+from cadetrdm import initialize_git_repo, ProjectRepo, initialize_from_remote
 
 
 @pytest.fixture(scope="module")
@@ -60,12 +60,13 @@ def example_generate_results_array(path_to_repo, output_folder):
     return results_array
 
 
-def try_initialize_git_repo(path_to_repo):
-    def try_init_gitpython_repo(repo_path):
-        os.path.exists(repo_path)
-        git.Repo(repo_path)
-        return True
+def try_init_gitpython_repo(repo_path):
+    os.path.exists(repo_path)
+    git.Repo(repo_path)
+    return True
+
 
+def try_initialize_git_repo(path_to_repo):
     if os.path.exists(path_to_repo):
         remove_dir(path_to_repo)
 
@@ -140,10 +141,19 @@ def try_add_remote(path_to_repo):
     assert "origin" in repo._git_repo.remotes
 
 
+def try_initialize_from_remote():
+    if os.path.exists("test_repo_from_remote"):
+        remove_dir("test_repo_from_remote")
+    initialize_from_remote("https://jugit.fz-juelich.de/IBG-1/ModSim/cadet/rdm-examples-fraunhofer-ime-aachen",
+                           "test_repo_from_remote")
+    assert try_init_gitpython_repo("test_repo_from_remote")
+
+
 def test_cadet_rdm(path_to_repo):
     # because these depend on one-another and there is no native support afaik for sequential tests
     # these tests are called sequentially here as try_ functions.
     try_initialize_git_repo(path_to_repo)
+    try_initialize_from_remote()
 
     try_add_remote(path_to_repo)
     # try_add_submodule(path_to_repo)