Rework CLI

.gitignore

@@ -6,4 +6,6 @@ __pycache__
 tests/test_repo*
 .coverage
 dist
-.token
\ No newline at end of file
+.token
+tmp
+.ipynb_checkpoints
\ No newline at end of file

cadetrdm/cli_integration.py

@@ -3,11 +3,6 @@ import subprocess
 
 import click
 
-from .repositories import ProjectRepo, BaseRepo
-from .initialize_repo import initialize_repo as initialize_git_repo_implementation
-from .initialize_repo import clone as clone_implementation
-from .conda_env_utils import prepare_conda_env as prepare_conda_env_implementation
-
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
 
@@ -16,123 +11,178 @@ def cli():
     pass
 
 
-@cli.command()
-@click.option('--path_to_repo', default=None,
-              help='Path to folder for the repository. Optional.')
+@cli.command(help="Create an empty CADET-RDM repository or initialize over an existing git repo.")
+@click.option('--output_repo_name', default="output",
+              help='Name of the folder where the tracked output should be stored. Optional. Default: "output".')
+@click.option('--gitignore', default=None,
+              help='List of files to be added to the gitignore file. Optional.')
+@click.option('--gitattributes', default=None,
+              help='List of files to be added to the gitattributes file. Optional.')
+@click.argument('path_to_repo')
+def init(path_to_repo: str, output_repo_name: (str | bool) = "output", gitignore: list = None,
+         gitattributes: list = None,
+         output_repo_kwargs: dict = None):
+    from cadetrdm.initialize_repo import initialize_repo as initialize_git_repo_implementation
+    initialize_git_repo_implementation(path_to_repo, output_repo_name, gitignore,
+                                       gitattributes, output_repo_kwargs)
+
+
+@cli.command(help="Clone a repository into a new directory.")
 @click.argument('project_url')
-def clone(project_url, path_to_repo: str = None):
-    clone_implementation(project_url, path_to_repo)
+@click.argument('dest', required=False)
+def clone(project_url, dest: str = None):
+    from cadetrdm.initialize_repo import clone as clone_implementation
+    clone_implementation(project_url, dest)
 
 
-@cli.command()
-@click.option('--target_repo_location', default=None,
-              help='Path to folder for the repository. Optional.')
-@click.argument('source_repo_location')
-@click.argument('source_repo_branch')
-def import_remote_repo(source_repo_location, source_repo_branch, target_repo_location=None):
+@cli.command(name="log", help="Show commit logs.")
+def print_log():
+    from cadetrdm.repositories import BaseRepo
+
     repo = BaseRepo(".")
-    repo.import_remote_repo(source_repo_location=source_repo_location,
-                            source_repo_branch=source_repo_branch,
-                            target_repo_location=target_repo_location)
+    repo.print_log()
 
 
-@cli.command()
-@click.argument('url')
-@click.argument('namespace')
-@click.argument('name')
-def create_gitlab_remotes(url, namespace, name):
+@cli.command(help="Push all changes to the project and output repositories.")
+def push():
+    from cadetrdm.repositories import ProjectRepo
     repo = ProjectRepo(".")
-    repo.create_gitlab_remotes(url=url, namespace=namespace, name=name)
+    repo.push(push_all=True)
 
 
-@cli.command()
-@click.argument('namespace')
-@click.argument('name')
-def create_github_remotes(namespace, name):
+@cli.command(help="Record changes to the repository")
+@click.option("--message", "-m", help="commit message")
+@click.option("--all", "-a", is_flag=True, help="commit all changed files")
+def commit(message, all):
+    from cadetrdm.repositories import ProjectRepo
     repo = ProjectRepo(".")
-    repo.create_github_remotes(namespace=namespace, name=name)
-
-@cli.command()
-def verify_unchanged_cache():
-    repo = BaseRepo(".")
-    repo.verify_unchanged_cache()
+    repo.commit(message, all)
 
 
-@cli.command()
-@click.option('--re_load', default=False,
-              help='Re-load all data.')
-def fill_data_from_cadet_rdm_json(re_load=False):
-    repo = BaseRepo(".")
-    repo.fill_data_from_cadet_rdm_json(re_load=re_load)
+@cli.group(help="Execute commands and track the results.")
+def run():
+    pass
 
 
-@cli.command()
+@run.command(name="python")
 @click.argument('file_name')
 @click.argument('results_commit_message')
 def run_python_file(file_name, results_commit_message):
+    from cadetrdm.repositories import ProjectRepo
     repo = ProjectRepo(".")
     repo.enter_context()
     subprocess.run(["python", file_name])
     repo.exit_context(results_commit_message)
 
 
-@cli.command()
-@click.argument('command')
+@run.command(name="command")
+@click.argument('command', nargs=-1)
 @click.argument('results_commit_message')
 def run_command(command, results_commit_message):
+    from cadetrdm.repositories import ProjectRepo
     repo = ProjectRepo(".")
     repo.enter_context()
-    subprocess.run(shlex.split(command))
+    subprocess.run(command)
     repo.exit_context(results_commit_message)
 
 
-@cli.command()
-@click.option('--output_repo_name', default="output",
-              help='Name of the folder where the tracked output should be stored. Optional. Default: "output".')
-@click.option('--gitignore', default=None,
-              help='List of files to be added to the gitignore file. Optional.')
-@click.option('--gitattributes', default=None,
-              help='List of files to be added to the gitattributes file. Optional.')
-@click.argument('path_to_repo')
-def initialize_repo(path_to_repo: str, output_repo_name: (str | bool) = "output", gitignore: list = None,
-                    gitattributes: list = None,
-                    output_repo_kwargs: dict = None):
-    initialize_git_repo_implementation(path_to_repo, output_repo_name, gitignore,
-                                       gitattributes, output_repo_kwargs)
+@cli.group(help="Create, add, and manage remotes.")
+def remote():
+    pass
 
 
-@cli.command()
-@click.option("-p", '--path_to_repo', default=".",
-              help='Path to repository to which the remote is added. Default is cwd.')
+@remote.command(name="add", help="Add")
+@click.option('--name', '-n', default=None)
 @click.argument('remote_url')
-def add_remote_to_repo(remote_url: str, path_to_repo="."):
-    repo = BaseRepo(path_to_repo)
-    repo.add_remote(remote_url)
+def add_remote(name: str = None, remote_url: str = None):
+    from cadetrdm.repositories import BaseRepo
+    repo = BaseRepo(".")
+    repo.add_remote(remote_url=remote_url, remote_name=name)
     print("Done.")
 
 
-@cli.command()
-@click.argument('file_type')
-def add_filetype_to_lfs(file_type: str, ):
+@remote.command(name="create")
+@click.option("-p", '--path_to_repo', default=".",
+              help='Path to repository to which the remote is added. Default is cwd.')
+@click.argument('url')
+@click.argument('namespace')
+@click.argument('name')
+def create_remotes(url, namespace, name):
+    from cadetrdm.repositories import ProjectRepo
+    repo = ProjectRepo(".")
+    if "github" in url:
+        repo.create_github_remotes(namespace=namespace, name=name)
+    else:
+        repo.create_gitlab_remotes(url=url, namespace=namespace, name=name)
+
+
+@remote.command(name="list")
+def list_remotes():
+    from cadetrdm.repositories import BaseRepo
     repo = BaseRepo(".")
-    repo.add_filetype_to_lfs(file_type)
+    for _remote, url in zip(repo.remotes, repo.remote_urls):
+        print(_remote, ": ", url)
 
 
-@cli.command()
-@click.option('--url', default=None,
-              help='Url to the environment.yml file.')
-def prepare_conda_env(url):
-    prepare_conda_env_implementation(url)
+@cli.group(help="Manage large file storage settings.")
+def lfs():
+    pass
 
 
-@cli.command()
-def print_output_log():
-    # ToDo: test if Project or Output repo
-    repo = ProjectRepo(".")
-    repo.print_output_log()
+@lfs.command(name="add", help="Add a filetype to git lfs.")
+@click.argument('file_types', nargs=-1)
+def add_filetype_to_lfs(file_types: list, ):
+    from cadetrdm.repositories import BaseRepo
+    repo = BaseRepo(".")
+    for f_type in file_types:
+        repo.add_filetype_to_lfs(f_type)
 
 
-@cli.command(help="Push all changes to the project and output repositories.")
-def push():
+@cli.group(help="Manage data and input-data-repositories.")
+def data():
+    pass
+
+
+@data.command(name="import", help="Import a remote repository into a given location.")
+@click.argument('source_repo_location')
+@click.argument('source_repo_branch')
+@click.argument('target_repo_location', required=False)
+def import_remote_repo(source_repo_location, source_repo_branch, target_repo_location=None):
+    from cadetrdm.repositories import BaseRepo
+    repo = BaseRepo(".")
+    repo.import_remote_repo(source_repo_location=source_repo_location,
+                            source_repo_branch=source_repo_branch,
+                            target_repo_location=target_repo_location)
+
+
+@data.command(name="fetch", help="Fill data cache based on cadet-rdm.json.")
+@click.option('--re_load', is_flag=True,
+              help='Re-load all data.')
+def fill_data_from_cadet_rdm_json(re_load=False):
+    from cadetrdm.repositories import ProjectRepo
     repo = ProjectRepo(".")
-    repo.push(push_all=True)
+    repo.fill_data_from_cadet_rdm_json(re_load=re_load)
+
+
+@data.command(name="verify", help="Verify that cache is unchanged.")
+def verify_unchanged_cache():
+    from cadetrdm.repositories import BaseRepo
+    repo = BaseRepo(".")
+    repo.verify_unchanged_cache()
+
+
+@data.command(name="log", help="Print data logs.")
+def print_data_log():
+    from cadetrdm.repositories import ProjectRepo, BaseRepo, OutputRepo
+    import json
+
+    repo = BaseRepo(".")
+
+    with open(repo.data_json_path, "r") as handle:
+        rdm_data = json.load(handle)
+    if rdm_data["is_project_repo"]:
+        repo = ProjectRepo(".")
+        repo.print_output_log()
+    else:
+        repo = OutputRepo()
+        repo.print_data_log()

cadetrdm/repositories.py

@@ -116,7 +116,7 @@ class BaseRepo:
     def cache_json_path(self):
         return self.working_dir / ".cadet-rdm-cache.json"
 
-    def add_remote(self, remote_url, remote_name="origin"):
+    def add_remote(self, remote_url, remote_name=None):
         """
         Add a remote to the repository.
 
@@ -124,6 +124,8 @@ class BaseRepo:
         :param remote_name:
         :return:
         """
+        if remote_name is None:
+            remote_name = "origin"
         self._git_repo.create_remote(remote_name, url=remote_url)
         with open(self.data_json_path, "r") as handle:
             rdm_data = json.load(handle)
@@ -366,9 +368,9 @@ class BaseRepo:
     def reset_hard_to_head(self, force_entry=False):
         if not force_entry:
             proceed = wait_for_user(f'The output directory contains the following uncommitted changes:\n'
-                                f'{self.untracked_files + self.changed_files}\n'
-                                f' These will be lost if you continue\n'
-                                f'Proceed?')
+                                    f'{self.untracked_files + self.changed_files}\n'
+                                    f' These will be lost if you continue\n'
+                                    f'Proceed?')
         else:
             proceed = True
         if not proceed:
@@ -685,25 +687,7 @@ class ProjectRepo(BaseRepo):
         self._output_repo._git.checkout(self._most_recent_branch)
 
     def print_output_log(self):
-        def insert_newlines(string, every=30):
-            lines = []
-            for i in range(0, len(string), every):
-                lines.append(string[i:i + every])
-            return '\n'.join(lines)
-
-        self.output_repo.checkout("master")
-
-        tsv_filepath = self.working_dir / self._output_folder / "log.tsv"
-
-        with open(tsv_filepath, "r") as filehandle:
-            lines = filehandle.readlines()
-
-        line_array = [line.replace("\n", "").split("\t") for line in lines]
-
-        # Print
-        print(tabulate(line_array[1:], headers=line_array[0]))
-
-        self.output_repo.checkout(self.output_repo._most_recent_branch)
+        self.output_repo.print_data_log()
 
     def fill_data_from_cadet_rdm_json(self, re_load=False):
         """
@@ -735,7 +719,6 @@ class ProjectRepo(BaseRepo):
                     source_repo_location=repo_info["source_repo_location"],
                     source_repo_branch=repo_info["branch_name"])
 
-
     def convert_csv_to_tsv_if_necessary(self):
         """
         If not tsv log is found AND a csv log is found, convert the csv to tsv.
@@ -1086,9 +1069,22 @@ class ProjectRepo(BaseRepo):
             self.exit_context(message=results_commit_message)
 
 
-
 class OutputRepo(BaseRepo):
-    pass
+
+    def print_data_log(self):
+        self.checkout("master")
+
+        tsv_filepath = self.working_dir / "log.tsv"
+
+        with open(tsv_filepath, "r") as filehandle:
+            lines = filehandle.readlines()
+
+        line_array = [line.replace("\n", "").split("\t") for line in lines]
+
+        # Print
+        print(tabulate(line_array[1:], headers=line_array[0]))
+
+        self.checkout(self._most_recent_branch)
 
 
 class JupyterInterfaceRepo(ProjectRepo):
@@ -1104,8 +1100,8 @@ class JupyterInterfaceRepo(ProjectRepo):
     def commit_nb_output(self, notebook_path: str, results_commit_message: str,
                          force_rerun=True, timeout=600, conversion_formats: list = None):
         if "nbconvert_call" in sys.argv:
-            return 
-        # This is reached in the first call of this function
+            return
+            # This is reached in the first call of this function
         if not Path(notebook_path).is_absolute():
             notebook_path = self.working_dir / notebook_path
 

setup.cfg

@@ -29,7 +29,7 @@ include_package_data = True
 
 [options.entry_points]
 console_scripts =
-    cadet-rdm = cadetrdm.cli_integration:cli
+    rdm = cadetrdm.cli_integration:cli
 
 [options.extras_require]
 testing =

tests/test_cli.py

+from pathlib import Path
+import random
+import os
+
+from click.testing import CliRunner
+
+from cadetrdm.cli_integration import cli
+from cadetrdm.io_utils import delete_path
+
+runner = CliRunner()
+if os.path.exists("test_repo_cli"):
+    delete_path("test_repo_cli")
+
+os.makedirs("test_repo_cli")
+os.chdir("test_repo_cli")
+
+
+def modify_code(path_to_repo):
+    # Add changes to the project code
+    random_number = random.randint(0, 265)
+    filepath = Path(path_to_repo) / f"print_random_number.py"
+    with open(filepath, "w") as file:
+        file.write(
+            f"print({random_number})\n"
+            'with open("output/data.txt", "w") as handle:\n'
+            f"    handle.write({random_number})\n"
+        )
+
+
+def test_01_initialize_repo():
+    result = runner.invoke(cli, ["init", "."])
+    print(result.output)
+    assert result.exit_code == 0
+
+
+def test_02_add_remote():
+    result = runner.invoke(cli, ["remote", "add", "https://jugit.fz-juelich.de/r.jaepel/API_test_project"])
+    print(result.output)
+    assert result.exit_code == 0
+    os.chdir("output")
+    result = runner.invoke(cli, ["remote", "add", "https://jugit.fz-juelich.de/r.jaepel/API_test_project_output"])
+    print(result.output)
+    os.chdir("..")
+    assert result.exit_code == 0
+
+
+def test_02b_clone():
+    os.chdir("..")
+    if os.path.exists("test_repo_cli_cloned"):
+        delete_path("test_repo_cli_cloned")
+    result = runner.invoke(cli, ["clone", "test_repo_cli", "test_repo_cli_cloned"])
+    print(result.output)
+    os.chdir("test_repo_cli")
+    assert result.exit_code == 0
+
+
+def test_03_commit_results_with_uncommited_code_changes():
+    modify_code(".")
+
+    result = runner.invoke(cli, ["run", "python", "print_random_number.py",
+                                 "create data"])
+    print(result.output)
+    assert result.exit_code != 0
+
+
+def test_04_commit_code():
+    modify_code(".")
+
+    result = runner.invoke(cli, ["commit", "-m", "add code", "-a"])
+    print(result.output)
+    assert result.exit_code == 0
+
+
+# def test_05_commit_results():
+#     result = runner.invoke(cli, ["commit", "-m", "add code", "-a"])
+#     print(result.output)
+#     assert result.exit_code == 0
+#     result = runner.invoke(cli, ["run", "python", "print_random_number.py",
+#                                  "create data"])
+#     print(result.output)
+#     assert result.exit_code == 0
+#
+#
+# def test_05b_execute_command():
+#     result = runner.invoke(cli, ["commit", "-m", "add code", "-a"])
+#     print(result.output)
+#     assert result.exit_code == 0
+#     result = runner.invoke(cli, ["run", "command", "python", "print_random_number.py",
+#                                  "create data"])
+#     print(result.output)
+#     assert result.exit_code == 0
+
+
+def test_06_print_log():
+    result = runner.invoke(cli, ["log"])
+    print(result.output)
+    assert result.exit_code == 0
+
+
+def test_07_lfs_add():
+    result = runner.invoke(cli, ["lfs", "add", "pptx"])
+    print(result.output)
+    assert result.exit_code == 0
+
+
+def test_08_data_import():
+    result = runner.invoke(cli,
+                           ["data", "import", "https://github.com/ronald-jaepel/workshop_demo_output",
+                            "2023-12-19_10-43-15_output_from_master_86541bc", "imported/repo/data"])
+    print(result.output)
+    assert result.exit_code == 0
+
+# def test_09_data_verify():
+#     with open()
+#     result = runner.invoke(cli, ["data", "verify"])