From b15c114379bc26fb990de7014eac0a64d878807e Mon Sep 17 00:00:00 2001
From: "r.jaepel" <r.jaepel@fz-juelich.de>
Date: Tue, 16 Jan 2024 11:01:49 +0100
Subject: [PATCH] Rework CLI
---
.gitignore | 4 +-
cadetrdm/cli_integration.py | 212 ++++++++++++++++++++++--------------
cadetrdm/repositories.py | 52 ++++-----
setup.cfg | 2 +-
tests/test_cli.py | 115 +++++++++++++++++++
5 files changed, 274 insertions(+), 111 deletions(-)
create mode 100644 tests/test_cli.py
diff --git a/.gitignore b/.gitignore
index 37eb7a0..c44a98c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,4 +6,6 @@ __pycache__
tests/test_repo*
.coverage
dist
-.token
\ No newline at end of file
+.token
+tmp
+.ipynb_checkpoints
\ No newline at end of file
diff --git a/cadetrdm/cli_integration.py b/cadetrdm/cli_integration.py
index 65b1ebb..e3b936e 100644
--- a/cadetrdm/cli_integration.py
+++ b/cadetrdm/cli_integration.py
@@ -3,11 +3,6 @@ import subprocess
import click
-from .repositories import ProjectRepo, BaseRepo
-from .initialize_repo import initialize_repo as initialize_git_repo_implementation
-from .initialize_repo import clone as clone_implementation
-from .conda_env_utils import prepare_conda_env as prepare_conda_env_implementation
-
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
@@ -16,123 +11,178 @@ def cli():
pass
-@cli.command()
-@click.option('--path_to_repo', default=None,
- help='Path to folder for the repository. Optional.')
+@cli.command(help="Create an empty CADET-RDM repository or initialize over an existing git repo.")
+@click.option('--output_repo_name', default="output",
+ help='Name of the folder where the tracked output should be stored. Optional. Default: "output".')
+@click.option('--gitignore', default=None,
+ help='List of files to be added to the gitignore file. Optional.')
+@click.option('--gitattributes', default=None,
+ help='List of files to be added to the gitattributes file. Optional.')
+@click.argument('path_to_repo')
+def init(path_to_repo: str, output_repo_name: (str | bool) = "output", gitignore: list = None,
+ gitattributes: list = None,
+ output_repo_kwargs: dict = None):
+ from cadetrdm.initialize_repo import initialize_repo as initialize_git_repo_implementation
+ initialize_git_repo_implementation(path_to_repo, output_repo_name, gitignore,
+ gitattributes, output_repo_kwargs)
+
+
+@cli.command(help="Clone a repository into a new directory.")
@click.argument('project_url')
-def clone(project_url, path_to_repo: str = None):
- clone_implementation(project_url, path_to_repo)
+@click.argument('dest', required=False)
+def clone(project_url, dest: str = None):
+ from cadetrdm.initialize_repo import clone as clone_implementation
+ clone_implementation(project_url, dest)
-@cli.command()
-@click.option('--target_repo_location', default=None,
- help='Path to folder for the repository. Optional.')
-@click.argument('source_repo_location')
-@click.argument('source_repo_branch')
-def import_remote_repo(source_repo_location, source_repo_branch, target_repo_location=None):
+@cli.command(name="log", help="Show commit logs.")
+def print_log():
+ from cadetrdm.repositories import BaseRepo
+
repo = BaseRepo(".")
- repo.import_remote_repo(source_repo_location=source_repo_location,
- source_repo_branch=source_repo_branch,
- target_repo_location=target_repo_location)
+ repo.print_log()
-@cli.command()
-@click.argument('url')
-@click.argument('namespace')
-@click.argument('name')
-def create_gitlab_remotes(url, namespace, name):
+@cli.command(help="Push all changes to the project and output repositories.")
+def push():
+ from cadetrdm.repositories import ProjectRepo
repo = ProjectRepo(".")
- repo.create_gitlab_remotes(url=url, namespace=namespace, name=name)
+ repo.push(push_all=True)
-@cli.command()
-@click.argument('namespace')
-@click.argument('name')
-def create_github_remotes(namespace, name):
+@cli.command(help="Record changes to the repository")
+@click.option("--message", "-m", help="commit message")
+@click.option("--all", "-a", is_flag=True, help="commit all changed files")
+def commit(message, all):
+ from cadetrdm.repositories import ProjectRepo
repo = ProjectRepo(".")
- repo.create_github_remotes(namespace=namespace, name=name)
-
-@cli.command()
-def verify_unchanged_cache():
- repo = BaseRepo(".")
- repo.verify_unchanged_cache()
+ repo.commit(message, all)
-@cli.command()
-@click.option('--re_load', default=False,
- help='Re-load all data.')
-def fill_data_from_cadet_rdm_json(re_load=False):
- repo = BaseRepo(".")
- repo.fill_data_from_cadet_rdm_json(re_load=re_load)
+@cli.group(help="Execute commands and track the results.")
+def run():
+ pass
-@cli.command()
+@run.command(name="python")
@click.argument('file_name')
@click.argument('results_commit_message')
def run_python_file(file_name, results_commit_message):
+ from cadetrdm.repositories import ProjectRepo
repo = ProjectRepo(".")
repo.enter_context()
subprocess.run(["python", file_name])
repo.exit_context(results_commit_message)
-@cli.command()
-@click.argument('command')
+@run.command(name="command")
+@click.argument('command', nargs=-1)
@click.argument('results_commit_message')
def run_command(command, results_commit_message):
+ from cadetrdm.repositories import ProjectRepo
repo = ProjectRepo(".")
repo.enter_context()
- subprocess.run(shlex.split(command))
+ subprocess.run(command)
repo.exit_context(results_commit_message)
-@cli.command()
-@click.option('--output_repo_name', default="output",
- help='Name of the folder where the tracked output should be stored. Optional. Default: "output".')
-@click.option('--gitignore', default=None,
- help='List of files to be added to the gitignore file. Optional.')
-@click.option('--gitattributes', default=None,
- help='List of files to be added to the gitattributes file. Optional.')
-@click.argument('path_to_repo')
-def initialize_repo(path_to_repo: str, output_repo_name: (str | bool) = "output", gitignore: list = None,
- gitattributes: list = None,
- output_repo_kwargs: dict = None):
- initialize_git_repo_implementation(path_to_repo, output_repo_name, gitignore,
- gitattributes, output_repo_kwargs)
+@cli.group(help="Create, add, and manage remotes.")
+def remote():
+ pass
-@cli.command()
-@click.option("-p", '--path_to_repo', default=".",
- help='Path to repository to which the remote is added. Default is cwd.')
+@remote.command(name="add", help="Add")
+@click.option('--name', '-n', default=None)
@click.argument('remote_url')
-def add_remote_to_repo(remote_url: str, path_to_repo="."):
- repo = BaseRepo(path_to_repo)
- repo.add_remote(remote_url)
+def add_remote(name: str = None, remote_url: str = None):
+ from cadetrdm.repositories import BaseRepo
+ repo = BaseRepo(".")
+ repo.add_remote(remote_url=remote_url, remote_name=name)
print("Done.")
-@cli.command()
-@click.argument('file_type')
-def add_filetype_to_lfs(file_type: str, ):
+@remote.command(name="create")
+@click.option("-p", '--path_to_repo', default=".",
+ help='Path to repository to which the remote is added. Default is cwd.')
+@click.argument('url')
+@click.argument('namespace')
+@click.argument('name')
+def create_remotes(url, namespace, name):
+ from cadetrdm.repositories import ProjectRepo
+ repo = ProjectRepo(".")
+ if "github" in url:
+ repo.create_github_remotes(namespace=namespace, name=name)
+ else:
+ repo.create_gitlab_remotes(url=url, namespace=namespace, name=name)
+
+
+@remote.command(name="list")
+def list_remotes():
+ from cadetrdm.repositories import BaseRepo
repo = BaseRepo(".")
- repo.add_filetype_to_lfs(file_type)
+ for _remote, url in zip(repo.remotes, repo.remote_urls):
+ print(_remote, ": ", url)
-@cli.command()
-@click.option('--url', default=None,
- help='Url to the environment.yml file.')
-def prepare_conda_env(url):
- prepare_conda_env_implementation(url)
+@cli.group(help="Manage large file storage settings.")
+def lfs():
+ pass
-@cli.command()
-def print_output_log():
- # ToDo: test if Project or Output repo
- repo = ProjectRepo(".")
- repo.print_output_log()
+@lfs.command(name="add", help="Add a filetype to git lfs.")
+@click.argument('file_types', nargs=-1)
+def add_filetype_to_lfs(file_types: list, ):
+ from cadetrdm.repositories import BaseRepo
+ repo = BaseRepo(".")
+ for f_type in file_types:
+ repo.add_filetype_to_lfs(f_type)
-@cli.command(help="Push all changes to the project and output repositories.")
-def push():
+@cli.group(help="Manage data and input-data-repositories.")
+def data():
+ pass
+
+
+@data.command(name="import", help="Import a remote repository into a given location.")
+@click.argument('source_repo_location')
+@click.argument('source_repo_branch')
+@click.argument('target_repo_location', required=False)
+def import_remote_repo(source_repo_location, source_repo_branch, target_repo_location=None):
+ from cadetrdm.repositories import BaseRepo
+ repo = BaseRepo(".")
+ repo.import_remote_repo(source_repo_location=source_repo_location,
+ source_repo_branch=source_repo_branch,
+ target_repo_location=target_repo_location)
+
+
+@data.command(name="fetch", help="Fill data cache based on cadet-rdm.json.")
+@click.option('--re_load', is_flag=True,
+ help='Re-load all data.')
+def fill_data_from_cadet_rdm_json(re_load=False):
+ from cadetrdm.repositories import ProjectRepo
repo = ProjectRepo(".")
- repo.push(push_all=True)
+ repo.fill_data_from_cadet_rdm_json(re_load=re_load)
+
+
+@data.command(name="verify", help="Verify that cache is unchanged.")
+def verify_unchanged_cache():
+ from cadetrdm.repositories import BaseRepo
+ repo = BaseRepo(".")
+ repo.verify_unchanged_cache()
+
+
+@data.command(name="log", help="Print data logs.")
+def print_data_log():
+ from cadetrdm.repositories import ProjectRepo, BaseRepo, OutputRepo
+ import json
+
+ repo = BaseRepo(".")
+
+ with open(repo.data_json_path, "r") as handle:
+ rdm_data = json.load(handle)
+ if rdm_data["is_project_repo"]:
+ repo = ProjectRepo(".")
+ repo.print_output_log()
+ else:
+ repo = OutputRepo()
+ repo.print_data_log()
diff --git a/cadetrdm/repositories.py b/cadetrdm/repositories.py
index 6b1938d..1445a67 100644
--- a/cadetrdm/repositories.py
+++ b/cadetrdm/repositories.py
@@ -116,7 +116,7 @@ class BaseRepo:
def cache_json_path(self):
return self.working_dir / ".cadet-rdm-cache.json"
- def add_remote(self, remote_url, remote_name="origin"):
+ def add_remote(self, remote_url, remote_name=None):
"""
Add a remote to the repository.
@@ -124,6 +124,8 @@ class BaseRepo:
:param remote_name:
:return:
"""
+ if remote_name is None:
+ remote_name = "origin"
self._git_repo.create_remote(remote_name, url=remote_url)
with open(self.data_json_path, "r") as handle:
rdm_data = json.load(handle)
@@ -366,9 +368,9 @@ class BaseRepo:
def reset_hard_to_head(self, force_entry=False):
if not force_entry:
proceed = wait_for_user(f'The output directory contains the following uncommitted changes:\n'
- f'{self.untracked_files + self.changed_files}\n'
- f' These will be lost if you continue\n'
- f'Proceed?')
+ f'{self.untracked_files + self.changed_files}\n'
+ f' These will be lost if you continue\n'
+ f'Proceed?')
else:
proceed = True
if not proceed:
@@ -685,25 +687,7 @@ class ProjectRepo(BaseRepo):
self._output_repo._git.checkout(self._most_recent_branch)
def print_output_log(self):
- def insert_newlines(string, every=30):
- lines = []
- for i in range(0, len(string), every):
- lines.append(string[i:i + every])
- return '\n'.join(lines)
-
- self.output_repo.checkout("master")
-
- tsv_filepath = self.working_dir / self._output_folder / "log.tsv"
-
- with open(tsv_filepath, "r") as filehandle:
- lines = filehandle.readlines()
-
- line_array = [line.replace("\n", "").split("\t") for line in lines]
-
- # Print
- print(tabulate(line_array[1:], headers=line_array[0]))
-
- self.output_repo.checkout(self.output_repo._most_recent_branch)
+ self.output_repo.print_data_log()
def fill_data_from_cadet_rdm_json(self, re_load=False):
"""
@@ -735,7 +719,6 @@ class ProjectRepo(BaseRepo):
source_repo_location=repo_info["source_repo_location"],
source_repo_branch=repo_info["branch_name"])
-
def convert_csv_to_tsv_if_necessary(self):
"""
If not tsv log is found AND a csv log is found, convert the csv to tsv.
@@ -1086,9 +1069,22 @@ class ProjectRepo(BaseRepo):
self.exit_context(message=results_commit_message)
-
class OutputRepo(BaseRepo):
- pass
+
+ def print_data_log(self):
+ self.checkout("master")
+
+ tsv_filepath = self.working_dir / "log.tsv"
+
+ with open(tsv_filepath, "r") as filehandle:
+ lines = filehandle.readlines()
+
+ line_array = [line.replace("\n", "").split("\t") for line in lines]
+
+ # Print
+ print(tabulate(line_array[1:], headers=line_array[0]))
+
+ self.checkout(self._most_recent_branch)
class JupyterInterfaceRepo(ProjectRepo):
@@ -1104,8 +1100,8 @@ class JupyterInterfaceRepo(ProjectRepo):
def commit_nb_output(self, notebook_path: str, results_commit_message: str,
force_rerun=True, timeout=600, conversion_formats: list = None):
if "nbconvert_call" in sys.argv:
- return
- # This is reached in the first call of this function
+ return
+ # This is reached in the first call of this function
if not Path(notebook_path).is_absolute():
notebook_path = self.working_dir / notebook_path
diff --git a/setup.cfg b/setup.cfg
index 42981b6..f77d301 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -29,7 +29,7 @@ include_package_data = True
[options.entry_points]
console_scripts =
- cadet-rdm = cadetrdm.cli_integration:cli
+ rdm = cadetrdm.cli_integration:cli
[options.extras_require]
testing =
diff --git a/tests/test_cli.py b/tests/test_cli.py
new file mode 100644
index 0000000..f71f75e
--- /dev/null
+++ b/tests/test_cli.py
@@ -0,0 +1,115 @@
+from pathlib import Path
+import random
+import os
+
+from click.testing import CliRunner
+
+from cadetrdm.cli_integration import cli
+from cadetrdm.io_utils import delete_path
+
+runner = CliRunner()
+if os.path.exists("test_repo_cli"):
+ delete_path("test_repo_cli")
+
+os.makedirs("test_repo_cli")
+os.chdir("test_repo_cli")
+
+
+def modify_code(path_to_repo):
+ # Add changes to the project code
+ random_number = random.randint(0, 265)
+ filepath = Path(path_to_repo) / f"print_random_number.py"
+ with open(filepath, "w") as file:
+ file.write(
+ f"print({random_number})\n"
+ 'with open("output/data.txt", "w") as handle:\n'
+ f" handle.write({random_number})\n"
+ )
+
+
+def test_01_initialize_repo():
+ result = runner.invoke(cli, ["init", "."])
+ print(result.output)
+ assert result.exit_code == 0
+
+
+def test_02_add_remote():
+ result = runner.invoke(cli, ["remote", "add", "https://jugit.fz-juelich.de/r.jaepel/API_test_project"])
+ print(result.output)
+ assert result.exit_code == 0
+ os.chdir("output")
+ result = runner.invoke(cli, ["remote", "add", "https://jugit.fz-juelich.de/r.jaepel/API_test_project_output"])
+ print(result.output)
+ os.chdir("..")
+ assert result.exit_code == 0
+
+
+def test_02b_clone():
+ os.chdir("..")
+ if os.path.exists("test_repo_cli_cloned"):
+ delete_path("test_repo_cli_cloned")
+ result = runner.invoke(cli, ["clone", "test_repo_cli", "test_repo_cli_cloned"])
+ print(result.output)
+ os.chdir("test_repo_cli")
+ assert result.exit_code == 0
+
+
+def test_03_commit_results_with_uncommited_code_changes():
+ modify_code(".")
+
+ result = runner.invoke(cli, ["run", "python", "print_random_number.py",
+ "create data"])
+ print(result.output)
+ assert result.exit_code != 0
+
+
+def test_04_commit_code():
+ modify_code(".")
+
+ result = runner.invoke(cli, ["commit", "-m", "add code", "-a"])
+ print(result.output)
+ assert result.exit_code == 0
+
+
+# def test_05_commit_results():
+# result = runner.invoke(cli, ["commit", "-m", "add code", "-a"])
+# print(result.output)
+# assert result.exit_code == 0
+# result = runner.invoke(cli, ["run", "python", "print_random_number.py",
+# "create data"])
+# print(result.output)
+# assert result.exit_code == 0
+#
+#
+# def test_05b_execute_command():
+# result = runner.invoke(cli, ["commit", "-m", "add code", "-a"])
+# print(result.output)
+# assert result.exit_code == 0
+# result = runner.invoke(cli, ["run", "command", "python", "print_random_number.py",
+# "create data"])
+# print(result.output)
+# assert result.exit_code == 0
+
+
+def test_06_print_log():
+ result = runner.invoke(cli, ["log"])
+ print(result.output)
+ assert result.exit_code == 0
+
+
+def test_07_lfs_add():
+ result = runner.invoke(cli, ["lfs", "add", "pptx"])
+ print(result.output)
+ assert result.exit_code == 0
+
+
+def test_08_data_import():
+ result = runner.invoke(cli,
+ ["data", "import", "https://github.com/ronald-jaepel/workshop_demo_output",
+ "2023-12-19_10-43-15_output_from_master_86541bc", "imported/repo/data"])
+ print(result.output)
+ assert result.exit_code == 0
+
+# def test_09_data_verify():
+# with open()
+# result = runner.invoke(cli, ["data", "verify"])
--
GitLab