Skip to content
Snippets Groups Projects
Commit b15c1143 authored by Ronald Jäpel's avatar Ronald Jäpel
Browse files

Rework CLI

parent 5a91e99f
No related branches found
No related tags found
No related merge requests found
......@@ -6,4 +6,6 @@ __pycache__
tests/test_repo*
.coverage
dist
.token
\ No newline at end of file
.token
tmp
.ipynb_checkpoints
\ No newline at end of file
......@@ -3,11 +3,6 @@ import subprocess
import click
from .repositories import ProjectRepo, BaseRepo
from .initialize_repo import initialize_repo as initialize_git_repo_implementation
from .initialize_repo import clone as clone_implementation
from .conda_env_utils import prepare_conda_env as prepare_conda_env_implementation
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
......@@ -16,123 +11,178 @@ def cli():
pass
@cli.command()
@click.option('--path_to_repo', default=None,
help='Path to folder for the repository. Optional.')
@cli.command(help="Create an empty CADET-RDM repository or initialize over an existing git repo.")
@click.option('--output_repo_name', default="output",
help='Name of the folder where the tracked output should be stored. Optional. Default: "output".')
@click.option('--gitignore', default=None,
help='List of files to be added to the gitignore file. Optional.')
@click.option('--gitattributes', default=None,
help='List of files to be added to the gitattributes file. Optional.')
@click.argument('path_to_repo')
def init(path_to_repo: str, output_repo_name: (str | bool) = "output", gitignore: list = None,
gitattributes: list = None,
output_repo_kwargs: dict = None):
from cadetrdm.initialize_repo import initialize_repo as initialize_git_repo_implementation
initialize_git_repo_implementation(path_to_repo, output_repo_name, gitignore,
gitattributes, output_repo_kwargs)
@cli.command(help="Clone a repository into a new directory.")
@click.argument('project_url')
def clone(project_url, path_to_repo: str = None):
clone_implementation(project_url, path_to_repo)
@click.argument('dest', required=False)
def clone(project_url, dest: str = None):
from cadetrdm.initialize_repo import clone as clone_implementation
clone_implementation(project_url, dest)
@cli.command()
@click.option('--target_repo_location', default=None,
help='Path to folder for the repository. Optional.')
@click.argument('source_repo_location')
@click.argument('source_repo_branch')
def import_remote_repo(source_repo_location, source_repo_branch, target_repo_location=None):
@cli.command(name="log", help="Show commit logs.")
def print_log():
from cadetrdm.repositories import BaseRepo
repo = BaseRepo(".")
repo.import_remote_repo(source_repo_location=source_repo_location,
source_repo_branch=source_repo_branch,
target_repo_location=target_repo_location)
repo.print_log()
@cli.command()
@click.argument('url')
@click.argument('namespace')
@click.argument('name')
def create_gitlab_remotes(url, namespace, name):
@cli.command(help="Push all changes to the project and output repositories.")
def push():
from cadetrdm.repositories import ProjectRepo
repo = ProjectRepo(".")
repo.create_gitlab_remotes(url=url, namespace=namespace, name=name)
repo.push(push_all=True)
@cli.command()
@click.argument('namespace')
@click.argument('name')
def create_github_remotes(namespace, name):
@cli.command(help="Record changes to the repository")
@click.option("--message", "-m", help="commit message")
@click.option("--all", "-a", is_flag=True, help="commit all changed files")
def commit(message, all):
from cadetrdm.repositories import ProjectRepo
repo = ProjectRepo(".")
repo.create_github_remotes(namespace=namespace, name=name)
@cli.command()
def verify_unchanged_cache():
repo = BaseRepo(".")
repo.verify_unchanged_cache()
repo.commit(message, all)
@cli.command()
@click.option('--re_load', default=False,
help='Re-load all data.')
def fill_data_from_cadet_rdm_json(re_load=False):
repo = BaseRepo(".")
repo.fill_data_from_cadet_rdm_json(re_load=re_load)
@cli.group(help="Execute commands and track the results.")
def run():
pass
@cli.command()
@run.command(name="python")
@click.argument('file_name')
@click.argument('results_commit_message')
def run_python_file(file_name, results_commit_message):
from cadetrdm.repositories import ProjectRepo
repo = ProjectRepo(".")
repo.enter_context()
subprocess.run(["python", file_name])
repo.exit_context(results_commit_message)
@cli.command()
@click.argument('command')
@run.command(name="command")
@click.argument('command', nargs=-1)
@click.argument('results_commit_message')
def run_command(command, results_commit_message):
from cadetrdm.repositories import ProjectRepo
repo = ProjectRepo(".")
repo.enter_context()
subprocess.run(shlex.split(command))
subprocess.run(command)
repo.exit_context(results_commit_message)
@cli.command()
@click.option('--output_repo_name', default="output",
help='Name of the folder where the tracked output should be stored. Optional. Default: "output".')
@click.option('--gitignore', default=None,
help='List of files to be added to the gitignore file. Optional.')
@click.option('--gitattributes', default=None,
help='List of files to be added to the gitattributes file. Optional.')
@click.argument('path_to_repo')
def initialize_repo(path_to_repo: str, output_repo_name: (str | bool) = "output", gitignore: list = None,
gitattributes: list = None,
output_repo_kwargs: dict = None):
initialize_git_repo_implementation(path_to_repo, output_repo_name, gitignore,
gitattributes, output_repo_kwargs)
@cli.group(help="Create, add, and manage remotes.")
def remote():
pass
@cli.command()
@click.option("-p", '--path_to_repo', default=".",
help='Path to repository to which the remote is added. Default is cwd.')
@remote.command(name="add", help="Add")
@click.option('--name', '-n', default=None)
@click.argument('remote_url')
def add_remote_to_repo(remote_url: str, path_to_repo="."):
repo = BaseRepo(path_to_repo)
repo.add_remote(remote_url)
def add_remote(name: str = None, remote_url: str = None):
from cadetrdm.repositories import BaseRepo
repo = BaseRepo(".")
repo.add_remote(remote_url=remote_url, remote_name=name)
print("Done.")
@cli.command()
@click.argument('file_type')
def add_filetype_to_lfs(file_type: str, ):
@remote.command(name="create")
@click.option("-p", '--path_to_repo', default=".",
help='Path to repository to which the remote is added. Default is cwd.')
@click.argument('url')
@click.argument('namespace')
@click.argument('name')
def create_remotes(url, namespace, name):
from cadetrdm.repositories import ProjectRepo
repo = ProjectRepo(".")
if "github" in url:
repo.create_github_remotes(namespace=namespace, name=name)
else:
repo.create_gitlab_remotes(url=url, namespace=namespace, name=name)
@remote.command(name="list")
def list_remotes():
from cadetrdm.repositories import BaseRepo
repo = BaseRepo(".")
repo.add_filetype_to_lfs(file_type)
for _remote, url in zip(repo.remotes, repo.remote_urls):
print(_remote, ": ", url)
@cli.command()
@click.option('--url', default=None,
help='Url to the environment.yml file.')
def prepare_conda_env(url):
prepare_conda_env_implementation(url)
@cli.group(help="Manage large file storage settings.")
def lfs():
pass
@cli.command()
def print_output_log():
# ToDo: test if Project or Output repo
repo = ProjectRepo(".")
repo.print_output_log()
@lfs.command(name="add", help="Add a filetype to git lfs.")
@click.argument('file_types', nargs=-1)
def add_filetype_to_lfs(file_types: list, ):
from cadetrdm.repositories import BaseRepo
repo = BaseRepo(".")
for f_type in file_types:
repo.add_filetype_to_lfs(f_type)
@cli.command(help="Push all changes to the project and output repositories.")
def push():
@cli.group(help="Manage data and input-data-repositories.")
def data():
pass
@data.command(name="import", help="Import a remote repository into a given location.")
@click.argument('source_repo_location')
@click.argument('source_repo_branch')
@click.argument('target_repo_location', required=False)
def import_remote_repo(source_repo_location, source_repo_branch, target_repo_location=None):
from cadetrdm.repositories import BaseRepo
repo = BaseRepo(".")
repo.import_remote_repo(source_repo_location=source_repo_location,
source_repo_branch=source_repo_branch,
target_repo_location=target_repo_location)
@data.command(name="fetch", help="Fill data cache based on cadet-rdm.json.")
@click.option('--re_load', is_flag=True,
help='Re-load all data.')
def fill_data_from_cadet_rdm_json(re_load=False):
from cadetrdm.repositories import ProjectRepo
repo = ProjectRepo(".")
repo.push(push_all=True)
repo.fill_data_from_cadet_rdm_json(re_load=re_load)
@data.command(name="verify", help="Verify that cache is unchanged.")
def verify_unchanged_cache():
from cadetrdm.repositories import BaseRepo
repo = BaseRepo(".")
repo.verify_unchanged_cache()
@data.command(name="log", help="Print data logs.")
def print_data_log():
from cadetrdm.repositories import ProjectRepo, BaseRepo, OutputRepo
import json
repo = BaseRepo(".")
with open(repo.data_json_path, "r") as handle:
rdm_data = json.load(handle)
if rdm_data["is_project_repo"]:
repo = ProjectRepo(".")
repo.print_output_log()
else:
repo = OutputRepo()
repo.print_data_log()
......@@ -116,7 +116,7 @@ class BaseRepo:
def cache_json_path(self):
return self.working_dir / ".cadet-rdm-cache.json"
def add_remote(self, remote_url, remote_name="origin"):
def add_remote(self, remote_url, remote_name=None):
"""
Add a remote to the repository.
......@@ -124,6 +124,8 @@ class BaseRepo:
:param remote_name:
:return:
"""
if remote_name is None:
remote_name = "origin"
self._git_repo.create_remote(remote_name, url=remote_url)
with open(self.data_json_path, "r") as handle:
rdm_data = json.load(handle)
......@@ -366,9 +368,9 @@ class BaseRepo:
def reset_hard_to_head(self, force_entry=False):
if not force_entry:
proceed = wait_for_user(f'The output directory contains the following uncommitted changes:\n'
f'{self.untracked_files + self.changed_files}\n'
f' These will be lost if you continue\n'
f'Proceed?')
f'{self.untracked_files + self.changed_files}\n'
f' These will be lost if you continue\n'
f'Proceed?')
else:
proceed = True
if not proceed:
......@@ -685,25 +687,7 @@ class ProjectRepo(BaseRepo):
self._output_repo._git.checkout(self._most_recent_branch)
def print_output_log(self):
def insert_newlines(string, every=30):
lines = []
for i in range(0, len(string), every):
lines.append(string[i:i + every])
return '\n'.join(lines)
self.output_repo.checkout("master")
tsv_filepath = self.working_dir / self._output_folder / "log.tsv"
with open(tsv_filepath, "r") as filehandle:
lines = filehandle.readlines()
line_array = [line.replace("\n", "").split("\t") for line in lines]
# Print
print(tabulate(line_array[1:], headers=line_array[0]))
self.output_repo.checkout(self.output_repo._most_recent_branch)
self.output_repo.print_data_log()
def fill_data_from_cadet_rdm_json(self, re_load=False):
"""
......@@ -735,7 +719,6 @@ class ProjectRepo(BaseRepo):
source_repo_location=repo_info["source_repo_location"],
source_repo_branch=repo_info["branch_name"])
def convert_csv_to_tsv_if_necessary(self):
"""
If not tsv log is found AND a csv log is found, convert the csv to tsv.
......@@ -1086,9 +1069,22 @@ class ProjectRepo(BaseRepo):
self.exit_context(message=results_commit_message)
class OutputRepo(BaseRepo):
pass
def print_data_log(self):
self.checkout("master")
tsv_filepath = self.working_dir / "log.tsv"
with open(tsv_filepath, "r") as filehandle:
lines = filehandle.readlines()
line_array = [line.replace("\n", "").split("\t") for line in lines]
# Print
print(tabulate(line_array[1:], headers=line_array[0]))
self.checkout(self._most_recent_branch)
class JupyterInterfaceRepo(ProjectRepo):
......@@ -1104,8 +1100,8 @@ class JupyterInterfaceRepo(ProjectRepo):
def commit_nb_output(self, notebook_path: str, results_commit_message: str,
force_rerun=True, timeout=600, conversion_formats: list = None):
if "nbconvert_call" in sys.argv:
return
# This is reached in the first call of this function
return
# This is reached in the first call of this function
if not Path(notebook_path).is_absolute():
notebook_path = self.working_dir / notebook_path
......
......@@ -29,7 +29,7 @@ include_package_data = True
[options.entry_points]
console_scripts =
cadet-rdm = cadetrdm.cli_integration:cli
rdm = cadetrdm.cli_integration:cli
[options.extras_require]
testing =
......
from pathlib import Path
import random
import os
from click.testing import CliRunner
from cadetrdm.cli_integration import cli
from cadetrdm.io_utils import delete_path
runner = CliRunner()
if os.path.exists("test_repo_cli"):
delete_path("test_repo_cli")
os.makedirs("test_repo_cli")
os.chdir("test_repo_cli")
def modify_code(path_to_repo):
# Add changes to the project code
random_number = random.randint(0, 265)
filepath = Path(path_to_repo) / f"print_random_number.py"
with open(filepath, "w") as file:
file.write(
f"print({random_number})\n"
'with open("output/data.txt", "w") as handle:\n'
f" handle.write({random_number})\n"
)
def test_01_initialize_repo():
result = runner.invoke(cli, ["init", "."])
print(result.output)
assert result.exit_code == 0
def test_02_add_remote():
result = runner.invoke(cli, ["remote", "add", "https://jugit.fz-juelich.de/r.jaepel/API_test_project"])
print(result.output)
assert result.exit_code == 0
os.chdir("output")
result = runner.invoke(cli, ["remote", "add", "https://jugit.fz-juelich.de/r.jaepel/API_test_project_output"])
print(result.output)
os.chdir("..")
assert result.exit_code == 0
def test_02b_clone():
os.chdir("..")
if os.path.exists("test_repo_cli_cloned"):
delete_path("test_repo_cli_cloned")
result = runner.invoke(cli, ["clone", "test_repo_cli", "test_repo_cli_cloned"])
print(result.output)
os.chdir("test_repo_cli")
assert result.exit_code == 0
def test_03_commit_results_with_uncommited_code_changes():
modify_code(".")
result = runner.invoke(cli, ["run", "python", "print_random_number.py",
"create data"])
print(result.output)
assert result.exit_code != 0
def test_04_commit_code():
modify_code(".")
result = runner.invoke(cli, ["commit", "-m", "add code", "-a"])
print(result.output)
assert result.exit_code == 0
# def test_05_commit_results():
# result = runner.invoke(cli, ["commit", "-m", "add code", "-a"])
# print(result.output)
# assert result.exit_code == 0
# result = runner.invoke(cli, ["run", "python", "print_random_number.py",
# "create data"])
# print(result.output)
# assert result.exit_code == 0
#
#
# def test_05b_execute_command():
# result = runner.invoke(cli, ["commit", "-m", "add code", "-a"])
# print(result.output)
# assert result.exit_code == 0
# result = runner.invoke(cli, ["run", "command", "python", "print_random_number.py",
# "create data"])
# print(result.output)
# assert result.exit_code == 0
def test_06_print_log():
result = runner.invoke(cli, ["log"])
print(result.output)
assert result.exit_code == 0
def test_07_lfs_add():
result = runner.invoke(cli, ["lfs", "add", "pptx"])
print(result.output)
assert result.exit_code == 0
def test_08_data_import():
result = runner.invoke(cli,
["data", "import", "https://github.com/ronald-jaepel/workshop_demo_output",
"2023-12-19_10-43-15_output_from_master_86541bc", "imported/repo/data"])
print(result.output)
assert result.exit_code == 0
# def test_09_data_verify():
# with open()
# result = runner.invoke(cli, ["data", "verify"])
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment