From 0815f8fe6a71694fb7cfac9144341ae967bd73ec Mon Sep 17 00:00:00 2001
From: "r.jaepel" <r.jaepel@fz-juelich.de>
Date: Thu, 17 Aug 2023 18:01:56 +0200
Subject: [PATCH] add tests for current use-cases
---
.gitignore | 3 +-
modsimdata/utils.py | 161 +-------------------------------------
tests/__init__.py | 0
tests/test_git_adapter.py | 139 ++++++++++++++++++++++++++++++++
4 files changed, 143 insertions(+), 160 deletions(-)
create mode 100644 tests/__init__.py
create mode 100644 tests/test_git_adapter.py
diff --git a/.gitignore b/.gitignore
index 0ec68de..50c03c9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,5 @@
build
*.egg-info
results
-__pycache__
\ No newline at end of file
+__pycache__
+tests/test_repo
\ No newline at end of file
diff --git a/modsimdata/utils.py b/modsimdata/utils.py
index 6d493be..cf509df 100644
--- a/modsimdata/utils.py
+++ b/modsimdata/utils.py
@@ -272,10 +272,10 @@ class ProjectRepo(BaseRepo):
previous_branch = self.output_repo.active_branch.name
self.output_repo.git.checkout(branch_name)
- source_filepath = os.path.join(self._output_folder, file_path)
+ source_filepath = os.path.join(self.output_repo.working_dir, file_path)
# target_folder = os.path.join(self._output_folder + "_cached", branch_name)
- target_folder = os.path.join(self._output_folder, "cached", branch_name)
+ target_folder = os.path.join(self.output_repo.working_dir, "cached", branch_name)
os.makedirs(target_folder, exist_ok=True)
target_filepath = os.path.join(target_folder, file_path)
@@ -414,7 +414,6 @@ def is_tool(name):
"""Check whether `name` is on PATH and marked as executable."""
from shutil import which
-
return which(name) is not None
@@ -471,159 +470,3 @@ def initialize_git_repo(path_to_repo: str, output_repo_name: (str | bool) = "out
repo.git.commit("-m", "initial commit")
os.chdir(starting_directory)
- return
-
-
-def example_generate_results_array(seed=None):
- import numpy as np
-
- if seed is not None:
- np.random.seed(seed)
-
- results_array = np.random.random((500, 3))
- np.savetxt(os.path.join("output", "result.csv"), results_array, delimiter=",")
- return results_array
-
-
-def example_generate_results_figures(input_array):
- import matplotlib.pyplot as plt
- import numpy as np
-
- plt.figure()
- plt.scatter(np.arange(0, 500), input_array[:, 0], alpha=0.5)
- plt.scatter(np.arange(0, 500), input_array[:, 1], alpha=0.5)
- plt.scatter(np.arange(0, 500), input_array[:, 2], alpha=0.5)
- plt.savefig(os.path.join("output", "fig.png"))
- plt.savefig(os.path.join("output", "fig.jpg"), dpi=1000)
- plt.savefig(os.path.join("output", f"fig_{np.random.randint(265)}_{random.randint(0, 1000)}.png"))
-
-
-def alter_code():
- # Add changes to the project code
- random_number = random.randint(0, 265)
- # random_number = 42
- with open("random_number.txt", "a") as file:
- file.write(str(random_number))
- return random_number
-
-
-def example_usage():
- """ Pretend this is a python file """
-
- home_dir = os.path.expanduser("~")
- os.chdir(os.path.join(home_dir, 'ModSimData'))
-
- random_number = alter_code()
-
- project_repo = ProjectRepo(".")
- project_repo.commit(message="fixed super important bug", update_packages=False)
-
- with project_repo.track_results(results_commit_message="Add figures and array"):
- # Generate data
- print("Generating results output")
- results_array = example_generate_results_array(seed=random_number)
- example_generate_results_figures(results_array)
-
-
-def example_write_array():
- """ Pretend this is a python file """
-
- home_dir = os.path.expanduser("~")
- os.chdir(os.path.join(home_dir, 'ModSimData'))
-
- # Add changes to the project code
- random_number = random.randint(0, 265)
- # random_number = 42
- with open(f"random_number_{random_number}.txt", "a") as file:
- file.write(str(random_number))
-
- project_repo = ProjectRepo(".")
- project_repo.commit("add code that writes an array to file", update_packages=False)
-
- with project_repo.track_results(results_commit_message="Add array"):
- example_generate_results_array()
-
- branch_name = str(project_repo.output_repo.active_branch)
- return branch_name
-
-
-def example_load(branch_name):
- """ Pretend this is a python file """
- import numpy as np
- """ move into home directory """
-
- home_dir = os.path.expanduser("~")
- os.chdir(os.path.join(home_dir, 'ModSimData'))
-
- # Add changes to the project code
- random_number = random.randint(0, 265)
- # random_number = 42
- with open(f"random_number_{random_number}.txt", "a") as file:
- file.write(str(random_number))
-
- project_repo = ProjectRepo(".")
- project_repo.commit("add code that creates figures based on an array", update_packages=False)
-
- with project_repo.track_results(results_commit_message="Add figures"):
- cached_array_path = project_repo.cache_previous_results(branch_name=branch_name,
- file_path="result.csv")
- previous_array = np.loadtxt(cached_array_path, delimiter=",")
-
- # with project_repo.load_previous_result_file(branch_name=branch_name,
- # file_path="result.csv") as file_handle:
- # pass
- example_generate_results_figures(previous_array)
-
- branch_name = str(project_repo.output_repo.active_branch)
- return branch_name
-
-
-def example_load_large(branch_name1, branch_name2):
- """ Pretend this is a python file """
- import numpy as np
- """ move into home directory """
-
- home_dir = os.path.expanduser("~")
- os.chdir(os.path.join(home_dir, 'ModSimData'))
-
- # Add changes to the project code
- random_number = random.randint(0, 265)
- # random_number = 42
- with open(f"random_number_{random_number}.txt", "a") as file:
- file.write(str(random_number))
-
- project_repo = ProjectRepo(".")
- project_repo.commit("add code that creates figures based on an array", update_packages=False)
-
- with project_repo.track_results(results_commit_message="Add figures"):
- # cached_fig_path = project_repo.cache_previous_results(branch_name=branch_name2,
- # file_path="fig.jpg")
- cached_array_path = project_repo.cache_previous_results(branch_name=branch_name1,
- file_path="result.csv")
- previous_array = np.loadtxt(cached_array_path, delimiter=",")
-
- example_generate_results_figures(previous_array)
-
- branch_name = str(project_repo.output_repo.active_branch)
- return branch_name
-
-
-def example_two_step_process():
- branch_name = example_write_array()
- branch_name2 = example_load(branch_name)
- example_load_large(branch_name, branch_name2)
-
-
-def create_example_repo():
- os.chdir(os.path.expanduser("~"))
-
- """ initialize Project directory """
- if not os.path.exists("ModSimData"):
- initialize_git_repo("ModSimData")
-
-
-if __name__ == '__main__':
- # pass
- create_example_repo()
- example_usage()
- example_two_step_process()
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/test_git_adapter.py b/tests/test_git_adapter.py
new file mode 100644
index 0000000..7272313
--- /dev/null
+++ b/tests/test_git_adapter.py
@@ -0,0 +1,139 @@
+import os.path
+import shutil
+import stat
+import random
+
+import pytest
+import git
+import numpy as np
+
+from modsimdata import initialize_git_repo, ProjectRepo
+
+
+@pytest.fixture(scope="module")
+def path_to_repo():
+ # a "fixture" serves up shared, ready variables to test functions that should use the fixture as a kwarg
+ return "test_repo"
+
+
+# @pytest.fixture(scope="module", autouse=True)
+# def my_fixture(path_to_repo):
+# print('INITIALIZATION')
+# if os.path.exists(path_to_repo):
+# remove_dir(path_to_repo)
+# yield "this is just here because something must yield"
+# print("TEAR DOWN")
+# remove_dir(path_to_repo)
+
+
+def remove_dir(path_to_dir):
+ def remove_readonly(func, path, exc_info):
+ "Clear the readonly bit and reattempt the removal"
+ # ERROR_ACCESS_DENIED = 5
+ if func not in (os.unlink, os.rmdir) or exc_info[1].winerror != 5:
+ raise exc_info[1]
+ os.chmod(path, stat.S_IWRITE)
+ func(path)
+
+ shutil.rmtree(path_to_dir, onerror=remove_readonly)
+
+
+def modify_code(path_to_repo):
+ # Add changes to the project code
+ random_number = random.randint(0, 265)
+ filepath = os.path.join(path_to_repo, f"print_random_number.py")
+ with open(filepath, "w") as file:
+ file.write(f"print({random_number})\n")
+
+
+def count_commit_number(repo):
+ commit_log = repo.git.log("--oneline").split("\n")
+ current_commit_number = len(commit_log)
+ return current_commit_number
+
+
+def example_generate_results_array(path_to_repo, output_folder):
+ results_array = np.random.random((500, 3))
+ np.savetxt(os.path.join(path_to_repo, output_folder, "result.csv"),
+ results_array,
+ delimiter=",")
+ return results_array
+
+
+def try_initialize_git_repo(path_to_repo):
+ def try_init_gitpython_repo(repo_path):
+ os.path.exists(repo_path)
+ git.Repo(repo_path)
+ return True
+
+ if os.path.exists(path_to_repo):
+ remove_dir(path_to_repo)
+
+ initialize_git_repo(path_to_repo=path_to_repo)
+
+ assert try_init_gitpython_repo(path_to_repo)
+ assert try_init_gitpython_repo(os.path.join(path_to_repo, "output"))
+
+
+def try_commit_code(path_to_repo):
+ repo = ProjectRepo(path_to_repo)
+ current_commit_number = count_commit_number(repo)
+
+ modify_code(path_to_repo)
+ repo.commit("add code to print random number", update_packages=False)
+
+ updated_commit_number = count_commit_number(repo)
+ assert current_commit_number + 1 == updated_commit_number
+
+
+def try_commit_code_without_code_changes(path_to_repo):
+ repo = ProjectRepo(path_to_repo)
+ current_commit_number = count_commit_number(repo)
+ repo.commit("This commit will not be made")
+ updated_commit_number = count_commit_number(repo)
+ assert current_commit_number == updated_commit_number
+
+
+def try_commit_results_data(path_to_repo):
+ repo = ProjectRepo(path_to_repo)
+ current_commit_number = count_commit_number(repo.output_repo)
+ with repo.track_results(results_commit_message="Add array"):
+ example_generate_results_array(path_to_repo, output_folder=repo._output_folder)
+ updated_commit_number = count_commit_number(repo.output_repo)
+ assert current_commit_number + 1 == updated_commit_number
+ return str(repo.output_repo.active_branch)
+
+
+def try_commit_results_with_uncommitted_code_changes(path_to_repo):
+ repo = ProjectRepo(path_to_repo)
+ modify_code(path_to_repo)
+ with pytest.raises(Exception):
+ with repo.track_results(results_commit_message="Add array"):
+ example_generate_results_array(path_to_repo, output_folder=repo._output_folder)
+ repo.commit("add code to print random number", update_packages=False)
+
+
+def try_load_previous_result(path_to_repo, branch_name):
+ repo = ProjectRepo(path_to_repo)
+ with repo.track_results(results_commit_message="Load array and extend"):
+ cached_array_path = repo.cache_previous_results(branch_name=branch_name,
+ file_path="result.csv")
+ previous_array = np.loadtxt(cached_array_path, delimiter=",")
+ extended_array = np.concatenate([previous_array, previous_array])
+ extended_array_file_path = os.path.join(path_to_repo, repo._output_folder, "extended_result.csv")
+ np.savetxt(extended_array_file_path,
+ extended_array,
+ delimiter=",")
+ assert os.path.exists(cached_array_path)
+ assert os.path.exists(extended_array_file_path)
+
+
+def test_cadet_rdm(path_to_repo):
+ # because these depend on one-another and there is no native support afaik for sequential tests
+ # these tests are called sequentially here as try_ functions.
+ try_initialize_git_repo(path_to_repo)
+ try_commit_code(path_to_repo)
+ try_commit_code_without_code_changes(path_to_repo)
+ results_branch_name = try_commit_results_data(path_to_repo)
+ try_commit_results_with_uncommitted_code_changes(path_to_repo)
+ try_load_previous_result(path_to_repo, results_branch_name)
--
GitLab