Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
C
CADET-RDM
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Package Registry
Container Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
IBG-1
ModSim
CADET
CADET-RDM
Commits
7e7036ca
Commit
7e7036ca
authored
1 year ago
by
Ronald Jäpel
Browse files
Options
Downloads
Patches
Plain Diff
Add ipynb test
parent
b15c1143
No related branches found
Branches containing commit
No related tags found
Tags containing commit
No related merge requests found
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
tests/print_random_number.py
+3
-0
3 additions, 0 deletions
tests/print_random_number.py
tests/test_jupyter.ipynb
+331
-0
331 additions, 0 deletions
tests/test_jupyter.ipynb
token instructions.png
+0
-0
0 additions, 0 deletions
token instructions.png
with
334 additions
and
0 deletions
tests/print_random_number.py
0 → 100644
+
3
−
0
View file @
7e7036ca
print
(
203
)
with
open
(
"
output/data.txt
"
,
"
w
"
)
as
handle
:
handle
.
write
(
203
)
This diff is collapsed.
Click to expand it.
tests/test_jupyter.ipynb
0 → 100644
+
331
−
0
View file @
7e7036ca
{
"cells": [
{
"cell_type": "code",
"execution_count": 13,
"id": "a5c6949c-da6a-40ab-aec7-de4534582ab2",
"metadata": {},
"outputs": [],
"source": [
"import nbformat as nbf\n",
"import sys\n",
"import os\n",
"from pathlib import Path\n",
"\n",
"import nbformat\n",
"import time\n",
"from nbconvert.preprocessors import ExecutePreprocessor\n",
"\n",
"from nbconvert.nbconvertapp import NbConvertApp\n",
"from ipylab import JupyterFrontEnd"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "e4dd4791-5e5b-4fa7-89f1-8a9e574c300e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1\n"
]
}
],
"source": [
"print(1)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "e983e571-58f5-4310-8584-064fe4ac3570",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2\n"
]
}
],
"source": [
"print(2)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "b28fb6d3-faa9-4788-86a1-4ec0d1026aa8",
"metadata": {},
"outputs": [],
"source": [
"def check_execution_order(notebook_path,\n",
" check_all_executed=False,\n",
" check_all_except_last_executed=True,\n",
" check_top_to_bottom=False,\n",
" check_in_order=True,\n",
" exclude_last_cell=False):\n",
" ntbk = nbf.read(notebook_path, nbf.NO_CONVERT)\n",
"\n",
" # extract all code cells (disregard markdown, raw and others), then extract the execution order\n",
" output_cells = [cell for cell in ntbk.cells if cell[\"cell_type\"] == \"code\"]\n",
" # remove empty cells\n",
" non_empty_cells = [cell for cell in output_cells if cell[\"source\"] != \"\"]\n",
" execution_counts = [cell[\"execution_count\"] for cell in non_empty_cells]\n",
"\n",
" def _all_none(item_list):\n",
" return all([i is None for i in item_list])\n",
"\n",
" # return early if no cells were executed\n",
" if _all_none(execution_counts):\n",
" return True\n",
"\n",
" pass_check = [True]\n",
"\n",
" def _check_all_executed(execution_counts: list) -> bool:\n",
" \"\"\"Check all cells were executed.\n",
"\n",
" Parameters\n",
" ----------\n",
" execution_counts : list\n",
" execution_counts\n",
"\n",
" Returns\n",
" -------\n",
" bool\n",
" \"\"\"\n",
" return not None in execution_counts\n",
"\n",
" def _check_in_order(execution_counts: list) -> bool:\n",
" \"\"\"Check that execution counts that aren't None go from 1 to N.\n",
"\n",
" Parameters\n",
" ----------\n",
" execution_counts : list\n",
" execution counts\n",
"\n",
" Returns\n",
" -------\n",
" bool\n",
" \"\"\"\n",
" print(execution_counts)\n",
" execution_counts = [x for x in execution_counts if x is not None]\n",
" count_range = len(execution_counts)\n",
" if exclude_last_cell:\n",
" count_range = count_range - 1\n",
" is_in_order = all([execution_counts[i] < execution_counts[i + 1] for i in range(count_range - 1)])\n",
" return is_in_order\n",
"\n",
" if check_in_order:\n",
" pass_check.append(_check_in_order(execution_counts))\n",
"\n",
" if check_all_executed:\n",
" pass_check.append(_check_all_executed(execution_counts))\n",
" \n",
" if check_all_except_last_executed:\n",
" pass_check.append(_check_all_executed(execution_counts[:-1]))\n",
"\n",
" if check_top_to_bottom:\n",
" pass_check.append(\n",
" _check_all_executed(execution_counts) and _check_in_order(execution_counts)\n",
" )\n",
" return all(pass_check)\n",
"\n",
"\n",
"def save_ipynb():\n",
" app = JupyterFrontEnd()\n",
" print(\"Saving\", end=\"\")\n",
" # note: docmanager:save doesn't lock the python thread until saving is completed.\n",
" # Sometimes, new changes aren't completely saved before checks are performed.\n",
" # Waiting for 0.1 seconds seems to prevent that.\n",
" app.commands.execute('docmanager:save')\n",
" time.sleep(0.1)\n",
" print(\"\")\n",
"\n",
"\n",
"def reload_notebook():\n",
" app = JupyterFrontEnd()\n",
" app.commands.execute('docmanager:reload')\n",
"\n",
"\n",
"def wait_for_user(message):\n",
" proceed = input(message + \" Y/n\")\n",
" if proceed.lower() == \"y\" or proceed == \"\":\n",
" return True\n",
" else:\n",
" return False\n",
"\n",
"\n",
"def clear_and_rerun_notebook(notebook_filename, force_rerun=False, timeout=600):\n",
" if \"nbconvert_call\" in sys.argv:\n",
" print(\"Finished rerun\")\n",
" return\n",
"\n",
" save_ipynb()\n",
" time.sleep(1)\n",
"\n",
" is_in_order = check_execution_order(notebook_filename)\n",
"\n",
" if is_in_order and not force_rerun:\n",
" print(\"Notebook was already executed in order.\")\n",
" return\n",
" else:\n",
" rerun_confirmed_bool = wait_for_user(\"Notebook was not in order, rerun notebook now?\")\n",
" if not rerun_confirmed_bool and not force_rerun:\n",
" print(\"Aborting.\")\n",
" return\n",
"\n",
" print(\"Rerunning.\")\n",
" with open(notebook_filename) as f:\n",
" nb = nbformat.read(f, as_version=4)\n",
"\n",
" ep = ExecutePreprocessor(timeout=timeout, kernel_name='python3', extra_arguments=[\"nbconvert_call\"])\n",
" ep.preprocess(nb, )\n",
"\n",
" with open(notebook_filename, 'w', encoding='utf-8') as f:\n",
" nbformat.write(nb, f)\n",
"\n",
" reload_notebook()\n",
"\n",
"\n",
"def convert_ipynb(filepath, formats=None):\n",
" if formats is None:\n",
" formats = [\"html\", \"python\"]\n",
" app = NbConvertApp()\n",
" app.initialize()\n",
" output_root_directory = os.path.join(r\"C:\\Users\\ronal\\PycharmProjects\\git_lfs_test_2\", \"results\",\n",
" os.path.basename(filepath.replace('.', '_')))\n",
" for format in formats:\n",
" app.export_format = format\n",
" app.notebooks = [filepath]\n",
" app.output_base = os.path.join(output_root_directory, os.path.basename(filepath.replace('.ipynb', '')))\n",
" if not os.path.exists(output_root_directory):\n",
" os.makedirs(output_root_directory)\n",
" app.start()\n",
"\n",
"\n",
"\n",
"def export_all_figures():\n",
" import junix\n",
"\n",
" for file in os.listdir(git_repo.working_dir):\n",
" if file.endswith(\"ipynb\"):\n",
" file_without_extension = Path(file).stem\n",
" images = junix.export_images(filepath=os.path.join(git_repo.working_dir, file),\n",
" output_dir=os.path.join(git_repo.working_dir, \"results\",\n",
" file.replace(\".\", \"_\")),\n",
" prefix=file_without_extension)\n",
" convert_ipynb(filepath=os.path.join(git_repo.working_dir, file), formats=[\"html\"])\n"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "99404052-11f1-499b-a23f-4a90ed917300",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Saving\n",
"[13, 14, 15, 24, None, 25, 26]\n"
]
},
{
"name": "stdin",
"output_type": "stream",
"text": [
"Notebook was not in order, rerun notebook now? Y/n n\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Aborting.\n"
]
}
],
"source": [
"clear_and_rerun_notebook(\"test_jupyter.ipynb\")"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "fec9df11-5622-49d8-8e08-c6e8bbe73dfd",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Saving\n",
"[13, 14, 15, 24, 27, None, 26]\n"
]
},
{
"data": {
"text/plain": [
"False"
]
},
"execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"save_ipynb()\n",
"time.sleep(1)\n",
"check_execution_order(\"test_jupyter.ipynb\")"
]
},
{
"cell_type": "code",
"execution_count": 29,
"id": "81841f17-c8d7-4a1e-b13d-d3f5a19adfd4",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Saving\n",
"[13, 14, 15, 24, 27, 28, None]\n",
"Notebook was already executed in order.\n"
]
}
],
"source": [
"clear_and_rerun_notebook(\"test_jupyter.ipynb\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.0"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
%% Cell type:code id:a5c6949c-da6a-40ab-aec7-de4534582ab2 tags:
```
python
import
nbformat
as
nbf
import
sys
import
os
from
pathlib
import
Path
import
nbformat
import
time
from
nbconvert.preprocessors
import
ExecutePreprocessor
from
nbconvert.nbconvertapp
import
NbConvertApp
from
ipylab
import
JupyterFrontEnd
```
%% Cell type:code id:e4dd4791-5e5b-4fa7-89f1-8a9e574c300e tags:
```
python
print
(
1
)
```
%% Output
1
%% Cell type:code id:e983e571-58f5-4310-8584-064fe4ac3570 tags:
```
python
print
(
2
)
```
%% Output
2
%% Cell type:code id:b28fb6d3-faa9-4788-86a1-4ec0d1026aa8 tags:
```
python
def
check_execution_order
(
notebook_path
,
check_all_executed
=
False
,
check_all_except_last_executed
=
True
,
check_top_to_bottom
=
False
,
check_in_order
=
True
,
exclude_last_cell
=
False
):
ntbk
=
nbf
.
read
(
notebook_path
,
nbf
.
NO_CONVERT
)
# extract all code cells (disregard markdown, raw and others), then extract the execution order
output_cells
=
[
cell
for
cell
in
ntbk
.
cells
if
cell
[
"
cell_type
"
]
==
"
code
"
]
# remove empty cells
non_empty_cells
=
[
cell
for
cell
in
output_cells
if
cell
[
"
source
"
]
!=
""
]
execution_counts
=
[
cell
[
"
execution_count
"
]
for
cell
in
non_empty_cells
]
def
_all_none
(
item_list
):
return
all
([
i
is
None
for
i
in
item_list
])
# return early if no cells were executed
if
_all_none
(
execution_counts
):
return
True
pass_check
=
[
True
]
def
_check_all_executed
(
execution_counts
:
list
)
->
bool
:
"""
Check all cells were executed.
Parameters
----------
execution_counts : list
execution_counts
Returns
-------
bool
"""
return
not
None
in
execution_counts
def
_check_in_order
(
execution_counts
:
list
)
->
bool
:
"""
Check that execution counts that aren
'
t None go from 1 to N.
Parameters
----------
execution_counts : list
execution counts
Returns
-------
bool
"""
print
(
execution_counts
)
execution_counts
=
[
x
for
x
in
execution_counts
if
x
is
not
None
]
count_range
=
len
(
execution_counts
)
if
exclude_last_cell
:
count_range
=
count_range
-
1
is_in_order
=
all
([
execution_counts
[
i
]
<
execution_counts
[
i
+
1
]
for
i
in
range
(
count_range
-
1
)])
return
is_in_order
if
check_in_order
:
pass_check
.
append
(
_check_in_order
(
execution_counts
))
if
check_all_executed
:
pass_check
.
append
(
_check_all_executed
(
execution_counts
))
if
check_all_except_last_executed
:
pass_check
.
append
(
_check_all_executed
(
execution_counts
[:
-
1
]))
if
check_top_to_bottom
:
pass_check
.
append
(
_check_all_executed
(
execution_counts
)
and
_check_in_order
(
execution_counts
)
)
return
all
(
pass_check
)
def
save_ipynb
():
app
=
JupyterFrontEnd
()
print
(
"
Saving
"
,
end
=
""
)
# note: docmanager:save doesn't lock the python thread until saving is completed.
# Sometimes, new changes aren't completely saved before checks are performed.
# Waiting for 0.1 seconds seems to prevent that.
app
.
commands
.
execute
(
'
docmanager:save
'
)
time
.
sleep
(
0.1
)
print
(
""
)
def
reload_notebook
():
app
=
JupyterFrontEnd
()
app
.
commands
.
execute
(
'
docmanager:reload
'
)
def
wait_for_user
(
message
):
proceed
=
input
(
message
+
"
Y/n
"
)
if
proceed
.
lower
()
==
"
y
"
or
proceed
==
""
:
return
True
else
:
return
False
def
clear_and_rerun_notebook
(
notebook_filename
,
force_rerun
=
False
,
timeout
=
600
):
if
"
nbconvert_call
"
in
sys
.
argv
:
print
(
"
Finished rerun
"
)
return
save_ipynb
()
time
.
sleep
(
1
)
is_in_order
=
check_execution_order
(
notebook_filename
)
if
is_in_order
and
not
force_rerun
:
print
(
"
Notebook was already executed in order.
"
)
return
else
:
rerun_confirmed_bool
=
wait_for_user
(
"
Notebook was not in order, rerun notebook now?
"
)
if
not
rerun_confirmed_bool
and
not
force_rerun
:
print
(
"
Aborting.
"
)
return
print
(
"
Rerunning.
"
)
with
open
(
notebook_filename
)
as
f
:
nb
=
nbformat
.
read
(
f
,
as_version
=
4
)
ep
=
ExecutePreprocessor
(
timeout
=
timeout
,
kernel_name
=
'
python3
'
,
extra_arguments
=
[
"
nbconvert_call
"
])
ep
.
preprocess
(
nb
,
)
with
open
(
notebook_filename
,
'
w
'
,
encoding
=
'
utf-8
'
)
as
f
:
nbformat
.
write
(
nb
,
f
)
reload_notebook
()
def
convert_ipynb
(
filepath
,
formats
=
None
):
if
formats
is
None
:
formats
=
[
"
html
"
,
"
python
"
]
app
=
NbConvertApp
()
app
.
initialize
()
output_root_directory
=
os
.
path
.
join
(
r
"
C:\Users\ronal\PycharmProjects\git_lfs_test_2
"
,
"
results
"
,
os
.
path
.
basename
(
filepath
.
replace
(
'
.
'
,
'
_
'
)))
for
format
in
formats
:
app
.
export_format
=
format
app
.
notebooks
=
[
filepath
]
app
.
output_base
=
os
.
path
.
join
(
output_root_directory
,
os
.
path
.
basename
(
filepath
.
replace
(
'
.ipynb
'
,
''
)))
if
not
os
.
path
.
exists
(
output_root_directory
):
os
.
makedirs
(
output_root_directory
)
app
.
start
()
def
export_all_figures
():
import
junix
for
file
in
os
.
listdir
(
git_repo
.
working_dir
):
if
file
.
endswith
(
"
ipynb
"
):
file_without_extension
=
Path
(
file
).
stem
images
=
junix
.
export_images
(
filepath
=
os
.
path
.
join
(
git_repo
.
working_dir
,
file
),
output_dir
=
os
.
path
.
join
(
git_repo
.
working_dir
,
"
results
"
,
file
.
replace
(
"
.
"
,
"
_
"
)),
prefix
=
file_without_extension
)
convert_ipynb
(
filepath
=
os
.
path
.
join
(
git_repo
.
working_dir
,
file
),
formats
=
[
"
html
"
])
```
%% Cell type:code id:99404052-11f1-499b-a23f-4a90ed917300 tags:
```
python
clear_and_rerun_notebook
(
"
test_jupyter.ipynb
"
)
```
%% Output
Saving
[13, 14, 15, 24, None, 25, 26]
Notebook was not in order, rerun notebook now? Y/n n
Aborting.
%% Cell type:code id:fec9df11-5622-49d8-8e08-c6e8bbe73dfd tags:
```
python
save_ipynb
()
time
.
sleep
(
1
)
check_execution_order
(
"
test_jupyter.ipynb
"
)
```
%% Output
Saving
[13, 14, 15, 24, 27, None, 26]
False
%% Cell type:code id:81841f17-c8d7-4a1e-b13d-d3f5a19adfd4 tags:
```
python
clear_and_rerun_notebook
(
"
test_jupyter.ipynb
"
)
```
%% Output
Saving
[13, 14, 15, 24, 27, 28, None]
Notebook was already executed in order.
This diff is collapsed.
Click to expand it.
token instructions.png
0 → 100644
+
0
−
0
View file @
7e7036ca
57.5 KiB
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment