Skip to content
Snippets Groups Projects
Commit 0e4503fa authored by Lars Bilke's avatar Lars Bilke
Browse files

User recipes, fixed Dockerfile output.

parent f905ffef
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
README.md
+ 4
− 3
View file @ 0e4503fa
......@@ -248,14 +248,15 @@ def run_cmd(cmd):
print(f"Executing: {cmd}")
!poetry run {cmd}
out = setup_ui(run_cmd)
user_recipes = ["user_recipes/my_recipe.py"] # optional user recipes
out = setup_ui(run_cmd, user_recipes)
```
Options `--build` and `--convert` are enabeld per default, then click on button `CREATE CONTAINER`. Optionally run a second cell to get a download link to the image file:
```py
from ogscm.jupyter import display_download_link
display_download_link(out.outputs[0]['text'])
from ogscm.jupyter import display_download_link, display_source
display_source(display_download_link(out.outputs[0]['text']), "docker")
```
Output is stored in `[notebook-location]/_out`. UI state is preserved when notebook was saved. Enable recipes from the tab widget by clicking on the `Disabled`-button (`compiley.py`-recipe is enabled by default).
Test.ipynb
+ 282
− 19
View file @ 0e4503fa
%% Cell type:code id:fc9c6efb-336a-47f0-85f1-812b1c5bad9a tags:
``` python
from ogscm.jupyter import setup_ui
def run_cmd(cmd):
print(f"Executing: {cmd}")
!poetry run {cmd}
out = setup_ui(run_cmd)
user_recipes = ["ogscm/user_recipes/foo.py"]
out = setup_ui(run_cmd, user_recipes)
```
%% Output
General image config
Image build options
Packages to install
%% Cell type:code id:b9c0249b-f350-4a0f-b903-cc4354c4c89b tags:
``` python
from ogscm.jupyter import display_download_link
display_download_link(out.outputs[0]['text'])
from ogscm.jupyter import display_download_link, display_source
display_source(display_download_link(out.outputs[0]['text']), "docker")
```
%% Cell type:code id:a5189654-2d4d-4f6f-a779-0d9c3d4e1846 tags:
%% Output
``` python
```
Download container:
\begin{Verbatim}[commandchars=\\\{\}]
\PY{c}{\PYZsh{} syntax=docker/dockerfile:experimental}
\PY{k}{FROM} \PY{l+s}{ubuntu:20.04} \PY{k}{AS} \PY{l+s}{build}
\PY{c}{\PYZsh{} Generated with ogs\PYZhy{}container\PYZhy{}maker 2.0.0}
\PY{k}{RUN} apt\PYZhy{}get update \PYZhy{}y \PY{o}{\PYZam{}\PYZam{}} \PY{l+s+se}{\PYZbs{}}
\PY{n+nv}{DEBIAN\PYZus{}FRONTEND}\PY{o}{=}noninteractive apt\PYZhy{}get install \PYZhy{}y \PYZhy{}\PYZhy{}no\PYZhy{}install\PYZhy{}recommends \PY{l+s+se}{\PYZbs{}}
curl \PY{l+s+se}{\PYZbs{}}
make \PY{l+s+se}{\PYZbs{}}
tar \PY{l+s+se}{\PYZbs{}}
unzip \PY{l+s+se}{\PYZbs{}}
wget \PY{o}{\PYZam{}\PYZam{}} \PY{l+s+se}{\PYZbs{}}
rm \PYZhy{}rf /var/lib/apt/lists/*
\PY{c}{\PYZsh{} \PYZhy{}\PYZhy{}\PYZhy{} Begin compiler.py \PYZhy{}\PYZhy{}\PYZhy{}}
\PY{c}{\PYZsh{} GNU compiler}
\PY{k}{RUN} apt\PYZhy{}get update \PYZhy{}y \PY{o}{\PYZam{}\PYZam{}} \PY{l+s+se}{\PYZbs{}}
\PY{n+nv}{DEBIAN\PYZus{}FRONTEND}\PY{o}{=}noninteractive apt\PYZhy{}get install \PYZhy{}y \PYZhy{}\PYZhy{}no\PYZhy{}install\PYZhy{}recommends \PY{l+s+se}{\PYZbs{}}
g++ \PY{l+s+se}{\PYZbs{}}
gcc \PY{o}{\PYZam{}\PYZam{}} \PY{l+s+se}{\PYZbs{}}
rm \PYZhy{}rf /var/lib/apt/lists/*
\PY{c}{\PYZsh{} \PYZhy{}\PYZhy{}\PYZhy{} End compiler.py \PYZhy{}\PYZhy{}\PYZhy{}}
\PY{c}{\PYZsh{} Begin ogscm/user\PYZus{}recipes/foo.py}
\PY{c}{\PYZsh{} \PYZhy{}\PYZhy{}\PYZhy{} End ogscm/user\PYZus{}recipes/foo.py \PYZhy{}\PYZhy{}\PYZhy{}}
\end{Verbatim}
# syntax=docker/dockerfile:experimental
FROM ubuntu:20.04 AS build
# Generated with ogs-container-maker 2.0.0
RUN apt-get update -y && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
curl \
make \
tar \
unzip \
wget && \
rm -rf /var/lib/apt/lists/*
# --- Begin compiler.py ---
# GNU compiler
RUN apt-get update -y && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
g++ \
gcc && \
rm -rf /var/lib/apt/lists/*
# --- End compiler.py ---
# Begin ogscm/user_recipes/foo.py
# --- End ogscm/user_recipes/foo.py ---
......
ogscm/cli.py
+ 15
− 13
View file @ 0e4503fa
......@@ -64,21 +64,23 @@ def main(): # pragma: no cover
# https://stackoverflow.com/a/1463370/80480
ldict = {"filename": recipe}
try:
recipe_builtin = pkg_resources.read_text(recipes, recipe)
exec(compile(recipe_builtin, recipe, "exec"), locals(), ldict)
except Exception as err:
error_class = err.__class__.__name__
detail = err.args[0]
cl, exc, tb = sys.exc_info()
line_number = traceback.extract_tb(tb)[-1][1]
print(f"{error_class} at line {line_number}: {detail}")
if not os.path.exists(recipe):
print(f"{recipe} does not exist!")
exit(1)
if os.path.exists(recipe):
with open(recipe, "r") as reader:
exec(compile(reader.read(), recipe, "exec"), locals(), ldict)
else:
try:
recipe_builtin = pkg_resources.read_text(recipes, recipe)
exec(compile(recipe_builtin, recipe, "exec"), locals(), ldict)
except Exception as err:
error_class = err.__class__.__name__
detail = err.args[0]
cl, exc, tb = sys.exc_info()
line_number = traceback.extract_tb(tb)[-1][1]
print(f"{error_class} at line {line_number}: {detail}")
if not os.path.exists(recipe):
print(f"{recipe} does not exist!")
exit(1)
if "out_dir" in ldict:
out_dir = ldict["out_dir"]
if "toolchain" in ldict:
......
ogscm/jupyter.py
+ 98
− 34
View file @ 0e4503fa
......@@ -4,18 +4,20 @@ import os
import pathlib
import importlib.resources as pkg_resources
def on_button_recipe_clicked(b):
# print(b)
if b['new']:
if b["new"]:
b.owner.icon = "check"
b.owner.description = "Enabled"
b.owner.button_style='success'
b.owner.button_style = "success"
else:
b.owner.icon = ""
b.owner.description = "Disabled"
b.owner.button_style=''
b.owner.button_style = ""
def setup_ui(run_cmd):
def setup_ui(run_cmd, user_recipes=[]):
from ogscm.args import setup_args_parser
from ogscm import recipes
import ipywidgets as widgets
......@@ -24,36 +26,50 @@ def setup_ui(run_cmd):
args_dict = {}
for group in parser._action_groups:
if group.title in ['positional arguments', 'optional arguments', 'Image deployment', 'Maintenance']:
if group.title in [
"positional arguments",
"optional arguments",
"Image deployment",
"Maintenance",
]:
continue
print(group.title)
display(setup_args(group._group_actions, args_dict))
dirname = pathlib.Path(__file__).parent.parent / "ogscm" / "recipes"
recipes = sorted(os.listdir(dirname))
for user_recipe in user_recipes:
if os.path.exists(user_recipe):
recipes.append(user_recipe)
tab = widgets.Tab()
tabs = []
tab_names = []
for filename in sorted(os.listdir(dirname)):
for filename in recipes:
if filename.endswith(".py") and not filename.startswith("_"):
parser = argparse.ArgumentParser(add_help=False)
ldict = {"filename": filename}
execute = False
recipe_builtin = pkg_resources.read_text(recipes, filename)
exec(compile(recipe_builtin, filename, "exec"), locals(), ldict)
try:
recipe_builtin = pkg_resources.read_text(recipes, filename)
exec(compile(recipe_builtin, filename, "exec"), locals(), ldict)
except Exception as err:
if os.path.exists(filename):
with open(filename, "r") as reader:
exec(compile(reader.read(), filename, "exec"), locals(), ldict)
if filename == "compiler.py":
button = widgets.ToggleButton(
value=True,
description='Enabled',
button_style='success',
icon = "check"
description="Enabled",
button_style="success",
icon="check",
)
else:
button = widgets.ToggleButton(
value=False,
description='Disabled',
description="Disabled",
)
button.observe(on_button_recipe_clicked, 'value')
button.observe(on_button_recipe_clicked, "value")
args_dict[filename] = button
grid = setup_args(parser._actions, args_dict)
tabs.append(widgets.VBox(children=[button, grid]))
......@@ -64,17 +80,27 @@ def setup_ui(run_cmd):
tab.set_title(idx, val)
display(tab)
button = widgets.Button(description="CREATE CONTAINER", button_style="primary", layout=widgets.Layout(width='100%', height='35px'))
button = widgets.Button(
description="CREATE CONTAINER",
button_style="primary",
layout=widgets.Layout(width="100%", height="35px"),
)
global out
out = widgets.Output(layout={'border': '1px solid black'})
out = widgets.Output(layout={"border": "1px solid black"})
display(button, out)
button.on_click(functools.partial(on_button_clicked, out=out, args_dict=args_dict, run_cmd=run_cmd))
button.on_click(
functools.partial(
on_button_clicked, out=out, args_dict=args_dict, run_cmd=run_cmd
)
)
return out
def setup_args(actions, args_dict):
import ipywidgets as widgets
items = []
for arg in actions:
name = arg.option_strings[0]
......@@ -88,19 +114,27 @@ def setup_args(actions, args_dict):
default_value = ""
if name == "--build_args":
default_value = "\"'--progress=plain'\""
widget = widgets.Text(value=default_value , placeholder=f"{arg.default}", description="(?)", description_tooltip=help)
widget = widgets.Text(
value=default_value,
placeholder=f"{arg.default}",
description="(?)",
description_tooltip=help,
)
items.append(widgets.Label(value=name))
items.append(widget)
args_dict[name] = widget
gridbox = widgets.GridBox(
children = items,
layout = widgets.Layout(
grid_template_columns='150px auto 150px auto',
grid_template_rows='auto',
grid_gap='10px 10px'))
children=items,
layout=widgets.Layout(
grid_template_columns="150px auto 150px auto",
grid_template_rows="auto",
grid_gap="10px 10px",
),
)
return gridbox
def create_cli(items):
cli_string = ""
recipes = ""
......@@ -118,6 +152,7 @@ def create_cli(items):
return f"{recipes}{cli_string}"
def on_button_clicked(b, out, args_dict, run_cmd):
with out:
out.clear_output()
......@@ -125,8 +160,11 @@ def on_button_clicked(b, out, args_dict, run_cmd):
cmd = f"ogscm {cli_string}"
run_cmd(cmd)
from IPython.display import FileLink
import os
class DownloadFileLink(FileLink):
html_link_str = "<a href='{link}' download={file_name}>{link_text}</a>"
......@@ -138,10 +176,37 @@ class DownloadFileLink(FileLink):
def _format_path(self):
from html import escape
fp = ''.join([self.url_prefix, escape(self.path)])
return ''.join([self.result_html_prefix,
self.html_link_str.format(link=fp, file_name=self.file_name, link_text=self.link_text),
self.result_html_suffix])
fp = "".join([self.url_prefix, escape(self.path)])
return "".join(
[
self.result_html_prefix,
self.html_link_str.format(
link=fp, file_name=self.file_name, link_text=self.link_text
),
self.result_html_suffix,
]
)
def display_source(code, language):
import IPython
def _jupyterlab_repr_html_(self):
from pygments import highlight
from pygments.formatters import HtmlFormatter
fmt = HtmlFormatter()
style = "<style>{}\n{}</style>".format(
fmt.get_style_defs(".output_html"), fmt.get_style_defs(".jp-RenderedHTML")
)
return style + highlight(self.data, self._get_lexer(), fmt)
# Replace _repr_html_ with our own version that adds the 'jp-RenderedHTML' class
# in addition to 'output_html'.
IPython.display.Code._repr_html_ = _jupyterlab_repr_html_
return IPython.display.Code(data=code, language=language)
def display_download_link(output):
from pygments import highlight
......@@ -151,24 +216,23 @@ def display_download_link(output):
import re
# Sif
p = re.compile('.*Build.*: (.*\.sif)')
p = re.compile(".*Build.*: (.*\.sif)")
m = p.search(output)
print("Download container:")
if m:
sif_file = m.group(1)
from ogscm.jupyter import DownloadFileLink
import os
print("Download container:")
display(DownloadFileLink(os.path.relpath(sif_file)))
# Dockerfile
p = re.compile('Created definition (.*)')
p = re.compile("Created definition (.*)")
m = p.search(output)
dockerfile = m.group(1)
with open(dockerfile) as f:
code = f.read()
formatter = HtmlFormatter()
IPython.display.HTML('<style type="text/css">{}</style>{}'.format(
formatter.get_style_defs('.highlight'),
highlight(code, DockerLexer(), formatter)))
# Does not work?
# display_source(code, "python3")
return code
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment