diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d30a240c4e8fac311146bd3ae5144483fa673b41..589a271c04568eb708b28b54a54a5e56cd4a7c9b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,7 +57,10 @@ include: - if: $CI_PIPELINE_SOURCE != "schedule" && $CI_MERGE_REQUEST_LABELS !~ /.*ci::\w* only.*/ - local: "/scripts/ci/pipelines/web.yml" rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /.*ci::web only.*/ + - if: $CI_MERGE_REQUEST_LABELS =~ /.*ci::web only?($|,)/ + - local: "/scripts/ci/pipelines/web-fast.yml" + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /.*ci::web only \(fast.*/ - local: "/scripts/ci/pipelines/scheduled.yml" rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' diff --git a/Tests/Data/requirements-dev.txt b/Tests/Data/requirements-dev.txt index b8b7930bd3cb36c9e6330a10333c076efa16d622..e04f5bd31f84f31c155cfc047175c856ebdc8f29 100644 --- a/Tests/Data/requirements-dev.txt +++ b/Tests/Data/requirements-dev.txt @@ -1,4 +1,4 @@ -git+https://github.com/bilke/nb2hugo@53d62ae5aef1be271eb91491d3b37da487bd1498#egg=nb2hugo +git+https://github.com/bilke/nb2hugo@f9744903ed17d46afb3877ffe244420a50aaecc6#egg=nb2hugo nbconvert==7.2.9 nbformat==5.7.3 toml==0.10.2 diff --git a/scripts/ci/jobs/web-deploy.yml b/scripts/ci/jobs/web-deploy.yml index 661c2dc4fb027fec5d9acde5a0c51e93f91a67dc..d4ce199bde50259f36024973abdb86ef229d9fbc 100644 --- a/scripts/ci/jobs/web-deploy.yml +++ b/scripts/ci/jobs/web-deploy.yml @@ -21,6 +21,7 @@ deploy web site: # Copy notebook pages - cp -rl build/*/web/content web/ - cd web + - python scripts/convert_notebooks.py - yarn - yarn build - netlify deploy --prod --dir=public --site=$OGS_NETLIFY_SITE_ID diff --git a/scripts/ci/jobs/web-preview.yml b/scripts/ci/jobs/web-preview.yml index 59cbad21410c1e1b1eaedee0d166955374704523..2314945f8f3cfd9e9957bc2e724f0354500569c5 100644 --- a/scripts/ci/jobs/web-preview.yml +++ b/scripts/ci/jobs/web-preview.yml @@ -1,15 +1,21 @@ preview web site: stage: build image: $WEB_IMAGE - needs: [ci_images, "build linux arch", "build mac petsc arm64"] + needs: + - job: ci_images + - job: "build linux arch" + optional: true + - job: "build mac petsc arm64" + optional: true variables: HUGO_ENVIRONMENT: "staging" script: # Copy notebook pages - - cp -rl build/*/web/content web/ + - cp -rl build/*/web/content web/ || true - cd web # Symlink for prj link checks (data-link shortcode) - ln -s ../Tests . + - python scripts/convert_notebooks.py - yarn - yarn build rules: diff --git a/scripts/ci/pipelines/web-fast.yml b/scripts/ci/pipelines/web-fast.yml new file mode 100644 index 0000000000000000000000000000000000000000..f317ead02e12ef3c19df12a4546b2fd864cc4c50 --- /dev/null +++ b/scripts/ci/pipelines/web-fast.yml @@ -0,0 +1,4 @@ +include: + - local: "/scripts/ci/jobs/ci_images.yml" + - local: "/scripts/ci/jobs/pre-commit.yml" + - local: "/scripts/ci/jobs/web-preview.yml" diff --git a/scripts/docker/Dockerfile.web b/scripts/docker/Dockerfile.web index 98214bcc2be748a3ac30f4862ea51c1d0e89d142..67c8511717b2a53428c414ce5d06438066930761 100644 --- a/scripts/docker/Dockerfile.web +++ b/scripts/docker/Dockerfile.web @@ -1,5 +1,5 @@ FROM python:3-slim -RUN pip install LinkChecker==10.1.0 +RUN pip install LinkChecker==10.2.1 CMD [ "/bin/bash" ] RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ --no-install-recommends curl git gnupg2 && \ @@ -14,8 +14,8 @@ RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \ && rm -rf /var/lib/apt/lists/* RUN yarn global add node-gyp RUN yarn global add netlify-cli -ENV HUGO_VERSION=0.101.0 -RUN curl -fSL -O "https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_Linux-64bit.deb" \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y /hugo_extended_${HUGO_VERSION}_Linux-64bit.deb \ - && rm /hugo_extended_${HUGO_VERSION}_Linux-64bit.deb -RUN pip install https://github.com/bilke/nb2hugo/archive/0930c6591a4dae07c794b797a264fbcac1ad91aa.zip +ENV HUGO_VERSION=0.117.0 +RUN curl -fSL -O "https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb" \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y /hugo_extended_${HUGO_VERSION}_linux-amd64.deb \ + && rm /hugo_extended_${HUGO_VERSION}_linux-amd64.deb +RUN pip install git+https://github.com/bilke/nb2hugo@f9744903ed17d46afb3877ffe244420a50aaecc6#egg=nb2hugo diff --git a/web/content/docs/devguide/documentation/jupyter-docs/index.md b/web/content/docs/devguide/documentation/jupyter-docs/index.md index d7b217fb7879efecdd57a805baeef90709e92b2d..d52a31b31c637e04b4c2a59e241a61419416e2e1 100644 --- a/web/content/docs/devguide/documentation/jupyter-docs/index.md +++ b/web/content/docs/devguide/documentation/jupyter-docs/index.md @@ -10,16 +10,78 @@ parent = "development-workflows" ## The big picture -[Jupyter notebooks](https://jupyter.org) are interactive computing environments where prose and code can be combined. In the OGS project notebooks can be used to define complex benchmark workflows and its results can be converted to be shown on the OGS web page (see an [example here](/docs/benchmarks/small-deformations/linear_disc_with_hole/)). +[Jupyter notebooks](https://jupyter.org) are interactive computing environments where prose and code can be combined. In the OGS project notebooks can be used to define complex benchmark workflows and its results can be converted to be shown on the OGS web page (see an [example here](/docs/benchmarks/small-deformations/linear_disc_with_hole/)). Notebooks can be used in two ways: -## Create a new notebook +1. To [directly generate a web page](#1-direct-conversion) (direct conversion from notebook to web page). +2. To be [executed during CI](#2-executed-notebooks) whose results are converted to a web page. + +## 1. Direct conversion + +Consider direct conversion for the following use cases: + +- Execution of the notebook is very time-consuming (too long for regular CI tests). +- The notebook uses Python packages which are not part of the testing environment (see `Tests/Data/requirements*.txt` for available packages). +- The notebooks needs other custom tools or environment. +- The shown functionality is not required to be regularly tested. + +### Create a new notebook + +Create a new notebook file in `web/contents/docs/[some-section]/my-page/my-page.ipynb`. It is important that the notebook filename is the same as the containing folder name! + +If you use additional images put them into the `my-page`-folder. + +### Add web meta information + +If the notebook result should appear as a page on the web documentation a frontmatter with some meta information (similar to [regular web pages]({{< ref "web-docs.md" >}})) is required as the first cell in the notebook: + +- Add a new cell and move it to the first position in the notebook +- Change the cell type to `raw`! +- Add meta information, conclude with a end-of-file marker (`<!--eofm-->`) e.g.: + + ```md + title = "BHE Meshing" + date = "2023-08-18" + author = "Joy Brato Shil, Haibing Shao" + <!--eofm--> + ``` + +--- + +Make sure that you execute the cells in the notebook and save the notebook (with generated outputs). + +### Preview locally + +To get a preview of the web page run the `convert_notebooks`-script: + +```bash +# You need the converter-tool nb2hugo installed. Recommended way is to +# create and activate a virtual environment and install it there: +python -m venv .venv # or `python3 -m ...` on some systems +pip install git+https://github.com/bilke/nb2hugo@ogs + +python web/scripts/convert_notebooks.py + +cd web +hugo server +# open http://localhost:1313 +``` + +The notebook needs some meta information (only `title`, `date` and `author` is required) as [outlined below](#add-web-meta-information). + +Also make sure that you also provide necessary data files and please don't use machine specific paths (e.g. assume that `ogs` and other tools are in the `PATH`). + +## 2. Executed notebooks + +These notebooks are part of the regular CI testing. Please try to keep the notebook execution time low. + +### Create a new notebook Create a new notebook file in `Tests/Data` (either as regular `.ipynb`-files or as Markdown-based notebooks via [Jupytext](https://jupytext.readthedocs.io/en/latest)). See examples: - [SimpleMechanics.ipynb](https://gitlab.opengeosys.org/ogs/ogs/-/blob/master/Tests/Data/Mechanics/Linear/SimpleMechanics.ipynb) (regular `.ipynb`-notebook) - [Linear_Disc_with_hole.md](https://gitlab.opengeosys.org/ogs/ogs/-/blob/master/Tests/Data/Mechanics/Linear/DiscWithHole/Linear_Disc_with_hole.md) (Jupytext-based notebook in Markdown) -## Add web meta information +### Add web meta information If the notebook result should appear as a page on the web documentation a frontmatter with some meta information (similar to [regular web pages]({{< ref "web-docs.md" >}})) is required as the first cell in the notebook: @@ -36,6 +98,8 @@ If the notebook result should appear as a page on the web documentation a frontm <!--eofm--> ``` +`web_subsection` needs to be set to a sub-folder in [web/content/docs/benchmarks](https://gitlab.opengeosys.org/ogs/ogs/-/tree/master/web/content/docs/benchmarks) (if not set the notebook page will not be linked from navigation bar / benchmark gallery on the web page). + <div class='note'> In Jupytext-based notebooks you can add the frontmatter within the `<!-- #raw -->`- and `<!-- #endraw -->`-markers: @@ -50,13 +114,11 @@ web_subsection = "small-deformations" <!-- #endraw --> ``` -`web_subsection` needs to be set to a sub-folder in [web/content/docs/benchmarks](https://gitlab.opengeosys.org/ogs/ogs/-/tree/master/web/content/docs/benchmarks) (if not set the notebook page will not be linked from navigation bar / benchmark gallery on the web page). - </div> -## Notebook setup +### Notebook setup -### Python cells +#### Python cells - Do not use machine-specific or absolute paths! See the following example to set up notebook output paths: @@ -87,7 +149,7 @@ web_subsection = "small-deformations" - Do not write anything into the source directories. Use an `out_dir` as above. - Assume that `ogs` and other tools are in the `PATH`. -### Markdown cells +#### Markdown cells - Do not use HTML inside Markdown blocks. - Static images e.g. for the gallery image or to be used in Markdown cells have to be located in either `images`- or `figures`-subdirectories beneath the Notebook file! Otherwise they will not show up in the web site. @@ -99,7 +161,7 @@ web_subsection = "small-deformations" - Please note that in merge request web previews static images are not shown at all. -## Execution environment +### Execution environment In CI the notebooks are executed with all dependencies installed into a virtual environment in the build directory. The installed packages are defined in `Test/Data/requirements.txt`. The same setup can be enabled locally by setting the CMake option `OGS_USE_PIP=ON`. E.g. @@ -109,9 +171,9 @@ source ../build/release/.venv/bin/activate # Activates the virtual environment jupyter lab # Starts Jupyter Lab ``` -## Register with CTest +### Register with CTest -Add the notebook to CTest with e.g.: +Add the notebook to CTest ([example](https://gitlab.opengeosys.org/ogs/ogs/-/blob/master/ProcessLib/SmallDeformation/Tests.cmake#L272-281)) with e.g.: ```cmake if(NOT OGS_USE_PETSC) @@ -127,9 +189,9 @@ source .venv/bin/activate # Is created with OGS_USE_PIP=ON, see above note on en ctest -R nb -j 4 --output-on-failure ``` -## Advanced topics +### Advanced topics -### Jupytext usage +#### Jupytext usage If you use the [execution environment](#execution-environment) [Jupytext](https://jupytext.readthedocs.io/en/latest) is already configured and its usage is transparent: @@ -137,7 +199,7 @@ If you use the [execution environment](#execution-environment) [Jupytext](https: - Upon saving or executing a linked `.ipynb`-file is created in the background which stores outputs - You still edit the Markdown file but don't notice the difference to regular notebooks in the Lab UI -### Run a notebook in BinderHub +#### Run a notebook in BinderHub On the web site or MR web previews on pages generated by a notebook there is a new banner: @@ -147,7 +209,7 @@ On the web site or MR web previews on pages generated by a notebook there is a n - The environment running in BinderHub is defined in [`bilke/binder-ogs-requirements` at GitHub](https://github.com/bilke/binder-ogs-requirements) - When clicking the link it launches a Jupyter Lab instance pre-configured with ogs [via wheel](https://gitlab.opengeosys.org/ogs/ogs/-/blob/master/Tests/Data/requirements-ogs.txt#L2), clones the current ogs repo in it and opens the respective notebook ready to run. Please note that startup times may be several minutes and the computing resources are limited (1 core, 2GB RAM). For improved performance we would need to setup own infrastructure. Also currently only works for serial ogs configurations. -### PyVista notebooks on headless Linux systems +#### PyVista notebooks on headless Linux systems PyVista (or VTK) requires a windowing environment for rendering. You can provide a virtual window with `xvfb-run`: diff --git a/web/scripts/convert_notebooks.py b/web/scripts/convert_notebooks.py new file mode 100644 index 0000000000000000000000000000000000000000..19237c167b78a66b46e9e0b9458793dbac39b54d --- /dev/null +++ b/web/scripts/convert_notebooks.py @@ -0,0 +1,26 @@ +import os +import subprocess +import sys +from pathlib import Path + +content_dir = Path(__file__).resolve().parent.parent / "content" +os.chdir(content_dir) +notebooks = Path().glob("**/*.ipynb") +exit_code = 0 + +for notebook in notebooks: + nb = Path(notebook) + if nb.parent.stem != nb.stem: + print( + f"Error for {notebook}: Notebook must have the same name as its parent folder!" + ) + exit_code = 1 + continue + print(f"Converting {notebook} ...") + subprocess.run( + f"nb2hugo --site-dir .. --section {nb.parent.parent} {notebook}", + shell=True, + check=True, + ) + +sys.exit(exit_code)