[WIP] Decouple examples gallery from the main repository

.github/workflows/execute_tutorial_notebooks_in_external_repo.yml

@@ -0,0 +1,88 @@
+########################################################################################
+# Trigger the notebooks' execution in the external repository.
+#
+# If the "main" branch triggered this event, then the nb execution is triggered in the
+# "main" branch, otherwise in the dev branch.
+#
+# Naming convention used in this workflow:
+#
+# "main repository": Project repository with the python package and the documentation.
+# "linked repository": Repository where the example gallery of jupyter notebooks is
+#                      stored.
+########################################################################################
+name: Trigger notebook execution in the external repository
+
+env:
+  # URL for the external repository linked with the notebooks in this project.
+  LINKED_REPO: pysteps/pysteps_tutorials
+  THIS_REPO: pysteps/pysteps
+
+on:
+  push:
+    branches: [ main, dev ]
+  pull_request:
+    branches: [ main, dev ]
+
+jobs:
+  trigger_nb_execution:
+    name: Trigger the execution in the external repository with the notebooks
+    # The triggering is done by pushing an empty commit to the linked repository.
+    # The commit message contains the Hash of the main repository's commit that
+    # trigger the event.
+    runs-on: "ubuntu-latest"
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      - name: Find the branch that trigger the event
+        id: get_dest_branch
+        run: |
+          if [[ "${GITHUB_EVENT_NAME}" == "push" ]]; then
+            event_branch=$(echo ${GITHUB_REF##*/})
+          elif [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
+            event_branch=$(echo $GITHUB_BASE_REF)
+          else
+            event_branch=unknown
+          fi
+          echo "::set-output name=event_branch::${event_branch}"
+
+          # We only push to latest or to dev.
+          if [[ "${event_branch}" == "main" ]]; then
+            echo "::set-output name=dest_branch::main"
+          else
+            echo "::set-output name=dest_branch::dev"
+          fi
+
+      - name: Print debug information
+        env:
+          DEST_BRANCH: ${{steps.get_dest_branch.outputs.dest_branch}}
+          EVENT_BRANCH: ${{steps.get_dest_branch.outputs.event_branch}}
+        run: |
+          echo "EVENT_BRANCH=${EVENT_BRANCH}"
+          echo "GITHUB_SHA=${GITHUB_SHA}"
+          echo "DEST_BRANCH=${DEST_BRANCH}"
+
+      - name: Clone linked repository
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false # Avoid using the GITHUB_TOKEN  instead of the personal access token
+          fetch-depth: 0 # Avoid errors pushing refs to the destination repository.
+          repository: ${{ env.LINKED_REPO }}
+          ref: ${{steps.get_dest_branch.outputs.dest_branch}}
+
+      - name: Create empty commit in linked repo
+        run: |
+          git config user.name 'github-actions[bot]'
+          git config user.email 'github-actions[bot]@users.noreply.github.com'
+          git commit --allow-empty \
+            -m "Triggering nb execution from ${GITHUB_SHA::8}" \
+            -m "https://github.com/${THIS_REPO}/commit/{$GITHUB_SHA}"
+
+      - name: Push the empty commit to trigger the workflow
+        uses: ad-m/github-push-action@master
+        with:
+          repository: ${{ env.LINKED_REPO }}
+          github_token: ${{ secrets.LINKED_TOKEN }}
+          branch: ${{steps.get_dest_branch.outputs.dest_branch}}

.gitignore

@@ -85,3 +85,6 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+/doc/source/user_guide/examples_gallery/
+/doc/source/examples_gallery/
+/examples/

doc/requirements.txt

@@ -6,4 +6,3 @@ sphinx_rtd_theme
 sphinx_gallery
 scikit-image
 pandas
-

doc/source/conf.py

@@ -1,30 +1,21 @@
-# -*- coding: utf-8 -*-
-
-# All configuration values have a default; values that are commented out
-# serve to show the default.
-
+import json
 import os
+import shutil
 import subprocess
 import sys
+import tempfile
+from pathlib import Path
 
-import json
 from jsmin import jsmin
 
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-if "READTHEDOCS" not in os.environ:
-    sys.path.insert(1, os.path.abspath("../../"))
-
-# -- General configuration ------------------------------------------------
+# -- Global variables ------------------------------------------------
+DOCS_DIR = Path(__file__).parent / ".."
+PROJECT_ROOT_DIR = DOCS_DIR / ".."
+EXAMPLES_GALLERY_REPOSITORY = "https://github.com/pySTEPS/pysteps_tutorials.git"
 
-# If your documentation needs a minimal Sphinx version, state it here.
-#
-needs_sphinx = "1.6"
+sys.path.insert(1, os.path.abspath(str(PROJECT_ROOT_DIR)))
 
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
+# -- General configuration ------------------------------------------------
 extensions = [
     "sphinx.ext.autodoc",
     "sphinx.ext.autosummary",
@@ -36,73 +27,41 @@
 ]
 
 bibtex_bibfiles = ["references.bib"]
-
-# numpydoc_show_class_members = False
-# Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
-
-# The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
-#
-# source_suffix = ['.rst', '.md']
 source_suffix = ".rst"
-
-# The master toctree document.
 master_doc = "index"
-
-# General information about the project.
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints"]
+todo_include_todos = False
 project = "pysteps"
 copyright = "2020, PySteps developers"
 author = "PySteps developers"
 
 
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
 def get_version():
-    """
-    Returns project version as string from 'git describe' command.
-    """
-
+    """Returns project version as string from 'git describe' command."""
     from subprocess import check_output
 
     _version = check_output(["git", "describe", "--tags", "--always"])
-
     if _version:
-        return _version.decode("utf-8")
+        _version = _version.decode("utf-8")
     else:
-        return "X.Y"
-
+        _version = "X.Y"
 
-# The short X.Y version.
-version = get_version().lstrip("v").rstrip().split("-")[0]
+    return _version.lstrip("v").rstrip().split("-")[0]
 
-# The full version, including alpha/beta/rc tags.
-release = version
 
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
+release = get_version()
 
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = "sphinx"
-
-# If true, `todo` and `todoList` produce output, else they produce nothing.
-todo_include_todos = False
+# Sphinx-Gallery options
+sphinx_gallery_conf = {
+    "examples_dirs": "../../examples",  # path to your example scripts
+    "gallery_dirs": "user_guide/examples_gallery",  # path where to save gallery generated examples
+    "filename_pattern": r"/*\.py",  # Include all the files in the examples dir
+    "plot_gallery": False,  # Do not execute the examples
+}
 
 
 # -- Read the Docs build --------------------------------------------------
-
-
 def set_root():
     fn = os.path.abspath(os.path.join("..", "..", "pysteps", "pystepsrc"))
     with open(fn, "r") as f:
@@ -119,56 +78,49 @@ def set_root():
         json.dump(rcparams, f, indent=4)
 
 
-if "READTHEDOCS" in os.environ:
-    repourl = "https://github.com/pySTEPS/pysteps-data.git"
-    dir = os.path.join(os.getcwd(), "..", "..", "pysteps-data")
-    dir = os.path.abspath(dir)
-    subprocess.check_call(["rm", "-rf", dir])
-    subprocess.check_call(["git", "clone", repourl, dir])
-    set_root()
-    pystepsrc = os.path.abspath(os.path.join("..", "..", "pystepsrc.rtd"))
-    os.environ["PYSTEPSRC"] = pystepsrc
+def get_branch_to_pull():
+    try:
+        rtd_version = subprocess.check_output(
+            ["git", "describe", "--abbrev=0", "--tags"], universal_newlines=True
+        ).strip()
+    except subprocess.CalledProcessError:
+        # If we are not in a tag, use latest.
+        rtd_version = "latest"
 
-# -- Options for HTML output ----------------------------------------------
+    return rtd_version
 
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
-# html_theme = 'alabaster'
-# html_theme = 'classic'
-html_theme = "sphinx_rtd_theme"
 
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
-# html_theme_options = {}
+def pull_example_gallery_from_external_repo(branch):
+    global EXAMPLES_GALLERY_REPOSITORY
 
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ["../_static"]
-html_css_files = ["../_static/pysteps.css"]
+    with tempfile.TemporaryDirectory() as work_dir:
+        cmd = f"git clone {EXAMPLES_GALLERY_REPOSITORY} --depth 1 --branch {branch} {work_dir}"
+        subprocess.check_output(cmd.split(" "))
 
-# Custom sidebar templates, must be a dictionary that maps document names
-# to template names.
-#
-# This is required for the alabaster theme
-# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
-html_sidebars = {
-    "**": [
-        "relations.html",  # needs 'show_related': True theme option to display
-        "searchbox.html",
-    ]
-}
+        examples_gallery_dir = DOCS_DIR / "source/examples_gallery"
+        shutil.rmtree(examples_gallery_dir, ignore_errors=True)
 
-html_domain_indices = True
+        examples_dir = PROJECT_ROOT_DIR / "examples"
+        shutil.rmtree(examples_dir, ignore_errors=True)
 
+        work_dir = Path(work_dir)
+        example_gallery_dir_in_remote = work_dir / "docs/examples_gallery"
+        shutil.copytree(example_gallery_dir_in_remote, examples_gallery_dir)
+
+        examples_dir_in_remote = work_dir / "examples"
+        shutil.copytree(examples_dir_in_remote, examples_dir)
+
+
+pull_example_gallery_from_external_repo(get_branch_to_pull())
+
+# -- Options for HTML output ----------------------------------------------
+html_theme = "sphinx_rtd_theme"
+html_static_path = ["../_static"]
+html_css_files = ["../_static/pysteps.css"]
+html_domain_indices = True
 autosummary_generate = True
 
 # -- Options for HTMLHelp output ------------------------------------------
-
-# Output file base name for HTML help builder.
 htmlhelp_basename = "pystepsdoc"
 
 # -- Options for LaTeX output ---------------------------------------------
@@ -199,10 +151,7 @@ def set_root():
 latex_elements = {
     "papersize": "a4paper",
     "pointsize": "10pt",
-    "preamble": latex_preamble
-    # Latex figure (float) alignment
-    #
-    # 'figure_align': 'htbp',
+    "preamble": latex_preamble,
 }
 
 latex_domain_indices = False
@@ -236,13 +185,3 @@ def set_root():
         "Miscellaneous",
     ),
 ]
-
-# -- Options for Sphinx-Gallery -------------------------------------------
-
-# The configuration dictionary for Sphinx-Gallery
-
-sphinx_gallery_conf = {
-    "examples_dirs": "../../examples",  # path to your example scripts
-    "gallery_dirs": "auto_examples",  # path where to save gallery generated examples
-    "filename_pattern": r"/*\.py",  # Include all the files in the examples dir
-}

doc/source/user_guide/index.rst

@@ -16,6 +16,6 @@ the package, see the :ref:`pysteps-reference`.
     install_pysteps
     example_data
     set_pystepsrc
-    ../auto_examples/index
+    ../examples_gallery/index
     machine_learning_pysteps