Merge pull request #606 from jeffjennings/requirements

Use individual notebook requirements

Author: adrn

.github/get_modified_tutorials.py

@@ -1,7 +1,18 @@
 from git import Repo
+from pathlib import Path
 
 
-def main(repo_path, main_branch, **kw):
+def get_all_tutorials():
+    paths = ""
+    for path in Path("./tutorials").rglob("*.ipynb"):
+        paths += f" {path}"
+    print(paths)
+
+
+def main(repo_path, main_branch, return_all, **kw):
+    if return_all:
+        return get_all_tutorials()
+
     repo = Repo(repo_path)
 
     # Check committed changes on this branch against the main branch,
@@ -10,16 +21,20 @@ def main(repo_path, main_branch, **kw):
     diff_lists = [
         repo.commit(main_branch).diff(repo.head),
         repo.head.commit.diff(),
-        repo.head.commit.diff(None)
+        repo.head.commit.diff(None),
     ]
 
     files_changed = set()
     for diffs in diff_lists:
-        files_changed = files_changed.union([
-            diff.b_path for diff in diffs
-            if diff.change_type in ['M', 'A', 'R']  # modified, added, renamed
-            and diff.b_path.endswith('.ipynb')
-        ])
+        files_changed = files_changed.union(
+            [
+                diff.b_path
+                for diff in diffs
+                if diff.change_type
+                in ["M", "A", "R"]  # modified, added, renamed
+                and diff.b_path.endswith(".ipynb")
+            ]
+        )
 
     if files_changed:
         print(" ".join(files_changed))
@@ -30,17 +45,28 @@ def main(repo_path, main_branch, **kw):
 
     parser = ArgumentParser()
     parser.add_argument(
-        '-r', '--repo-path',
-        dest='repo_path',
-        default='.',
-        help='The path to the root of the astropy-tutorials repository folder.'
+        "-r",
+        "--repo-path",
+        dest="repo_path",
+        default=".",
+        help="The path to the root of the astropy-tutorials repository folder.",
+    )
+    parser.add_argument(
+        "--main-branch",
+        dest="main_branch",
+        default="main",
+        help=(
+            "The name of the main branch to compare against. Default is "
+            '"main" but on CI it should be origin/main.'
+        ),
     )
     parser.add_argument(
-        '--main-branch',
-        dest='main_branch',
-        default='main',
-        help=('The name of the main branch to compare against. Default is '
-              '"main" but on CI it should be origin/main.')
+        "--return_all",
+        dest="return_all",
+        default=False,
+        help=(
+            "Whether to return all tutorials rather than just those modified."
+        ),
     )
     args = parser.parse_args()
     main(**vars(args))

.github/workflows/build.yml

@@ -30,10 +30,12 @@ jobs:
       - name: Execute the notebooks
         run: |
           make executeall
+        shell: bash
 
       - name: Convert the notebooks to HTML
         run: |
           make convertall
+        shell: bash
 
       - name: Name artifact
         id: nameartifact

.github/workflows/prs.yml

@@ -36,16 +36,17 @@ jobs:
         if: contains(github.event.pull_request.labels.*.name, 'Run all tutorials')
         run: |
           export TUTORIALS_MAIN_BRANCH=origin/main
-          make executeall
-          make convertall
+          make clean
+          make buildall
+        shell: bash
 
       # Otherwise, only run tutorials that have been modified
       - name: Run only modified tutorials
         if: "!contains(github.event.pull_request.labels.*.name, 'Run all tutorials')"
         run: |
           export TUTORIALS_MAIN_BRANCH=origin/main
-          make execute
-          make convert
+          make build
+        shell: bash
 
       - name: Name artifact
         id: nameartifact

Makefile

@@ -1,32 +1,65 @@
 default: build
 
+SHELL := /bin/bash
+
 TUTORIALS_MAIN_BRANCH ?= main
-MODIFIED := $(shell python .github/get_modified_tutorials.py --main-branch $(TUTORIALS_MAIN_BRANCH))
+
+# paths to the individual notebooks that have been modified
+MODIFIED_NOTEBOOKS := $(shell python .github/get_modified_tutorials.py --main-branch $(TUTORIALS_MAIN_BRANCH))
+# paths to the individual requirements.txt files in the directories in which 1+ notebooks have been modified
+MODIFIED_RQT_PATHS := $(foreach var,$(MODIFIED_NOTEBOOKS),$(addsuffix requirements.txt,$(dir $(var))))
+
+ALL_NOTEBOOKS := $(shell python .github/get_modified_tutorials.py --main-branch $(TUTORIALS_MAIN_BRANCH) --return_all true)
+ALL_RQT_PATHS := $(foreach var,$(ALL_NOTEBOOKS),$(addsuffix requirements.txt,$(dir $(var))))
 
 FLAGS = --flatten --build-path=. -v
-CONVERTFLAGS = --make-index --preprocessors=nbconvert.preprocessors.ExtractOutputPreprocessor --index-template=templates/index.tpl
+CONVERTFLAGS = --make-index --preprocessors=nbconvert.preprocessors.ExtractOutputPreprocessor --index-template=templates/index.tpl --overwrite
 
 init:
 	python -m pip install -U -r requirements-dev.txt
-	pre-commit install
-
-build: envcheck execute convert
-buildall: envcheck executeall convertall
 
-envcheck:
-	python -c "import pkg_resources; pkg_resources.require(open('requirements.txt', mode='r')); print('Your environment is all set!')"
+build: convert
+buildall: convertall
 
 execute:
-	nbcollection execute --timeout=600 ${FLAGS} ${MODIFIED}
+	i=0; \
+	_paths=($(MODIFIED_RQT_PATHS)); \
+	for notebook in ${MODIFIED_NOTEBOOKS}; do \
+		echo Installing requirements from $${_paths[i]}; \
+		python -m pip install --force-reinstall -r $${_paths[i]} > /dev/null; \
+		nbcollection execute --timeout=600 ${FLAGS} $$notebook; \
+		i=$$((i+1)); \
+	done
 
 convert:
-	nbcollection convert ${CONVERTFLAGS} ${FLAGS} ${MODIFIED}
+	i=0; \
+	_paths=($(MODIFIED_RQT_PATHS)); \
+	for notebook in ${MODIFIED_NOTEBOOKS}; do \
+		echo Installing requirements from $${_paths[i]}; \
+		python -m pip install --force-reinstall -r $${_paths[i]} > /dev/null; \
+		nbcollection convert ${CONVERTFLAGS} ${FLAGS} $$notebook; \
+		i=$$((i+1)); \
+	done
 
 executeall:
-	nbcollection execute --timeout=600 ${FLAGS} tutorials
+	i=0; \
+	_paths=(${ALL_RQT_PATHS}); \
+	for notebook in ${ALL_NOTEBOOKS}; do \
+		echo Installing requirements from $${_paths[i]}; \
+		python -m pip install --force-reinstall -r $${_paths[i]} > /dev/null; \
+		nbcollection execute --timeout=600 ${FLAGS} $$notebook; \
+		i=$$((i+1)); \
+	done
 
 convertall:
-	nbcollection convert ${CONVERTFLAGS} ${FLAGS} tutorials
+	i=0; \
+	_paths=($(ALL_RQT_PATHS)); \
+	for notebook in ${ALL_NOTEBOOKS}; do \
+		echo Installing requirements from $${_paths[i]}; \
+		python -m pip install --force-reinstall -r $${_paths[i]} > /dev/null; \
+		nbcollection convert ${CONVERTFLAGS} ${FLAGS} $$notebook; \
+		i=$$((i+1)); \
+	done
 
 clean:
 	rm -rf _build

requirements-dev.txt

@@ -1,4 +1,6 @@
--r requirements.txt
+# Note: these are the requirements *in addition to* those listed in the unique
+# requirements for each set of notebooks - see astropy-tutorials/tutorials/*/requirements.txt
 gitpython
 git+https://github.com/astropy/nbcollection
 pre-commit
+ipykernel

requirements.txt

@@ -27,25 +27,25 @@
     "\n",
     "The primary libraries we'll be using are: [astroquery](http://www.astropy.org/astroquery/), [spectral_cube](https://spectral-cube.readthedocs.io/en/latest/), [reproject](https://reproject.readthedocs.io/en/stable/#), [matplotlib](https://matplotlib.org/)) \n",
     "\n",
-    "They can be installed using conda:"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
+    "They can be installed using conda:\n",
+    "\n",
     "```\n",
     "conda install -c conda-forge astroquery\n",
     "conda install -c conda-forge spectral-cube\n",
     "conda install -c conda-forge reproject\n",
-    "```"
+    "```\n",
+    "\n",
+    "Alternatively, if you don't use conda, you can use pip."
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
+   "outputs": [],
    "source": [
-    "Alternatively, if you don't use conda, you can use pip."
+    "with open('requirements.txt') as f:\n",
+    "    print(f\"Required packages for this notebook:\\n{f.read()}\")"
    ]
   },
   {

tutorials/FITS-cubes/FITS-cubes.ipynb

@@ -1,6 +1,6 @@
 astropy
-astroquery
+astroquery>=0.4.8.dev9474  # 2024-09-24 pinned for Gaia column capitalization issue
 matplotlib
 numpy
 reproject
-spectral_cube
+spectral_cube @ git+https://github.com/radio-astro-tools/spectral-cube  # as of: 2024-10-10 for issue with 'distutils'

tutorials/FITS-cubes/requirements.txt

@@ -24,6 +24,16 @@
     "keyword."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open('requirements.txt') as f:\n",
+    "    print(f\"Required packages for this notebook:\\n{f.read()}\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,