From 5c5b0b375b84e1a470926627beb5fb33c49782cb Mon Sep 17 00:00:00 2001 From: Bradley-Augstein_finastra Date: Thu, 12 Sep 2024 08:36:26 +0100 Subject: [PATCH 01/13] Run notebook directly --- .github/workflows/JupyterTest.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/JupyterTest.yml b/.github/workflows/JupyterTest.yml index 34bc73f..d3e939d 100644 --- a/.github/workflows/JupyterTest.yml +++ b/.github/workflows/JupyterTest.yml @@ -36,18 +36,10 @@ jobs: - name: Install dependencies run: | - pip install jupyter nbconvert tqdm + pip install jupyter tqdm - name: Convert notebook to script run: | - jupyter nbconvert --to script UnitTest/NotebookToTest.ipynb - - - name: Print converted script - run: | - cat UnitTest/NotebookToTest.py - - - name: Run converted script - run: | - python UnitTest/NotebookToTest.py + pipx run jupyter execute UnitTest/NotebookToTest.ipynb From e4c864727d64be97d68bcb11a3b37c10e97670ac Mon Sep 17 00:00:00 2001 From: baugstein <87702063+ucapbba@users.noreply.github.com> Date: Thu, 12 Sep 2024 08:49:17 +0100 Subject: [PATCH 02/13] Update JupyterTest.yml --- .github/workflows/JupyterTest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/JupyterTest.yml b/.github/workflows/JupyterTest.yml index d3e939d..755d7ba 100644 --- a/.github/workflows/JupyterTest.yml +++ b/.github/workflows/JupyterTest.yml @@ -23,7 +23,7 @@ jobs: - name: Create data directory if it doesn't exist run: | - mkdir -p data + mkdir -p UnitTest/data - name: Cache data id: cache-data From 3784ad0e9f1e7faff10a18befc196d39b195b3a0 Mon Sep 17 00:00:00 2001 From: baugstein <87702063+ucapbba@users.noreply.github.com> Date: Thu, 12 Sep 2024 08:55:10 +0100 Subject: [PATCH 03/13] Update JupyterTest.yml --- .github/workflows/JupyterTest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/JupyterTest.yml b/.github/workflows/JupyterTest.yml index 755d7ba..a8a5552 100644 --- a/.github/workflows/JupyterTest.yml +++ b/.github/workflows/JupyterTest.yml @@ -36,10 +36,10 @@ jobs: - name: Install dependencies run: | - pip install jupyter tqdm + pip install jupyter tqdm pipx - name: Convert notebook to script run: | - pipx run jupyter execute UnitTest/NotebookToTest.ipynb + pipx run --show-output jupyter execute UnitTest/NotebookToTest.ipynb From 52ded0665971b14e29bd2840ebd6405acbeecf02 Mon Sep 17 00:00:00 2001 From: Bradley-Augstein_finastra Date: Thu, 12 Sep 2024 09:02:24 +0100 Subject: [PATCH 04/13] Add convert to script to see logs --- .github/workflows/JupyterTest.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/JupyterTest.yml b/.github/workflows/JupyterTest.yml index a8a5552..3204840 100644 --- a/.github/workflows/JupyterTest.yml +++ b/.github/workflows/JupyterTest.yml @@ -38,8 +38,12 @@ jobs: run: | pip install jupyter tqdm pipx - - name: Convert notebook to script + - name: Run notebook directly run: | - pipx run --show-output jupyter execute UnitTest/NotebookToTest.ipynb + pipx run jupyter execute UnitTest/NotebookToTest.ipynb + + - name: Convert to script to see logs + run: | + pipx runpip jupyter nbconvert --to notebook --execute UnitTest/NotebookToTest.ipynb --stdout From 3cee76225ccbbdd85a641ea3e725d310d7467ce3 Mon Sep 17 00:00:00 2001 From: baugstein <87702063+ucapbba@users.noreply.github.com> Date: Thu, 12 Sep 2024 09:07:12 +0100 Subject: [PATCH 05/13] Update JupyterTest.yml --- .github/workflows/JupyterTest.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/JupyterTest.yml b/.github/workflows/JupyterTest.yml index 3204840..ade01f0 100644 --- a/.github/workflows/JupyterTest.yml +++ b/.github/workflows/JupyterTest.yml @@ -42,8 +42,12 @@ jobs: run: | pipx run jupyter execute UnitTest/NotebookToTest.ipynb - - name: Convert to script to see logs + - name: Convert notebook to script to see logs run: | - pipx runpip jupyter nbconvert --to notebook --execute UnitTest/NotebookToTest.ipynb --stdout + jupyter nbconvert --to script UnitTest/NotebookToTest.ipynb + + - name: Run converted script + run: | + python UnitTest/NotebookToTest.py From dc67397b8a4de6cd79337d38b68f93a437fcaa86 Mon Sep 17 00:00:00 2001 From: baugstein <87702063+ucapbba@users.noreply.github.com> Date: Thu, 12 Sep 2024 09:38:04 +0100 Subject: [PATCH 06/13] Run over multiple python versions --- .github/workflows/JupyterTest.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/JupyterTest.yml b/.github/workflows/JupyterTest.yml index ade01f0..ac46bcb 100644 --- a/.github/workflows/JupyterTest.yml +++ b/.github/workflows/JupyterTest.yml @@ -10,8 +10,11 @@ on: jobs: build: - runs-on: windows-latest - + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + steps: - name: Checkout repository uses: actions/checkout@v3 @@ -19,7 +22,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: '3.8' + python-version: ${{ matrix.python-version }} - name: Create data directory if it doesn't exist run: | From bd631e8f9d514a9b80d403e72da8f3afbe74a83a Mon Sep 17 00:00:00 2001 From: baugstein <87702063+ucapbba@users.noreply.github.com> Date: Thu, 12 Sep 2024 09:40:34 +0100 Subject: [PATCH 07/13] Update JupyterTest.yml --- .github/workflows/JupyterTest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/JupyterTest.yml b/.github/workflows/JupyterTest.yml index ac46bcb..7210978 100644 --- a/.github/workflows/JupyterTest.yml +++ b/.github/workflows/JupyterTest.yml @@ -10,7 +10,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: windows-latest strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] From 53db02a07783b21a3a96a095ef1f14ab2677b9ca Mon Sep 17 00:00:00 2001 From: baugstein <87702063+ucapbba@users.noreply.github.com> Date: Thu, 12 Sep 2024 09:44:15 +0100 Subject: [PATCH 08/13] Refresh cache --- .github/workflows/JupyterTest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/JupyterTest.yml b/.github/workflows/JupyterTest.yml index 7210978..ed36226 100644 --- a/.github/workflows/JupyterTest.yml +++ b/.github/workflows/JupyterTest.yml @@ -33,9 +33,9 @@ jobs: uses: actions/cache@v3 with: path: ./data - key: ${{ runner.os }}-data-${{ hashFiles('testfile.txt') }} + key: ${{ runner.os }}-data-${{ github.run_number }}-${{ hashFiles('testfile.txt') }} restore-keys: | - ${{ runner.os }}-data-2 + ${{ runner.os }}-data-${{ github.run_number }}- - name: Install dependencies run: | From 32651ec2279749b1e3d3b6a24645b87cc2b6097d Mon Sep 17 00:00:00 2001 From: Bradley-Augstein_finastra Date: Thu, 12 Sep 2024 10:14:12 +0100 Subject: [PATCH 09/13] Add numpy import --- UnitTest/NotebookToTest.ipynb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/UnitTest/NotebookToTest.ipynb b/UnitTest/NotebookToTest.ipynb index 64c47ec..dfb5aae 100644 --- a/UnitTest/NotebookToTest.ipynb +++ b/UnitTest/NotebookToTest.ipynb @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 1, "id": "2292a595", "metadata": {}, "outputs": [ @@ -40,12 +40,13 @@ ], "source": [ "import os\n", + "import numpy\n", "print(os.getcwd())" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "90bb750c", "metadata": {}, "outputs": [], @@ -62,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 6, "id": "747c02f6", "metadata": {}, "outputs": [ @@ -70,14 +71,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "Could not find cached data\n" + "Could not find cached data - downloading ...............\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "testfile.txt: 16.4kB [00:00, 35.8kB/s] \n" + "data/testfile.txt: 16.4kB [00:00, 18.8kB/s] \n" ] } ], From 7fafd2abfd29cc4d7e9a30e6ac2daeef6fcf374e Mon Sep 17 00:00:00 2001 From: baugstein <87702063+ucapbba@users.noreply.github.com> Date: Thu, 12 Sep 2024 10:18:30 +0100 Subject: [PATCH 10/13] Update JupyterTest.yml --- .github/workflows/JupyterTest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/JupyterTest.yml b/.github/workflows/JupyterTest.yml index ed36226..c4c8396 100644 --- a/.github/workflows/JupyterTest.yml +++ b/.github/workflows/JupyterTest.yml @@ -39,7 +39,7 @@ jobs: - name: Install dependencies run: | - pip install jupyter tqdm pipx + pip install jupyter tqdm pipx numpy - name: Run notebook directly run: | From d40d69324c079e068368b3cca6533bd9d5dae8b3 Mon Sep 17 00:00:00 2001 From: baugstein <87702063+ucapbba@users.noreply.github.com> Date: Thu, 12 Sep 2024 10:32:10 +0100 Subject: [PATCH 11/13] Update JupyterTest.yml --- .github/workflows/JupyterTest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/JupyterTest.yml b/.github/workflows/JupyterTest.yml index c4c8396..8f757e9 100644 --- a/.github/workflows/JupyterTest.yml +++ b/.github/workflows/JupyterTest.yml @@ -37,9 +37,9 @@ jobs: restore-keys: | ${{ runner.os }}-data-${{ github.run_number }}- - - name: Install dependencies - run: | - pip install jupyter tqdm pipx numpy + - run: pip install pipx + - run: pipx install jupyter + - run: pipx runpip jupyter install numpy matplotlib - name: Run notebook directly run: | From ac3bf87dcf878d8a67c4af9356364db690211246 Mon Sep 17 00:00:00 2001 From: baugstein <87702063+ucapbba@users.noreply.github.com> Date: Thu, 12 Sep 2024 10:35:26 +0100 Subject: [PATCH 12/13] Update JupyterTest.yml --- .github/workflows/JupyterTest.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/JupyterTest.yml b/.github/workflows/JupyterTest.yml index 8f757e9..cc8ad1c 100644 --- a/.github/workflows/JupyterTest.yml +++ b/.github/workflows/JupyterTest.yml @@ -37,12 +37,17 @@ jobs: restore-keys: | ${{ runner.os }}-data-${{ github.run_number }}- - - run: pip install pipx - - run: pipx install jupyter - - run: pipx runpip jupyter install numpy matplotlib + - name: Set PIPX_HOME to a path without spaces + run: | + export PIPX_HOME=$HOME/.local/pipx + mkdir -p $PIPX_HOME + pip install pipx + pipx install jupyter + pipx runpip jupyter install numpy matplotlib ducc0 healpy - name: Run notebook directly run: | + export PIPX_HOME=$HOME/.local/pipx pipx run jupyter execute UnitTest/NotebookToTest.ipynb - name: Convert notebook to script to see logs From 22babfc0315bfa1b289b626982b7854958dea74d Mon Sep 17 00:00:00 2001 From: Bradley-Augstein_finastra Date: Thu, 12 Sep 2024 10:41:33 +0100 Subject: [PATCH 13/13] Remove progress bar for Linux tests --- .github/workflows/JupyterTest.yml | 2 +- UnitTest/NotebookToTest.ipynb | 46 ++++++++++++++----------------- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/.github/workflows/JupyterTest.yml b/.github/workflows/JupyterTest.yml index cc8ad1c..ed9c715 100644 --- a/.github/workflows/JupyterTest.yml +++ b/.github/workflows/JupyterTest.yml @@ -10,7 +10,7 @@ on: jobs: build: - runs-on: windows-latest + runs-on: ubuntu-latest strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] diff --git a/UnitTest/NotebookToTest.ipynb b/UnitTest/NotebookToTest.ipynb index dfb5aae..0325d00 100644 --- a/UnitTest/NotebookToTest.ipynb +++ b/UnitTest/NotebookToTest.ipynb @@ -46,24 +46,35 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "90bb750c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'import urllib.request\\nfrom tqdm import tqdm\\n# Define a progress bar function\\nclass DownloadProgressBar(tqdm):\\n def update_to(self, b=1, bsize=1, tsize=None):\\n if tsize is not None:\\n self.total = tsize\\n self.update(b * bsize - self.n)'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "import urllib.request\n", + "'''import urllib.request\n", "from tqdm import tqdm\n", "# Define a progress bar function\n", "class DownloadProgressBar(tqdm):\n", " def update_to(self, b=1, bsize=1, tsize=None):\n", " if tsize is not None:\n", " self.total = tsize\n", - " self.update(b * bsize - self.n)" + " self.update(b * bsize - self.n)'''" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 4, "id": "747c02f6", "metadata": {}, "outputs": [ @@ -71,14 +82,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Could not find cached data - downloading ...............\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "data/testfile.txt: 16.4kB [00:00, 18.8kB/s] \n" + "Data is already cached.\n" ] } ], @@ -87,26 +91,18 @@ "if not os.path.exists(data_path):\n", " print(\"Could not find cached data - downloading ...............\")\n", " url = \"https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf\"\n", - " with DownloadProgressBar(unit='B', unit_scale=True, miniters=1, desc=data_path) as t:\n", - " urllib.request.urlretrieve(url, data_path, reporthook=t.update_to)\n", + " #with DownloadProgressBar(unit='B', unit_scale=True, miniters=1, desc=data_path) as t:\n", + " urllib.request.urlretrieve(url, data_path, reporthook=t.update_to)\n", "else:\n", " print(\"Data is already cached.\")" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "94ac107d", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "catalog.fits: 57.3kB [00:01, 53.2kB/s] \n" - ] - } - ], + "outputs": [], "source": [] }, {