diff --git a/.github/workflows/ExportNotebooks.yml b/.github/workflows/ExportNotebooks.yml index b8bbb81f..46d23427 100644 --- a/.github/workflows/ExportNotebooks.yml +++ b/.github/workflows/ExportNotebooks.yml @@ -8,12 +8,13 @@ on: - main workflow_dispatch: concurrency: - group: export + group: export-${{ github.ref }} cancel-in-progress: true jobs: build-and-deploy: runs-on: ubuntu-latest + timeout-minutes: 360 steps: - name: Checkout source uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -23,23 +24,17 @@ jobs: with: version: "1.11.2" - - name: ⏱ Cache notebook states - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 - with: - path: _cache - key: ${{ runner.os }}-pluto_state_cache-v3-${{ hashFiles('**/Project.toml', '**/Manifest.toml') }}-${{ github.run_id }} - restore-keys: | - ${{ runner.os }}-pluto_state_cache-v3-${{ hashFiles('**/Project.toml', '**/Manifest.toml') }} - - name: ⏱ Cache .julia uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: path: ~/.julia - key: ${{ runner.os }}-dotjulia-v1-${{ hashFiles('**/Project.toml', '**/Manifest.toml') }}-${{ github.run_id }} + key: ${{ runner.os }}-dotjulia-v1-${{ hashFiles('pluto-deployment-environment/*.toml') }}-${{ github.run_id }} restore-keys: | - ${{ runner.os }}-dotjulia-v1-${{ hashFiles('**/Project.toml', '**/Manifest.toml') }} + ${{ runner.os }}-dotjulia-v1-${{ hashFiles('pluto-deployment-environment/*.toml') }} - name: 🪴 Generate site + env: + GKSwstype: "100" run: julia --project=pluto-deployment-environment -e ' import Pkg; Pkg.instantiate(); @@ -70,4 +65,4 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} branch: gh-pages folder: _site - target-folder: "previews/PR${{ github.event.number }}" # The website preview is going to be stored in the previews subfolder \ No newline at end of file + target-folder: "previews/PR${{ github.event.number }}" # The website preview is going to be stored in the previews subfolder diff --git a/.github/workflows/KeepCacheFresh.yml b/.github/workflows/KeepCacheFresh.yml deleted file mode 100644 index 28007bc6..00000000 --- a/.github/workflows/KeepCacheFresh.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Keep caches fresh -on: - schedule: - - cron: "5 4 1/4 * *" # every 4 days - -jobs: - build-and-deploy: - runs-on: ubuntu-latest - steps: - - name: ⏱ Cache notebook states - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 - with: - path: _cache - key: ${{ runner.os }}-pluto_state_cache-v3-${{ hashFiles('**/Project.toml', '**/Manifest.toml') }}-${{ github.run_id }} - restore-keys: | - ${{ runner.os }}-pluto_state_cache-v3-${{ hashFiles('**/Project.toml', '**/Manifest.toml') }} - - - name: ⏱ Cache .julia - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 - with: - path: ~/.julia - key: ${{ runner.os }}-dotjulia-v1-${{ hashFiles('**/Project.toml', '**/Manifest.toml') }}-${{ github.run_id }} - restore-keys: | - ${{ runner.os }}-dotjulia-v1-${{ hashFiles('**/Project.toml', '**/Manifest.toml') }} - diff --git a/.gitignore b/.gitignore index a4ad7a74..3106e164 100644 --- a/.gitignore +++ b/.gitignore @@ -343,4 +343,361 @@ Manifest.toml # images are by default ignored, force add them if you must *.png *.jpg -*.svg \ No newline at end of file +*.svg + +# Local build output from develop.jl / generate.jl — the site is built by CI +_site/ +generation_report.html +build.log + +## REPO + +figures/* +notebook-checks/src +notebook-checks/build + +## Julia +Manifest.toml + + +## Core latex/pdflatex auxiliary files: +*.aux +*.lof +*.log +*.lot +*.fls +*.out +*.toc +*.fmt +*.fot +*.cb +*.cb2 +.*.lb + +## Intermediate documents: +*.dvi +*.xdv +*-converted-to.* +# these rules might exclude image files for figures etc. +*.ps +*.eps +*.pdf + +## Generated if empty string is given at "Please type another file name for output:" +.pdf + +## Bibliography auxiliary files (bibtex/biblatex/biber): +*.bbl +*.bcf +*.blg +*-blx.aux +*-blx.bib +*.run.xml + +## Build tool auxiliary files: +*.fdb_latexmk +*.synctex +*.synctex(busy) +*.synctex.gz +*.synctex.gz(busy) +*.pdfsync + +## Build tool directories for auxiliary files +# latexrun +latex.out/ + +## Auxiliary and intermediate files from other packages: +# algorithms +*.alg +*.loa + +# achemso +acs-*.bib + +# amsthm +*.thm + +# beamer +*.nav +*.pre +*.snm +*.vrb + +# changes +*.soc + +# comment +*.cut + +# cprotect +*.cpt + +# elsarticle (documentclass of Elsevier journals) +*.spl + +# endnotes +*.ent + +# fixme +*.lox + +# feynmf/feynmp +*.mf +*.mp +*.t[1-9] +*.t[1-9][0-9] +*.tfm + +#(r)(e)ledmac/(r)(e)ledpar +*.end +*.?end +*.[1-9] +*.[1-9][0-9] +*.[1-9][0-9][0-9] +*.[1-9]R +*.[1-9][0-9]R +*.[1-9][0-9][0-9]R +*.eledsec[1-9] +*.eledsec[1-9]R +*.eledsec[1-9][0-9] +*.eledsec[1-9][0-9]R +*.eledsec[1-9][0-9][0-9] +*.eledsec[1-9][0-9][0-9]R + +# glossaries +*.acn +*.acr +*.glg +*.glo +*.gls +*.glsdefs +*.lzo +*.lzs +*.slg +*.slo +*.sls + +# uncomment this for glossaries-extra (will ignore makeindex's style files!) +# *.ist + +# gnuplot +*.gnuplot +*.table + +# gnuplottex +*-gnuplottex-* + +# gregoriotex +*.gaux +*.glog +*.gtex + +# htlatex +*.4ct +*.4tc +*.idv +*.lg +*.trc +*.xref + +# hyperref +*.brf + +# knitr +*-concordance.tex +# TODO Uncomment the next line if you use knitr and want to ignore its generated tikz files +# *.tikz +*-tikzDictionary + +# listings +*.lol + +# luatexja-ruby +*.ltjruby + +# makeidx +*.idx +*.ilg +*.ind + +# minitoc +*.maf +*.mlf +*.mlt +*.mtc[0-9]* +*.slf[0-9]* +*.slt[0-9]* +*.stc[0-9]* + +# minted +_minted* +*.pyg + +# morewrites +*.mw + +# newpax +*.newpax + +# nomencl +*.nlg +*.nlo +*.nls + +# pax +*.pax + +# pdfpcnotes +*.pdfpc + +# sagetex +*.sagetex.sage +*.sagetex.py +*.sagetex.scmd + +# scrwfile +*.wrt + +# svg +svg-inkscape/ + +# sympy +*.sout +*.sympy +sympy-plots-for-*.tex/ + +# pdfcomment +*.upa +*.upb + +# pythontex +*.pytxcode +pythontex-files-*/ + +# tcolorbox +*.listing + +# thmtools +*.loe + +# TikZ & PGF +*.dpth +*.md5 +*.auxlock + +# titletoc +*.ptc + +# todonotes +*.tdo + +# vhistory +*.hst +*.ver + +# easy-todo +*.lod + +# xcolor +*.xcp + +# xmpincl +*.xmpi + +# xindy +*.xdy + +# xypic precompiled matrices and outlines +*.xyc +*.xyd + +# endfloat +*.ttt +*.fff + +# Latexian +TSWLatexianTemp* + +## Editors: +# WinEdt +*.bak +*.sav + +# Texpad +.texpadtmp + +# LyX +*.lyx~ + +# Kile +*.backup + +# gummi +.*.swp + +# KBibTeX +*~[0-9]* + +# TeXnicCenter +*.tps + +# auto folder when using emacs and auctex +./auto/* +*.el + +# expex forward references with \gathertags +*-tags.tex + +# standalone packages +*.sta + +# Makeindex log files +*.lpz + +# xwatermark package +*.xwm + +# REVTeX puts footnotes in the bibliography by default, unless the nofootinbib +# option is specified. Footnotes are the stored in a file with suffix Notes.bib. +# Uncomment the next line to have this generated file ignored. +#*Notes.bib + +# vscode +.vscode/ + +examples/launch_pluto/Manifest.toml + +# Files generated by invoking Julia with --code-coverage +*.jl.cov +*.jl.*.cov + +# Files generated by invoking Julia with --track-allocation +*.jl.mem + +# System-specific files and directories generated by the BinaryProvider and BinDeps packages +# They contain absolute paths specific to the host computer, and so should not be committed +deps/deps.jl +deps/build.log +deps/downloads/ +deps/usr/ +deps/src/ + +# Build artifacts for creating documentation generated by the Documenter package +docs/build/ +docs/site/ + +# File generated by Pkg, the package manager, based on a corresponding Project.toml +# It records a fixed state of all packages used by the project. As such, it should not be +# committed for packages, but should be committed for applications that require a static +# environment. +Manifest.toml + +# images are by default ignored, force add them if you must +*.png +*.jpg +*.svg +# The site build does not execute notebooks anymore, so there is no notebook +# output cache to seed. _cache is a local leftover and must stay out of the repo. +_cache/ +_site/ +generation_report.html diff --git a/PlutoPages.jl b/PlutoPages.jl index 5cd53fde..b8a50f98 100644 --- a/PlutoPages.jl +++ b/PlutoPages.jl @@ -183,6 +183,40 @@ md""" ## `.jl`: PlutoSliderServer.jl """ +# ╔═╡ c83871e0-90ec-11f1-9518-4946ff967f7b +""" +Load a Pluto notebook **without running it**, and return the packed statefile. + +Nothing from the notebook is executed: no notebook process is started and the +notebook package environment is not instantiated. Pluto still prerenders +text-only cells (i.e. plain `md"..."` cells) in this process, so the page shows +rendered prose plus the source code of every other cell. Visitors run the +notebook themselves with the "Edit or run this notebook" button. +""" +function unrun_notebook_state(absolute_path::String)::Vector{UInt8} + session = Pluto.ServerSession() + session.options.server.disable_writing_notebook_files = true + session.options.server.launch_browser = false + # NOTE: do not set `run_notebook_on_load = false` here: that code path is + # unsupported and broken in Pluto 0.20.13 (`DEFAULT_PRECEDENCE_HEURISTIC` is + # not qualified in Run.jl). `execution_allowed=false` below is enough. + + notebook = Logging.with_logger(Logging.NullLogger()) do + Pluto.SessionActions.open(session, absolute_path; + run_async=false, + execution_allowed=false, # no worker process, no Pkg instantiate + ) + end + + state = Pluto.notebook_to_js(notebook) + delete!(state, "status_tree") + delete!(session.notebooks, notebook.notebook_id) + + io = IOBuffer() + Pluto.pack(io, state) + take!(io) +end + # ╔═╡ bb905046-59b7-4da6-97ad-dbb9055d823a const pluto_deploy_settings = PlutoSliderServer.get_configuration(PlutoSliderServer.default_config_path()) @@ -465,12 +499,6 @@ const output_dir = mkpath(joinpath(@__DIR__, "_site")) # ╔═╡ 37b2cecc-e4c7-4b80-b7d9-71c68f3c0339 -# ╔═╡ 7a95681a-df77-408f-919a-2bee5afd7777 -""" -This directory can be used to store cache files that are persisted between builds. Currently used as PlutoSliderServer.jl cache. -""" -const cache_dir = mkpath(joinpath(@__DIR__, "_cache")) - # ╔═╡ f3d225b8-b9a5-4639-97eb-7785b1a78f5a md""" ## Running a dev web server @@ -495,11 +523,11 @@ md""" md""" ## Running the templates -(This can take a while if you are running this for the first time with an empty cache.) +Notebooks are **not executed**: they are loaded, text-only cells are prerendered, and the result is embedded as a static Pluto editor. So there is no notebook output cache to warm up. """ # ╔═╡ f700357f-e21c-4d23-b56c-be4f9c90465f -const NUM_PARALLEL_WORKERS = 4 +const NUM_PARALLEL_WORKERS = 2 # ╔═╡ aaad71bd-5425-4783-952c-82e4d4fa7bb8 md""" @@ -678,38 +706,37 @@ end # ╔═╡ e2510a44-df48-4c05-9453-8822deadce24 function template_handler( - ::Val{Symbol(".jl")}, + ::Val{Symbol(".jl")}, input::TemplateInput )::TemplateOutput if Pluto.is_pluto_notebook(input.absolute_path) - temp_out = mktempdir() - Logging.with_logger(Logging.NullLogger()) do - PlutoSliderServer.export_notebook( - input.absolute_path; - Export_create_index=false, - Export_cache_dir=cache_dir, - Export_baked_state=false, - Export_baked_notebookfile=false, - Export_output_dir=temp_out, - ) - end - d = readdir(temp_out) + name = basename(Pluto.without_pluto_file_extension(input.absolute_path)) + + statefile_contents = unrun_notebook_state(input.absolute_path) - statefile = find(contains("state") ∘ last ∘ splitext, d) - notebookfile = find(!contains("html") ∘ last ∘ splitext, d) + reg_s = register_asset(statefile_contents, "$(name).plutostate") + reg_n = register_asset(input.contents, basename(input.absolute_path)) - reg_s = register_asset(read(joinpath(temp_out, statefile)), statefile) - reg_n = register_asset(read(joinpath(temp_out, notebookfile)), notebookfile) + # Only advertise Binder when it is actually configured. Our notebooks + # activate `pluto-deployment-environment` by a relative path, which does + # not exist in a Binder session, so the button would always fail. + binder_url = pluto_deploy_settings.Export.offer_binder ? + something(pluto_deploy_settings.Export.binder_url, Pluto.default_binder_url) : + nothing # TODO these relative paths can't be right... h = @htl """ - +

This page shows the notebook without its output: the code is not executed when the website is built.

+

To run it, and to get the plots and the interactive elements: install Julia, clone the course repository, run julia launch_pluto.jl in it, and open this notebook in Pluto. You can also download the notebook file.

+ + """ @@ -745,6 +772,7 @@ template_results = let # let's go! running all the template handlers progressmap_async(allfiles; ntasks=NUM_PARALLEL_WORKERS) do f + println("Processing: ", f) absolute_path = joinpath(dir, f) input = TemplateInput(; @@ -969,6 +997,7 @@ end # ╠═7717e24f-62ee-4852-9dec-d09b734d0693 # ╠═692c1e0b-07e1-41b3-abcd-2156bda65b41 # ╟─adb1ddac-d992-49ca-820f-e1ed8ca33bf8 +# ╠═c83871e0-90ec-11f1-9518-4946ff967f7b # ╠═e2510a44-df48-4c05-9453-8822deadce24 # ╠═bb905046-59b7-4da6-97ad-dbb9055d823a # ╠═b638df55-fd74-4ae8-bdbd-ec7b18214b40 @@ -1001,7 +1030,6 @@ end # ╟─d314ab46-b866-44c6-bfca-9a413bc06514 # ╠═e01ebbab-dc9a-4aaf-ae16-200d171fcbd9 # ╠═37b2cecc-e4c7-4b80-b7d9-71c68f3c0339 -# ╟─7a95681a-df77-408f-919a-2bee5afd7777 # ╟─f3d225b8-b9a5-4639-97eb-7785b1a78f5a # ╠═c3a495c1-3e1f-42a1-ac08-8dc0b9175fe9 # ╠═3b2d1919-41d9-4bba-9774-c8497bba5003 diff --git a/pluto-deployment-environment/PlutoDeployment.toml b/pluto-deployment-environment/PlutoDeployment.toml index b0eb859d..e9ba9d17 100644 --- a/pluto-deployment-environment/PlutoDeployment.toml +++ b/pluto-deployment-environment/PlutoDeployment.toml @@ -1,7 +1,11 @@ [Export] baked_state = false baked_notebookfile = false -offer_binder = true +# Binder is disabled: our notebooks activate `pluto-deployment-environment` by a +# relative path, which does not exist in a Binder session, so every Binder run +# fails with "Package ... not found in current path". Re-enable this together +# with a Binder image that ships the environment. +offer_binder = false ignore_cache = [ "index.jl", ] diff --git a/src/_includes/layout.jlhtml b/src/_includes/layout.jlhtml index e8861e7f..701baf6b 100644 --- a/src/_includes/layout.jlhtml +++ b/src/_includes/layout.jlhtml @@ -11,6 +11,11 @@ $(begin r"" r"" r"" + # Pluto's default head sets a Binder URL for every page. Our + # notebooks cannot run on Binder (they activate a relative project + # path), and `` falls back to this global when it has + # no `binder_url` attribute, so the button would come back. + r"window\.pluto_binder_url = [^\n]*" ]; init=m[1]) do s,r replace(s, r => "") end |> HTML diff --git a/src/assets/styles/layout.css b/src/assets/styles/layout.css index e66798e6..8dfc1b1c 100644 --- a/src/assets/styles/layout.css +++ b/src/assets/styles/layout.css @@ -304,3 +304,36 @@ main { background-position: 0 0em; background-size: 2px auto; } + +/* NOTEBOOK NOTICE */ + +.run-notebook-notice { + max-width: 700px; + margin: 1rem auto 0 auto; + padding: 0.8rem 1rem; + border: 2px solid rgb(165, 213, 235); + border-radius: 0.4rem; + background: rgba(165, 213, 235, 0.15); + font-size: 0.9rem; +} + +.run-notebook-notice p { + margin: 0.4rem 0; +} + +.run-notebook-notice code { + padding: 0.1em 0.3em; + border-radius: 0.2em; + background: rgba(0, 0, 0, 0.07); +} + +@media (prefers-color-scheme: dark) { + .run-notebook-notice { + border-color: rgb(82, 118, 135); + background: rgba(82, 118, 135, 0.25); + } + + .run-notebook-notice code { + background: rgba(255, 255, 255, 0.12); + } +} diff --git a/website_maintenance.md b/website_maintenance.md index cb3ddadf..4ffa2217 100644 --- a/website_maintenance.md +++ b/website_maintenance.md @@ -32,11 +32,13 @@ Besides small inline values, you can also write big code blocks, with `$(begin . ## Pluto notebooks -Pluto notebooks will be rendered to HTML and included in the page. What you see is what you get! +Pluto notebooks are included in the page, but they are **not executed** during the site build. -On a separate system, we are running a PlutoSliderServer that is synchronized to the `Fall23` brach. This makes our notebooks interactive! +Each notebook is loaded, its plain markdown cells are prerendered (this happens inside the build process, no notebook is started), and the result is embedded as a static Pluto editor. So a page shows the prose and the source code of every cell, but no cell outputs: no plots, no numbers, no interactive sliders. -Notebook outputs are **cached** (for a long time) by the file hash. This means that a notebook file will only ever run once, which makes it much faster to work on the website. If you need to re-run your notebook, add a space somewhere in the code :) +Visitors run the notebook themselves with the **"Edit or run this notebook"** button in the top right: on Binder, or on their own computer with the repository checked out. + +Because nothing is executed, there is **no notebook output cache** anymore. Editing a notebook is picked up immediately — you never have to invalidate anything, and a full site build takes a few minutes instead of hours. ## `.css`, `.html`, `.gif`, etc @@ -77,20 +79,22 @@ For `.jlhtml`, we still need to figure something out 😄. Open this repository in VS Code, and install the recommended extensions. -To start running the development server, open the VS Code *command palette* (press `Cmd+Shift+P`), and search for **`Tasks: Run Task`**, then **`PlutoPages: run development server`**. The first run can take some time, as it builds up the notebook outputs cache. Leave it running. +To start running the development server, open the VS Code *command palette* (press `Cmd+Shift+P`), and search for **`Tasks: Run Task`**, then **`PlutoPages: run development server`**. The first run can take some time, as it precompiles the packages in `pluto-deployment-environment`. Leave it running. + +Use the same Julia version as the CI workflow (`1.11.2`, the version that `pluto-deployment-environment/Manifest.toml` was resolved with). With juliaup: `juliaup override set 1.11.2` inside this folder. This will start two things in parallel: the PlutoPages.jl notebook (which generates the website), and a static file server (with Deno_jll). It will open two tabs in your browser: one is the generation dashboard (PlutoPages), the other is the current site preview (Deno_jll). Whenever you edit a file, PlutoPages will automatically regenerate! Refresh your browser tab. If it does not pick up the change, go to the generation dashboard and click the "Read input files again" button. -This workflow is recommended for writing static content, styles, and for site maintenance. But for writing Pluto notebooks, it's best to prepare the notebook first, and then run the site (because it re-runs the entire notebook on any change). +This workflow is recommended for writing static content, styles, and for site maintenance. Notebooks are never executed by the site build, so editing one is cheap — but it also means you have to check the notebook's output in Pluto itself. ## Developing PlutoPages itself You need to manually run the notebook with Pluto: 1. Go to this folder, and run `julia --project=pluto-deployment-environment`. Then `import Pkg; Pkg.instantiate();`. -1. `import Pluto; Pluto.run()` and open the `PlutoPages.jl` notebook in this repository. The first run can take some time, as it builds up the notebook outputs cache. Leave it running. +1. `import Pluto; Pluto.run()` and open the `PlutoPages.jl` notebook in this repository. The first run can take some time, as it precompiles packages. Leave it running. 2. In a second terminal, go to this folder, and run `julia --project=pluto-deployment-environment`, then: ```julia import Deno_jll