From 045c72576de9495590d19c3fbc5adb1bad81ac3c Mon Sep 17 00:00:00 2001 From: Maximilian Heidenreich Date: Mon, 21 Aug 2023 11:51:26 +0200 Subject: [PATCH 1/3] adds broken link check integration --- astro.config.mjs | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/astro.config.mjs b/astro.config.mjs index bc40b5db..ae93ef71 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -17,9 +17,45 @@ const headingIcon = (node) => { return [e]; }; +/** + * Small astro integration to scan all generated pages links & + * match them with known routes to find (possibly) broken links. + * @returns + */ +const brokenLinkChecker = () => { + return { + name: "broken-link-checker", + hooks: { + "astro:build:done": async ({ routes, pages }) => { + pages = pages.map( + (e) => `/${e.pathname.endsWith("/") ? e.pathname.slice(0, -1) : e.pathname}` + ); + + console.log("Results for possibly broken links:"); + for (const route of routes) { + const fPath = route.distURL.pathname; + const content = await fs.readFile(fPath, "utf-8"); + + const pageLinks = Array.from(content.matchAll(/]*\bhref="([^#"][^"]*)"/g), (m) => m[1]) + .filter(e => !["/discovery", "/blog"].includes(e)) + .filter(l => !l.startsWith("https") && !l.startsWith("https") && !l.startsWith("mailto") && !l.startsWith("tel") && !l.startsWith("data")) + .map(l => (l.indexOf("#") > -1 ? l.slice(0, l.indexOf("#")) : l)) + .map(l => l.endsWith("/") ? l.slice(0, -1) : l ); + + for (const pLink of pageLinks) { + if (!pages.includes(pLink)) { + console.log(` - ${pLink} in ${fPath}`); + } + } + } + } + } + } +}; + // https://astro.build/config export default defineConfig({ - integrations: [astroExpressiveCode(), mdx(), svelte(), preact(), react()], + integrations: [astroExpressiveCode(), mdx(), svelte(), preact(), react(), brokenLinkChecker()], site: "https://deta.space/", base: "/", markdown: { From cbb931b408a5b4ce2a4b8552bee886d0e9d948eb Mon Sep 17 00:00:00 2001 From: Maximilian Heidenreich Date: Mon, 21 Aug 2023 12:01:19 +0200 Subject: [PATCH 2/3] fmt --- astro.config.mjs | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/astro.config.mjs b/astro.config.mjs index ae93ef71..698e781b 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -33,14 +33,28 @@ const brokenLinkChecker = () => { console.log("Results for possibly broken links:"); for (const route of routes) { + if (route.route.startsWith("/changelog") || route.route.startsWith("/manual") || route.route.startsWith("/migration")) continue; const fPath = route.distURL.pathname; const content = await fs.readFile(fPath, "utf-8"); - const pageLinks = Array.from(content.matchAll(/]*\bhref="([^#"][^"]*)"/g), (m) => m[1]) - .filter(e => !["/discovery", "/blog"].includes(e)) - .filter(l => !l.startsWith("https") && !l.startsWith("https") && !l.startsWith("mailto") && !l.startsWith("tel") && !l.startsWith("data")) - .map(l => (l.indexOf("#") > -1 ? l.slice(0, l.indexOf("#")) : l)) - .map(l => l.endsWith("/") ? l.slice(0, -1) : l ); + const pageLinks = Array.from( + content.matchAll(/]*\bhref="([^#"][^"]*)"/g), + (m) => m[1] + ) + .filter((e) => !["/discovery", "/blog"].includes(e)) + // https://deta.space/docs + .filter( + (l) => + l.startsWith("https://deta.space/docs") || + (!l.startsWith("https") && + !l.startsWith("https") && + !l.startsWith("mailto") && + !l.startsWith("tel") && + !l.startsWith("data")) + ) + .map((l) => (l.startsWith("https://deta.space/docs") ? l.slice(18) : l)) + .map((l) => (l.indexOf("#") > -1 ? l.slice(0, l.indexOf("#")) : l)) + .map((l) => (l.endsWith("/") ? l.slice(0, -1) : l)); for (const pLink of pageLinks) { if (!pages.includes(pLink)) { From f1f90e4e25ac98bad96e3ac7ea132f6f66fec57a Mon Sep 17 00:00:00 2001 From: Maximilian Heidenreich Date: Mon, 21 Aug 2023 14:08:13 +0200 Subject: [PATCH 3/3] fixes link checker logic & refactor to external file --- astro.config.mjs | 67 ++++++--------------------- src/utils/brokenLinkIntegration.mjs | 71 +++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 54 deletions(-) create mode 100644 src/utils/brokenLinkIntegration.mjs diff --git a/astro.config.mjs b/astro.config.mjs index 698e781b..df39a7bb 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -2,14 +2,16 @@ import { defineConfig } from "astro/config"; import svelte from "@astrojs/svelte"; import mdx from "@astrojs/mdx"; import fs from "fs/promises"; -import astroExpressiveCode from 'astro-expressive-code' +import path from "path"; +import astroExpressiveCode from "astro-expressive-code"; // TODO(@maxu): Remove these once we have them migrated! import react from "@astrojs/react"; import preact from "@astrojs/preact"; import rehypeSlug from "rehype-slug"; import rehypeAutolinkHeadings from "rehype-autolink-headings"; -import {generateIndexPages} from './src/generate-indexes' +import { generateIndexPages } from "./src/generate-indexes"; +import { brokenLinkIntegration } from "./src/utils/brokenLinkIntegration.mjs"; const headingIcon = (node) => { let e = new HTMLSpanElement(); @@ -17,59 +19,16 @@ const headingIcon = (node) => { return [e]; }; -/** - * Small astro integration to scan all generated pages links & - * match them with known routes to find (possibly) broken links. - * @returns - */ -const brokenLinkChecker = () => { - return { - name: "broken-link-checker", - hooks: { - "astro:build:done": async ({ routes, pages }) => { - pages = pages.map( - (e) => `/${e.pathname.endsWith("/") ? e.pathname.slice(0, -1) : e.pathname}` - ); - - console.log("Results for possibly broken links:"); - for (const route of routes) { - if (route.route.startsWith("/changelog") || route.route.startsWith("/manual") || route.route.startsWith("/migration")) continue; - const fPath = route.distURL.pathname; - const content = await fs.readFile(fPath, "utf-8"); - - const pageLinks = Array.from( - content.matchAll(/]*\bhref="([^#"][^"]*)"/g), - (m) => m[1] - ) - .filter((e) => !["/discovery", "/blog"].includes(e)) - // https://deta.space/docs - .filter( - (l) => - l.startsWith("https://deta.space/docs") || - (!l.startsWith("https") && - !l.startsWith("https") && - !l.startsWith("mailto") && - !l.startsWith("tel") && - !l.startsWith("data")) - ) - .map((l) => (l.startsWith("https://deta.space/docs") ? l.slice(18) : l)) - .map((l) => (l.indexOf("#") > -1 ? l.slice(0, l.indexOf("#")) : l)) - .map((l) => (l.endsWith("/") ? l.slice(0, -1) : l)); - - for (const pLink of pageLinks) { - if (!pages.includes(pLink)) { - console.log(` - ${pLink} in ${fPath}`); - } - } - } - } - } - } -}; - // https://astro.build/config export default defineConfig({ - integrations: [astroExpressiveCode(), mdx(), svelte(), preact(), react(), brokenLinkChecker()], + integrations: [ + astroExpressiveCode(), + mdx(), + svelte(), + preact(), + react(), + brokenLinkIntegration() + ], site: "https://deta.space/", base: "/", markdown: { @@ -79,7 +38,7 @@ export default defineConfig({ ] }, experimental: { - viewTransitions: true + viewTransitions: true }, vite: { build: { diff --git a/src/utils/brokenLinkIntegration.mjs b/src/utils/brokenLinkIntegration.mjs new file mode 100644 index 00000000..b30c5a3e --- /dev/null +++ b/src/utils/brokenLinkIntegration.mjs @@ -0,0 +1,71 @@ +import fs from "fs/promises"; +import path from "path"; + +/** + * Small astro integration to scan all generated pages links & + * match them with known routes to find (possibly) broken links. + * @returns + */ +export const brokenLinkIntegration = () => { + return { + name: "broken-link-checker", + hooks: { + "astro:build:done": async ({ pages }) => { + pages = pages.map( + (e) => `/${e.pathname.endsWith("/") ? e.pathname.slice(0, -1) : e.pathname}` + ); + // https://deta.space/docs/en/build/fundamentals/app-lifecycle + + + console.log("Results for possibly broken links:"); + + const searchPath = "./dist/docs"; + + let srcFiles = []; + async function getFiles(dir = "./") { + const entries = await fs.readdir(dir, { withFileTypes: true }); + + const files = entries + .filter((entry) => !entry.isDirectory()) + .map((file) => ({ ...file, dir: dir + file.name })); + + const folders = entries.filter((entry) => entry.isDirectory()); + for (const folder of folders) + files.push(...(await getFiles(`${path.join(dir, folder.name)}`))); + + return files; + } + srcFiles = (await getFiles(searchPath)).map((e) => path.join(e.path, e.name)); + + for (const srcFile of srcFiles) { + const content = await fs.readFile(srcFile, "utf-8"); + + const pageLinks = Array.from( + content.matchAll(/]*\bhref="([^#"][^"]*)"/g), + (m) => m[1] + ) + .filter((e) => !["/discovery", "/blog"].includes(e)) + .filter( + (l) => + l.startsWith("https://deta.space/docs") || + (!l.startsWith("https") && + !l.startsWith("http") && + !l.startsWith("mailto") && + !l.startsWith("tel") && + !l.startsWith("data") && + !l.startsWith("./")) + ) + .map((l) => (l.startsWith("https://deta.space/docs") ? l.slice(18) : l)) + .map((l) => (l.indexOf("#") > -1 ? l.slice(0, l.indexOf("#")) : l)) + .map((l) => (l.endsWith("/") ? l.slice(0, -1) : l)); + + for (const pLink of pageLinks) { + if (!pages.includes(pLink)) { + console.log(` - ${pLink} in ${srcFile}`); + } + } + } + } + } + }; +};