Improved script detection and validation on CI for examples.

.github/workflows/build_site.yml

@@ -34,6 +34,9 @@ jobs:
           python-version: 3.10.5
           architecture: x64
 
+      - name: Test example script resolution
+        run: python -m unittest test_example_scripts.py
+
       - name: Build Jekyll site
         run: bundle exec jekyll build
 

example_scripts.py

@@ -0,0 +1,104 @@
+import os
+import shutil
+from pathlib import Path, PurePosixPath
+
+
+SCRIPT_EXTENSIONS = {
+    ".script",
+    ".gui_script",
+    ".lua",
+    ".vp",
+    ".fp",
+    ".cp",
+    ".glsl",
+    ".render_script",
+}
+
+IGNORED_SCRIPT_DIRS = {
+    ".deps",
+    ".git",
+    ".internal",
+    "build",
+    "builtins",
+    "js-web",
+    "node_modules",
+}
+
+
+def split_example_scripts(scripts):
+    if isinstance(scripts, str):
+        return [script.strip() for script in scripts.split(",") if script.strip()]
+    if isinstance(scripts, list):
+        return [str(script).strip() for script in scripts if str(script).strip()]
+    return []
+
+
+def find_example_scripts(project_dir):
+    project_dir = Path(project_dir)
+    scripts = []
+    if not project_dir.exists():
+        return scripts
+
+    for root, dirnames, filenames in os.walk(project_dir):
+        dirnames[:] = [dirname for dirname in dirnames if dirname not in IGNORED_SCRIPT_DIRS]
+        root_path = Path(root)
+        for filename in filenames:
+            path = root_path / filename
+            if path.suffix in SCRIPT_EXTENSIONS:
+                scripts.append(path.relative_to(project_dir).as_posix())
+
+    return sorted(scripts)
+
+
+def resolve_example_script(script, available_scripts):
+    path = PurePosixPath(script)
+    if (
+        not script
+        or "\\" in script
+        or path.is_absolute()
+        or str(path) != script
+        or any(part in {"", ".", ".."} for part in path.parts)
+    ):
+        raise ValueError(
+            "scripts entry must be a file name or normalized project-relative path, got '{}'".format(script)
+        )
+
+    if len(path.parts) > 1:
+        if script not in available_scripts:
+            raise ValueError("scripts entry '{}' does not exist in the project".format(script))
+        return script
+
+    matches = [candidate for candidate in available_scripts if PurePosixPath(candidate).name == script]
+    if not matches:
+        raise ValueError("scripts entry '{}' does not exist in the project".format(script))
+    if len(matches) > 1:
+        raise ValueError(
+            "scripts entry '{}' is ambiguous; use an exact project-relative path: {}".format(
+                script, ", ".join(matches)
+            )
+        )
+    return matches[0]
+
+
+def resolve_example_scripts(scripts, available_scripts):
+    return [
+        (script, resolve_example_script(script, available_scripts))
+        for script in split_example_scripts(scripts)
+    ]
+
+
+def example_include_name(script):
+    path = PurePosixPath(script)
+    suffix = path.suffix
+    include_file = "{}_{}.md".format(path.stem, suffix.lstrip("."))
+    return str(path.with_name(include_file))
+
+
+def copy_example_scripts(project_dir, includes_dir, resolved_scripts):
+    project_dir = Path(project_dir)
+    includes_dir = Path(includes_dir)
+    for declared_script, resolved_script in resolved_scripts:
+        source = project_dir.joinpath(*PurePosixPath(resolved_script).parts)
+        target = includes_dir.joinpath(*PurePosixPath(example_include_name(declared_script)).parts)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(source, target)

llms.py

@@ -2,6 +2,7 @@
 import os
 import re
 
+from example_scripts import example_include_name, split_example_scripts
 from utils import list_files, read_as_json, read_as_string, rmtree, write_as_string
 
 
@@ -738,17 +739,8 @@ def generate_llms_examples():
             content = rewrite_example_links(content, path)
             content = clean_markdown(content)
             entry["llms_scripts"] = []
-            scripts_value = entry.get("scripts") or ""
-            if isinstance(scripts_value, list):
-                scripts_list = scripts_value
-            else:
-                scripts_list = [s.strip() for s in str(scripts_value).split(",") if s.strip()]
-            for script_name in scripts_list:
-                include_name = script_name
-                include_name = include_name.replace(".script", "_script.md")
-                include_name = include_name.replace(".gui_script", "_gui_script.md")
-                include_name = include_name.replace(".vp", "_vp.md")
-                include_name = include_name.replace(".fp", "_fp.md")
+            for script_name in split_example_scripts(entry.get("scripts")):
+                include_name = example_include_name(script_name)
                 include_path = os.path.join("_includes", "examples", path, include_name)
                 if not os.path.exists(include_path):
                     continue

test_example_scripts.py

@@ -0,0 +1,98 @@
+import tempfile
+import unittest
+from pathlib import Path
+
+from example_scripts import (
+    copy_example_scripts,
+    example_include_name,
+    find_example_scripts,
+    resolve_example_script,
+    resolve_example_scripts,
+)
+
+
+class ExampleScriptResolutionTests(unittest.TestCase):
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.project_dir = Path(self.temp_dir.name)
+
+    def tearDown(self):
+        self.temp_dir.cleanup()
+
+    def add_script(self, relative_path):
+        path = self.project_dir / relative_path
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text("function init(self) end\n", encoding="utf-8")
+
+    def test_unique_filename_is_found_anywhere_in_project(self):
+        self.add_script("main/scroll_manager/scroll_item.script")
+        available = find_example_scripts(self.project_dir)
+
+        self.assertEqual(
+            resolve_example_script("scroll_item.script", available),
+            "main/scroll_manager/scroll_item.script",
+        )
+
+    def test_exact_path_is_supported_but_suffix_path_is_not(self):
+        self.add_script("main/scroll_manager/scroll_item.script")
+        available = find_example_scripts(self.project_dir)
+
+        self.assertEqual(
+            resolve_example_script("main/scroll_manager/scroll_item.script", available),
+            "main/scroll_manager/scroll_item.script",
+        )
+        with self.assertRaisesRegex(ValueError, "does not exist"):
+            resolve_example_script("scroll_manager/scroll_item.script", available)
+
+    def test_ambiguous_filename_requires_an_exact_path(self):
+        self.add_script("main/first/controller.script")
+        self.add_script("main/second/controller.script")
+        available = find_example_scripts(self.project_dir)
+
+        with self.assertRaisesRegex(
+            ValueError,
+            "main/first/controller.script, main/second/controller.script",
+        ):
+            resolve_example_script("controller.script", available)
+
+    def test_generated_and_dependency_directories_are_ignored(self):
+        self.add_script("main/controller.script")
+        self.add_script("build/controller.script")
+        self.add_script(".internal/lib/controller.script")
+        self.add_script("node_modules/package/controller.script")
+
+        self.assertEqual(find_example_scripts(self.project_dir), ["main/controller.script"])
+
+    def test_declared_path_controls_nested_include_name(self):
+        self.add_script("main/scroll_manager/scroll_item.script")
+        available = find_example_scripts(self.project_dir)
+        includes_dir = self.project_dir / "includes"
+
+        resolved = resolve_example_scripts("scroll_item.script", available)
+        self.assertEqual(resolved, [("scroll_item.script", "main/scroll_manager/scroll_item.script")])
+        self.assertEqual(example_include_name("scroll_item.script"), "scroll_item_script.md")
+        self.assertEqual(
+            example_include_name("main/scroll_manager/scroll_item.script"),
+            "main/scroll_manager/scroll_item_script.md",
+        )
+
+        copy_example_scripts(self.project_dir, includes_dir, resolved)
+        self.assertTrue((includes_dir / "scroll_item_script.md").is_file())
+
+        nested = resolve_example_scripts("main/scroll_manager/scroll_item.script", available)
+        copy_example_scripts(self.project_dir, includes_dir, nested)
+        self.assertTrue((includes_dir / "main/scroll_manager/scroll_item_script.md").is_file())
+
+    def test_unsafe_or_non_normalized_paths_are_rejected(self):
+        for script in (
+            "/main/controller.script",
+            "./main/controller.script",
+            "main/../controller.script",
+            "main\\controller.script",
+        ):
+            with self.subTest(script=script), self.assertRaisesRegex(ValueError, "normalized"):
+                resolve_example_script(script, [])
+
+
+if __name__ == "__main__":
+    unittest.main()

update.py

@@ -20,6 +20,11 @@
 from scripts import dedupe_examples_wasm
 from argparse import ArgumentParser
 from contextlib import contextmanager
+from example_scripts import (
+    copy_example_scripts,
+    find_example_scripts,
+    resolve_example_scripts,
+)
 
 
 SHA1 = {}
@@ -32,8 +37,6 @@
 
 EXAMPLES_DEFOLD_CHANNEL = "alpha"
 EXAMPLES_BUILD_SERVER = "https://build-stage.defold.com/"
-EXAMPLE_CODE_FILE_PATTERNS = "*.script|*.gui_script|*.lua|*.vp|*.fp|*.cp|*.glsl|*.render_script"
-
 ASSETINDEX_JSON = os.path.join("_data", "assetindex.json")
 GAMES_JSON = os.path.join("_data", "games.json")
 AUTHORINDEX_JSON = os.path.join("_data", "authorindex.json")
@@ -207,17 +210,6 @@ def find_files(root_dir, file_patterns):
     matches.sort()
     return matches
 
-def example_include_name(filename):
-    file, ext = os.path.splitext(os.path.basename(filename))
-    return file + "_" + ext.replace(".", "") + ".md"
-
-def split_example_scripts(scripts):
-    if isinstance(scripts, str):
-        return [s.strip() for s in scripts.split(",") if s.strip()]
-    if isinstance(scripts, list):
-        return [str(s).strip() for s in scripts if str(s).strip()]
-    return []
-
 def write_as_json(filename, data, ensure_ascii=True):
     with open(filename, "w") as f:
         json.dump(data, f, indent=4, sort_keys=True, ensure_ascii=ensure_ascii)
@@ -1052,6 +1044,15 @@ def process_examples(download = False, examples_ref = "master", changed_examples
                 if os.path.isfile(example_src_dir):
                     continue
 
+                md_file = os.path.join(example_src_dir, "example.md")
+                fm = load_frontmatter(md_file)
+                available_scripts = find_example_scripts(example_src_dir)
+                try:
+                    resolved_scripts = resolve_example_scripts(fm.get("scripts"), available_scripts)
+                except ValueError as error:
+                    print("ERROR: {}: {}".format(example_path, error))
+                    sys.exit(1)
+
                 print("..processing %s" % example)
                 if rebuild:
                     print("...building %s" % example)
@@ -1078,8 +1079,6 @@ def process_examples(download = False, examples_ref = "master", changed_examples
                     os.makedirs(example_dst_dir, exist_ok=True)
 
                 print("...parsing example.md")
-                md_file = os.path.join(example_src_dir, "example.md")
-                fm = load_frontmatter(md_file)
                 fm["category"] = category
                 fm["path"] = example_path
                 fm["layout"] = "example"
@@ -1102,23 +1101,9 @@ def process_examples(download = False, examples_ref = "master", changed_examples
                             fm["twitter_image"] = image_path
 
                 print("...copying example scripts")
-                os.makedirs(os.path.join(includes_dir, category, example), exist_ok=True)
-                copied_scripts = set()
-                for script in find_files(os.path.join(example_src_dir, "example"), EXAMPLE_CODE_FILE_PATTERNS):
-                    include_name = example_include_name(script)
-                    copied_scripts.add(include_name)
-                    tgt = os.path.join(includes_dir, category, example, include_name)
-                    shutil.copyfile(script, tgt)
-
-                frontmatter_scripts = split_example_scripts(fm.get("scripts"))
-                if frontmatter_scripts:
-                    missing_scripts = []
-                    for script in frontmatter_scripts:
-                        if script != os.path.basename(script) or example_include_name(script) not in copied_scripts:
-                            missing_scripts.append(script)
-                    if missing_scripts:
-                        print("ERROR: {} references missing example script(s): {}".format(fm["path"], ", ".join(missing_scripts)))
-                        sys.exit(1)
+                example_includes_dir = os.path.join(includes_dir, category, example)
+                os.makedirs(example_includes_dir, exist_ok=True)
+                copy_example_scripts(example_src_dir, example_includes_dir, resolved_scripts)
 
                 examplesindex.append(fm)
                 replace_frontmatter(md_file, fm)