diff --git a/.github/workflows/build_site.yml b/.github/workflows/build_site.yml index 9e4d78c2c5..3870905eec 100644 --- a/.github/workflows/build_site.yml +++ b/.github/workflows/build_site.yml @@ -34,6 +34,9 @@ jobs: python-version: 3.10.5 architecture: x64 + - name: Test example script resolution + run: python -m unittest test_example_scripts.py + - name: Build Jekyll site run: bundle exec jekyll build diff --git a/example_scripts.py b/example_scripts.py new file mode 100644 index 0000000000..4c481201c6 --- /dev/null +++ b/example_scripts.py @@ -0,0 +1,104 @@ +import os +import shutil +from pathlib import Path, PurePosixPath + + +SCRIPT_EXTENSIONS = { + ".script", + ".gui_script", + ".lua", + ".vp", + ".fp", + ".cp", + ".glsl", + ".render_script", +} + +IGNORED_SCRIPT_DIRS = { + ".deps", + ".git", + ".internal", + "build", + "builtins", + "js-web", + "node_modules", +} + + +def split_example_scripts(scripts): + if isinstance(scripts, str): + return [script.strip() for script in scripts.split(",") if script.strip()] + if isinstance(scripts, list): + return [str(script).strip() for script in scripts if str(script).strip()] + return [] + + +def find_example_scripts(project_dir): + project_dir = Path(project_dir) + scripts = [] + if not project_dir.exists(): + return scripts + + for root, dirnames, filenames in os.walk(project_dir): + dirnames[:] = [dirname for dirname in dirnames if dirname not in IGNORED_SCRIPT_DIRS] + root_path = Path(root) + for filename in filenames: + path = root_path / filename + if path.suffix in SCRIPT_EXTENSIONS: + scripts.append(path.relative_to(project_dir).as_posix()) + + return sorted(scripts) + + +def resolve_example_script(script, available_scripts): + path = PurePosixPath(script) + if ( + not script + or "\\" in script + or path.is_absolute() + or str(path) != script + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise ValueError( + "scripts entry must be a file name or normalized project-relative path, got '{}'".format(script) + ) + + if len(path.parts) > 1: + if script not in available_scripts: + raise ValueError("scripts entry '{}' does not exist in the project".format(script)) + return script + + matches = [candidate for candidate in available_scripts if PurePosixPath(candidate).name == script] + if not matches: + raise ValueError("scripts entry '{}' does not exist in the project".format(script)) + if len(matches) > 1: + raise ValueError( + "scripts entry '{}' is ambiguous; use an exact project-relative path: {}".format( + script, ", ".join(matches) + ) + ) + return matches[0] + + +def resolve_example_scripts(scripts, available_scripts): + return [ + (script, resolve_example_script(script, available_scripts)) + for script in split_example_scripts(scripts) + ] + + +def example_include_name(script): + path = PurePosixPath(script) + suffix = path.suffix + include_file = "{}_{}.md".format(path.stem, suffix.lstrip(".")) + return str(path.with_name(include_file)) + + +def copy_example_scripts(project_dir, includes_dir, resolved_scripts): + project_dir = Path(project_dir) + includes_dir = Path(includes_dir) + for declared_script, resolved_script in resolved_scripts: + source = project_dir.joinpath(*PurePosixPath(resolved_script).parts) + target = includes_dir.joinpath(*PurePosixPath(example_include_name(declared_script)).parts) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) diff --git a/llms.py b/llms.py index 50c2ba5d1c..27624f3388 100644 --- a/llms.py +++ b/llms.py @@ -2,6 +2,7 @@ import os import re +from example_scripts import example_include_name, split_example_scripts from utils import list_files, read_as_json, read_as_string, rmtree, write_as_string @@ -738,17 +739,8 @@ def generate_llms_examples(): content = rewrite_example_links(content, path) content = clean_markdown(content) entry["llms_scripts"] = [] - scripts_value = entry.get("scripts") or "" - if isinstance(scripts_value, list): - scripts_list = scripts_value - else: - scripts_list = [s.strip() for s in str(scripts_value).split(",") if s.strip()] - for script_name in scripts_list: - include_name = script_name - include_name = include_name.replace(".script", "_script.md") - include_name = include_name.replace(".gui_script", "_gui_script.md") - include_name = include_name.replace(".vp", "_vp.md") - include_name = include_name.replace(".fp", "_fp.md") + for script_name in split_example_scripts(entry.get("scripts")): + include_name = example_include_name(script_name) include_path = os.path.join("_includes", "examples", path, include_name) if not os.path.exists(include_path): continue diff --git a/test_example_scripts.py b/test_example_scripts.py new file mode 100644 index 0000000000..ed013b0fb1 --- /dev/null +++ b/test_example_scripts.py @@ -0,0 +1,98 @@ +import tempfile +import unittest +from pathlib import Path + +from example_scripts import ( + copy_example_scripts, + example_include_name, + find_example_scripts, + resolve_example_script, + resolve_example_scripts, +) + + +class ExampleScriptResolutionTests(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.project_dir = Path(self.temp_dir.name) + + def tearDown(self): + self.temp_dir.cleanup() + + def add_script(self, relative_path): + path = self.project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("function init(self) end\n", encoding="utf-8") + + def test_unique_filename_is_found_anywhere_in_project(self): + self.add_script("main/scroll_manager/scroll_item.script") + available = find_example_scripts(self.project_dir) + + self.assertEqual( + resolve_example_script("scroll_item.script", available), + "main/scroll_manager/scroll_item.script", + ) + + def test_exact_path_is_supported_but_suffix_path_is_not(self): + self.add_script("main/scroll_manager/scroll_item.script") + available = find_example_scripts(self.project_dir) + + self.assertEqual( + resolve_example_script("main/scroll_manager/scroll_item.script", available), + "main/scroll_manager/scroll_item.script", + ) + with self.assertRaisesRegex(ValueError, "does not exist"): + resolve_example_script("scroll_manager/scroll_item.script", available) + + def test_ambiguous_filename_requires_an_exact_path(self): + self.add_script("main/first/controller.script") + self.add_script("main/second/controller.script") + available = find_example_scripts(self.project_dir) + + with self.assertRaisesRegex( + ValueError, + "main/first/controller.script, main/second/controller.script", + ): + resolve_example_script("controller.script", available) + + def test_generated_and_dependency_directories_are_ignored(self): + self.add_script("main/controller.script") + self.add_script("build/controller.script") + self.add_script(".internal/lib/controller.script") + self.add_script("node_modules/package/controller.script") + + self.assertEqual(find_example_scripts(self.project_dir), ["main/controller.script"]) + + def test_declared_path_controls_nested_include_name(self): + self.add_script("main/scroll_manager/scroll_item.script") + available = find_example_scripts(self.project_dir) + includes_dir = self.project_dir / "includes" + + resolved = resolve_example_scripts("scroll_item.script", available) + self.assertEqual(resolved, [("scroll_item.script", "main/scroll_manager/scroll_item.script")]) + self.assertEqual(example_include_name("scroll_item.script"), "scroll_item_script.md") + self.assertEqual( + example_include_name("main/scroll_manager/scroll_item.script"), + "main/scroll_manager/scroll_item_script.md", + ) + + copy_example_scripts(self.project_dir, includes_dir, resolved) + self.assertTrue((includes_dir / "scroll_item_script.md").is_file()) + + nested = resolve_example_scripts("main/scroll_manager/scroll_item.script", available) + copy_example_scripts(self.project_dir, includes_dir, nested) + self.assertTrue((includes_dir / "main/scroll_manager/scroll_item_script.md").is_file()) + + def test_unsafe_or_non_normalized_paths_are_rejected(self): + for script in ( + "/main/controller.script", + "./main/controller.script", + "main/../controller.script", + "main\\controller.script", + ): + with self.subTest(script=script), self.assertRaisesRegex(ValueError, "normalized"): + resolve_example_script(script, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/update.py b/update.py index e15ee98d58..b363a32f7c 100755 --- a/update.py +++ b/update.py @@ -20,6 +20,11 @@ from scripts import dedupe_examples_wasm from argparse import ArgumentParser from contextlib import contextmanager +from example_scripts import ( + copy_example_scripts, + find_example_scripts, + resolve_example_scripts, +) SHA1 = {} @@ -32,8 +37,6 @@ EXAMPLES_DEFOLD_CHANNEL = "alpha" EXAMPLES_BUILD_SERVER = "https://build-stage.defold.com/" -EXAMPLE_CODE_FILE_PATTERNS = "*.script|*.gui_script|*.lua|*.vp|*.fp|*.cp|*.glsl|*.render_script" - ASSETINDEX_JSON = os.path.join("_data", "assetindex.json") GAMES_JSON = os.path.join("_data", "games.json") AUTHORINDEX_JSON = os.path.join("_data", "authorindex.json") @@ -207,17 +210,6 @@ def find_files(root_dir, file_patterns): matches.sort() return matches -def example_include_name(filename): - file, ext = os.path.splitext(os.path.basename(filename)) - return file + "_" + ext.replace(".", "") + ".md" - -def split_example_scripts(scripts): - if isinstance(scripts, str): - return [s.strip() for s in scripts.split(",") if s.strip()] - if isinstance(scripts, list): - return [str(s).strip() for s in scripts if str(s).strip()] - return [] - def write_as_json(filename, data, ensure_ascii=True): with open(filename, "w") as f: json.dump(data, f, indent=4, sort_keys=True, ensure_ascii=ensure_ascii) @@ -1052,6 +1044,15 @@ def process_examples(download = False, examples_ref = "master", changed_examples if os.path.isfile(example_src_dir): continue + md_file = os.path.join(example_src_dir, "example.md") + fm = load_frontmatter(md_file) + available_scripts = find_example_scripts(example_src_dir) + try: + resolved_scripts = resolve_example_scripts(fm.get("scripts"), available_scripts) + except ValueError as error: + print("ERROR: {}: {}".format(example_path, error)) + sys.exit(1) + print("..processing %s" % example) if rebuild: print("...building %s" % example) @@ -1078,8 +1079,6 @@ def process_examples(download = False, examples_ref = "master", changed_examples os.makedirs(example_dst_dir, exist_ok=True) print("...parsing example.md") - md_file = os.path.join(example_src_dir, "example.md") - fm = load_frontmatter(md_file) fm["category"] = category fm["path"] = example_path fm["layout"] = "example" @@ -1102,23 +1101,9 @@ def process_examples(download = False, examples_ref = "master", changed_examples fm["twitter_image"] = image_path print("...copying example scripts") - os.makedirs(os.path.join(includes_dir, category, example), exist_ok=True) - copied_scripts = set() - for script in find_files(os.path.join(example_src_dir, "example"), EXAMPLE_CODE_FILE_PATTERNS): - include_name = example_include_name(script) - copied_scripts.add(include_name) - tgt = os.path.join(includes_dir, category, example, include_name) - shutil.copyfile(script, tgt) - - frontmatter_scripts = split_example_scripts(fm.get("scripts")) - if frontmatter_scripts: - missing_scripts = [] - for script in frontmatter_scripts: - if script != os.path.basename(script) or example_include_name(script) not in copied_scripts: - missing_scripts.append(script) - if missing_scripts: - print("ERROR: {} references missing example script(s): {}".format(fm["path"], ", ".join(missing_scripts))) - sys.exit(1) + example_includes_dir = os.path.join(includes_dir, category, example) + os.makedirs(example_includes_dir, exist_ok=True) + copy_example_scripts(example_src_dir, example_includes_dir, resolved_scripts) examplesindex.append(fm) replace_frontmatter(md_file, fm)