From 522bee4aa3a8a5084da21ca9f578efdf8934d95e Mon Sep 17 00:00:00 2001 From: bullman Date: Sun, 3 May 2026 20:25:09 -0700 Subject: [PATCH 1/5] Migrate step templates to source-first layout --- .gitignore | 5 +- gulpfile.babel.js | 243 +++++++++++-- spec/logos-validation-tests.js | 6 +- spec/step-template-validation-tests.js | 3 +- tools/migrate-source-first-reset.js | 106 ++++++ tools/migrate-source-first.js | 467 +++++++++++++++++++++++++ 6 files changed, 802 insertions(+), 28 deletions(-) create mode 100644 tools/migrate-source-first-reset.js create mode 100644 tools/migrate-source-first.js diff --git a/.gitignore b/.gitignore index 96cc31e06..0e2737b80 100644 --- a/.gitignore +++ b/.gitignore @@ -9,9 +9,8 @@ junitresults.xml scriptcs_packages/ scriptcs_packages.config -step-templates/*.ps1 -step-templates/*.sh -step-templates/*.py +/step-templates/ +/step-templates-orig/ diff-output/ /.vs !.vscode diff --git a/gulpfile.babel.js b/gulpfile.babel.js index eeec6dbcd..660782689 100644 --- a/gulpfile.babel.js +++ b/gulpfile.babel.js @@ -34,7 +34,7 @@ import http from "http"; import https from "https"; import jsonlint from "gulp-jsonlint"; import path from "path"; -import { spawn } from "child_process"; +import { execFileSync, spawn } from "child_process"; const sass = gulpSass(dartSass); const clientDir = "app"; @@ -42,6 +42,33 @@ const serverDir = "server"; const buildDir = "build"; const publishDir = "dist"; +const sourceStepTemplatesDir = "src/step-templates"; +const scriptDefinitions = [ + { + sourceBaseName: "scriptbody", + sourceExtensions: [".ps1", ".sh", ".py"], + propertyName: "Octopus.Action.Script.ScriptBody", + legacyBaseName: "ScriptBody", + }, + { + sourceBaseName: "predeploy", + sourceExtensions: [".ps1"], + propertyName: "Octopus.Action.CustomScripts.PreDeploy.ps1", + legacyBaseName: "PreDeploy", + }, + { + sourceBaseName: "deploy", + sourceExtensions: [".ps1"], + propertyName: "Octopus.Action.CustomScripts.Deploy.ps1", + legacyBaseName: "Deploy", + }, + { + sourceBaseName: "postdeploy", + sourceExtensions: [".ps1"], + propertyName: "Octopus.Action.CustomScripts.PostDeploy.ps1", + legacyBaseName: "PostDeploy", + }, +]; const $ = gulpLoadPlugins({ rename: { @@ -52,6 +79,156 @@ const $ = gulpLoadPlugins({ const reload = browserSync.reload; const argv = yargs.argv; +function isDirectory(targetPath) { + return fs.existsSync(targetPath) && fs.statSync(targetPath).isDirectory(); +} + +function ensureDirectory(dirPath) { + fs.mkdirSync(dirPath, { recursive: true }); +} + +function listMigratedTemplates() { + if (!isDirectory(sourceStepTemplatesDir)) { + return []; + } + + return fs + .readdirSync(sourceStepTemplatesDir) + .filter((entry) => !["logos", "tests"].includes(entry)) + .filter((entry) => isDirectory(path.join(sourceStepTemplatesDir, entry))) + .filter((entry) => fs.existsSync(path.join(sourceStepTemplatesDir, entry, "metadata.json"))) + .sort(); +} + +function getLegacyJsonPath(templateName) { + return path.join("step-templates", `${templateName}.json`); +} + +function getSourceTemplateDirectory(templateName) { + return path.join(sourceStepTemplatesDir, templateName); +} + +function getLegacySidecarFileName(templateName, sourceFileName, definition) { + const extension = path.extname(sourceFileName); + + return `${templateName}.${definition.legacyBaseName}${extension}`; +} + +function runPack(templateName) { + execFileSync(process.env.PWSH_PATH || "pwsh", ["-NoProfile", "-File", path.join("tools", "_pack.ps1"), "-SearchPattern", templateName], { + cwd: process.cwd(), + stdio: "inherit", + }); +} + +function runPackAll() { + execFileSync(process.env.PWSH_PATH || "pwsh", ["-NoProfile", "-File", path.join("tools", "_pack.ps1")], { + cwd: process.cwd(), + stdio: "inherit", + }); +} + +function cleanGeneratedSidecars(templateName) { + for (const definition of scriptDefinitions) { + for (const extension of definition.sourceExtensions) { + const sidecarPath = path.join("step-templates", getLegacySidecarFileName(templateName, `${definition.sourceBaseName}${extension}`, definition)); + if (fs.existsSync(sidecarPath)) { + fs.rmSync(sidecarPath, { force: true }); + } + } + } +} + +function cleanGeneratedSidecarsForTemplates(templateNames) { + for (const templateName of templateNames) { + cleanGeneratedSidecars(templateName); + } +} + +function materializeLegacyTemplate(templateName) { + const sourceDirectory = getSourceTemplateDirectory(templateName); + const metadataPath = path.join(sourceDirectory, "metadata.json"); + + if (!fs.existsSync(metadataPath)) { + return false; + } + + ensureDirectory("step-templates"); + fs.copyFileSync(metadataPath, getLegacyJsonPath(templateName)); + + for (const definition of scriptDefinitions) { + for (const extension of definition.sourceExtensions) { + const sourceFileName = `${definition.sourceBaseName}${extension}`; + const sourceFilePath = path.join(sourceDirectory, sourceFileName); + + if (!fs.existsSync(sourceFilePath)) { + continue; + } + + const legacySidecarPath = path.join("step-templates", getLegacySidecarFileName(templateName, sourceFileName, definition)); + fs.copyFileSync(sourceFilePath, legacySidecarPath); + break; + } + } + + return true; +} + +function generateMigratedTemplate(templateName) { + if (!materializeLegacyTemplate(templateName)) { + return false; + } + + try { + runPack(templateName); + } finally { + cleanGeneratedSidecars(templateName); + } + + return true; +} + +function generateAllMigratedTemplates() { + const templateNames = listMigratedTemplates(); + const materializedTemplateNames = templateNames.filter((templateName) => materializeLegacyTemplate(templateName)); + + if (materializedTemplateNames.length === 0) { + return false; + } + + try { + runPackAll(); + } finally { + cleanGeneratedSidecarsForTemplates(materializedTemplateNames); + } + + return true; +} + +function getChangedSourcePathType(changedPath) { + const absolutePath = path.resolve(changedPath); + const relativePath = path.relative(path.resolve(sourceStepTemplatesDir), absolutePath); + + if (relativePath.startsWith("..")) { + return { type: "outside" }; + } + + const [firstSegment] = relativePath.split(path.sep).filter(Boolean); + if (!firstSegment) { + return { type: "all" }; + } + + if (firstSegment === "logos") { + return { type: "logos" }; + } + + if (firstSegment === "tests") { + return { type: "tests" }; + } + + return { type: "template", templateName: firstSegment }; +} + function openBrowser(url) { if (process.env.CI) { return; @@ -123,6 +300,11 @@ function lint(files, options = {}) { gulp.task("lint:client", lint(`${clientDir}/**/*.jsx`)); gulp.task("lint:server", lint(`./${serverDir}/server.js`)); +gulp.task("prepare:step-templates", (done) => { + generateAllMigratedTemplates(); + done(); +}); + gulp.task("lint:step-templates", () => { return gulp .src("./step-templates/*.json") @@ -132,25 +314,24 @@ gulp.task("lint:step-templates", () => { .pipe(jsonlint.reporter()); }); -gulp.task( - "tests", - gulp.series("lint:step-templates", () => { - return ( - gulp - .src("./spec/*-tests.js") - // gulp-jasmine works on filepaths so you can't have any plugins before it - .pipe( - jasmine({ - includeStackTrace: false, - reporter: [new jasmineReporters.JUnitXmlReporter(), process.env.TEAMCITY_VERSION ? new jasmineReporters.TeamCityReporter() : new jasmineTerminalReporter()], - }) - ) - .on("error", function () { - process.exit(1); +gulp.task("test:step-templates", () => { + return ( + gulp + .src("./spec/*-tests.js") + // gulp-jasmine works on filepaths so you can't have any plugins before it + .pipe( + jasmine({ + includeStackTrace: false, + reporter: [new jasmineReporters.JUnitXmlReporter(), process.env.TEAMCITY_VERSION ? new jasmineReporters.TeamCityReporter() : new jasmineTerminalReporter()], }) - ); - }) -); + ) + .on("error", function () { + process.exit(1); + }) + ); +}); + +gulp.task("tests", gulp.series("prepare:step-templates", "lint:step-templates", "test:step-templates")); function humanize(categoryId) { switch (categoryId) { @@ -363,7 +544,9 @@ function provideMissingData() { template.Category = humanize(categoryId); if (!template.Logo) { - var logo = fs.readFileSync("./step-templates/logos/" + categoryId + ".png"); + var sourceLogoPath = "./src/step-templates/logos/" + categoryId + ".png"; + var legacyLogoPath = "./step-templates/logos/" + categoryId + ".png"; + var logo = fs.readFileSync(fs.existsSync(sourceLogoPath) ? sourceLogoPath : legacyLogoPath); template.Logo = Buffer.from(logo).toString("base64"); } @@ -504,6 +687,26 @@ gulp.task( gulp.watch(`${clientDir}/**/*.jsx`, gulp.series("scripts", "copy:app", reloadServer)); gulp.watch(`${clientDir}/content/styles/**/*.scss`, gulp.series("styles:client")); gulp.watch("step-templates/*.json", gulp.series("step-templates:data")); + gulp.watch(`${sourceStepTemplatesDir}/**/*`).on("all", (eventName, changedPath) => { + const change = changedPath ? getChangedSourcePathType(changedPath) : { type: "all" }; + + if (change.type === "template") { + generateMigratedTemplate(change.templateName); + } else if (change.type === "logos" || change.type === "all") { + generateAllMigratedTemplates(); + } else if (change.type === "outside" || change.type === "tests") { + return; + } + + gulp.series("step-templates:data")((error) => { + if (error) { + log.error(error); + return; + } + + reload(); + }); + }); gulp.watch(`${buildDir}/**/*.*`).on("change", reload); }) diff --git a/spec/logos-validation-tests.js b/spec/logos-validation-tests.js index ad0ad834a..38e28e28b 100644 --- a/spec/logos-validation-tests.js +++ b/spec/logos-validation-tests.js @@ -2,9 +2,9 @@ var fs = require("fs"); describe("logos", function () { it("logos have valid details", function (done) { - var filenameCounter = 0; - var stepTemplateCount = 0; - var dirname = "./step-templates/logos"; + var sourceDirname = "./src/step-templates/logos"; + var legacyDirname = "./step-templates/logos"; + var dirname = fs.existsSync(sourceDirname) ? sourceDirname : legacyDirname; fs.readdir(dirname, function (err, filenames) { if (err) { diff --git a/spec/step-template-validation-tests.js b/spec/step-template-validation-tests.js index 394a8d76a..182db1694 100644 --- a/spec/step-template-validation-tests.js +++ b/spec/step-template-validation-tests.js @@ -109,9 +109,8 @@ describe("step-templates", function () { } var otherThings = results.filter(function (file) { - var pesterFile = file.endsWith(".ScriptBody.ps1"); var jsonFile = file.endsWith(".json"); - return !pesterFile && !jsonFile && file !== "logos" && file !== "tests"; + return !jsonFile && file !== "logos" && file !== "tests"; }); expect(otherThings).toEqual([]); done(); diff --git a/tools/migrate-source-first-reset.js b/tools/migrate-source-first-reset.js new file mode 100644 index 000000000..bdaeef5c3 --- /dev/null +++ b/tools/migrate-source-first-reset.js @@ -0,0 +1,106 @@ +"use strict"; + +const fs = require("fs"); +const path = require("path"); +const readline = require("readline/promises"); +const { stdin, stdout } = require("process"); +const { execFileSync } = require("child_process"); + +const repoRoot = path.resolve(__dirname, ".."); +const legacyRoot = path.join(repoRoot, "step-templates"); +const sourceRoot = path.join(repoRoot, "src", "step-templates"); +const backupRoot = path.join(repoRoot, "step-templates-orig"); + +function pathExists(targetPath) { + return fs.existsSync(targetPath); +} + +function removePath(targetPath) { + if (pathExists(targetPath)) { + fs.rmSync(targetPath, { recursive: true, force: true }); + } +} + +function runGit(args) { + execFileSync("git", args, { + cwd: repoRoot, + stdio: "inherit", + }); +} + +function captureGit(args) { + return execFileSync("git", args, { + cwd: repoRoot, + encoding: "utf8", + stdio: ["ignore", "pipe", "inherit"], + }).trim(); +} + +function isTrackedAtHead(targetPath) { + return captureGit(["ls-tree", "--name-only", "HEAD", targetPath]).length > 0; +} + +async function confirmReset(rl) { + const answer = (await rl.question("Reset this migration run back to branch HEAD? [y/N] ")).trim().toLowerCase(); + return answer === "y" || answer === "yes"; +} + +async function main() { + const rl = readline.createInterface({ input: stdin, output: stdout }); + + try { + console.log("Source-first migration reset"); + console.log(`Repo root: ${repoRoot}`); + console.log("This will:"); + console.log("- restore tracked step-templates and src/step-templates to current branch HEAD"); + console.log("- remove step-templates-orig/"); + console.log("- remove any untracked src/step-templates/ leftovers"); + + const confirmed = await confirmReset(rl); + if (!confirmed) { + console.log("Aborted."); + return; + } + + const restorePaths = ["step-templates"]; + const sourceTrackedAtHead = isTrackedAtHead("src/step-templates"); + if (sourceTrackedAtHead) { + restorePaths.push("src/step-templates"); + } + + runGit(["restore", "--source=HEAD", "--staged", "--worktree", ...restorePaths]); + + if (!sourceTrackedAtHead) { + runGit(["rm", "-r", "-f", "--cached", "--ignore-unmatch", "src/step-templates"]); + } + + removePath(backupRoot); + + if (pathExists(sourceRoot)) { + const entries = fs.readdirSync(sourceRoot); + if (entries.length === 0) { + removePath(sourceRoot); + } + } + + if (pathExists(sourceRoot)) { + removePath(sourceRoot); + } + + if (!pathExists(legacyRoot)) { + throw new Error("step-templates/ is missing after reset."); + } + + console.log("Reset complete."); + console.log("- restored tracked step-templates and src/step-templates to branch HEAD"); + console.log("- removed step-templates-orig/"); + console.log("- removed src/step-templates/ leftovers"); + } finally { + rl.close(); + } +} + +main().catch((error) => { + console.error(error.message || error); + process.exit(1); +}); diff --git a/tools/migrate-source-first.js b/tools/migrate-source-first.js new file mode 100644 index 000000000..6ef4bca23 --- /dev/null +++ b/tools/migrate-source-first.js @@ -0,0 +1,467 @@ +"use strict"; + +const fs = require("fs"); +const path = require("path"); +const readline = require("readline/promises"); +const { stdin, stdout } = require("process"); +const { execFileSync } = require("child_process"); + +const repoRoot = path.resolve(__dirname, ".."); +const legacyRoot = path.join(repoRoot, "step-templates"); +const backupRoot = path.join(repoRoot, "step-templates-orig"); +const sourceRoot = path.join(repoRoot, "src", "step-templates"); +const placeholderPrefix = "__SOURCE_FILE__:"; +const ansiReset = "\u001b[0m"; +const ansiBold = "\u001b[1m"; +const ansiDim = "\u001b[2m"; +const ansiGreen = "\u001b[32m"; +const ansiRed = "\u001b[31m"; +const ansiYellow = "\u001b[33m"; +const ansiBlue = "\u001b[34m"; +const ansiCyan = "\u001b[36m"; + +const scriptDefinitions = [ + { + sourceBaseName: "scriptbody", + legacyBaseName: "ScriptBody", + propertyName: "Octopus.Action.Script.ScriptBody", + getExtension(template) { + const syntax = (((template || {}).Properties || {})["Octopus.Action.Script.Syntax"] || "PowerShell").toLowerCase(); + if (syntax === "bash") { + return ".sh"; + } + + if (syntax === "python") { + return ".py"; + } + + return ".ps1"; + }, + shouldExtract(template) { + const properties = (template || {}).Properties || {}; + const scriptSource = properties["Octopus.Action.Script.ScriptSource"] || "Inline"; + return scriptSource === "Inline" && typeof properties[this.propertyName] === "string" && properties[this.propertyName].length > 0; + }, + }, + { + sourceBaseName: "predeploy", + legacyBaseName: "PreDeploy", + propertyName: "Octopus.Action.CustomScripts.PreDeploy.ps1", + getExtension() { + return ".ps1"; + }, + shouldExtract(template) { + const value = (((template || {}).Properties || {})[this.propertyName] || ""); + return typeof value === "string" && value.length > 0; + }, + }, + { + sourceBaseName: "deploy", + legacyBaseName: "Deploy", + propertyName: "Octopus.Action.CustomScripts.Deploy.ps1", + getExtension() { + return ".ps1"; + }, + shouldExtract(template) { + const value = (((template || {}).Properties || {})[this.propertyName] || ""); + return typeof value === "string" && value.length > 0; + }, + }, + { + sourceBaseName: "postdeploy", + legacyBaseName: "PostDeploy", + propertyName: "Octopus.Action.CustomScripts.PostDeploy.ps1", + getExtension() { + return ".ps1"; + }, + shouldExtract(template) { + const value = (((template || {}).Properties || {})[this.propertyName] || ""); + return typeof value === "string" && value.length > 0; + }, + }, +]; + +const excludedValidationFields = new Set(["ExportedAt", "LastModifiedOn", "LastModifiedAt", "LastModfiedAt"]); + +function listTemplateJsonFiles(rootPath) { + return fs + .readdirSync(rootPath) + .filter((entry) => entry.endsWith(".json")) + .sort(); +} + +function runGit(args) { + execFileSync("git", args, { + cwd: repoRoot, + stdio: "inherit", + }); +} + +function colorize(color, text) { + return `${color}${text}${ansiReset}`; +} + +function log(message, type = "info") { + if (type === "blank") { + console.log(""); + return; + } + + if (type === "banner") { + console.log(colorize(ansiBold + ansiBlue, message)); + return; + } + + if (type === "error") { + console.log(colorize(ansiBold + ansiRed, message)); + return; + } + + if (type === "step") { + console.log(""); + console.log(colorize(ansiBold + ansiBlue, message)); + return; + } + + if (type === "note") { + console.log(colorize(ansiYellow, `NOTE ${message}`)); + return; + } + + if (type === "success") { + console.log(`${colorize(ansiGreen, "DONE")} ${message}`); + return; + } + + if (type === "action") { + console.log(`${colorize(ansiCyan, "ACTION")} ${message}`); + return; + } + + if (type === "pass") { + console.log(`${ansiGreen}PASS${ansiReset} ${message}`); + return; + } + + if (type === "fail") { + console.log(`${ansiRed}FAIL${ansiReset} ${message}`); + return; + } + + console.log(message); +} + +function normalizeForComparison(value) { + if (Array.isArray(value)) { + return value.map(normalizeForComparison); + } + + if (value && typeof value === "object") { + const result = {}; + for (const [key, childValue] of Object.entries(value)) { + if (excludedValidationFields.has(key)) { + continue; + } + + result[key] = normalizeForComparison(childValue); + } + + return result; + } + + return value; +} + +function diffObjects(left, right, currentPath = "$") { + if (typeof left !== typeof right) { + return [`${currentPath}: type mismatch (${typeof left} !== ${typeof right})`]; + } + + if (Array.isArray(left) && Array.isArray(right)) { + if (left.length !== right.length) { + return [`${currentPath}: array length mismatch (${left.length} !== ${right.length})`]; + } + + for (let index = 0; index < left.length; index += 1) { + const nested = diffObjects(left[index], right[index], `${currentPath}[${index}]`); + if (nested.length > 0) { + return nested; + } + } + + return []; + } + + if (left && typeof left === "object" && right && typeof right === "object") { + const leftKeys = Object.keys(left).sort(); + const rightKeys = Object.keys(right).sort(); + + if (leftKeys.join("|") !== rightKeys.join("|")) { + return [`${currentPath}: key mismatch (${leftKeys.join(", ")} !== ${rightKeys.join(", ")})`]; + } + + for (const key of leftKeys) { + const nested = diffObjects(left[key], right[key], `${currentPath}.${key}`); + if (nested.length > 0) { + return nested; + } + } + + return []; + } + + if (left !== right) { + return [`${currentPath}: value mismatch (${JSON.stringify(left)} !== ${JSON.stringify(right)})`]; + } + + return []; +} + +async function confirmStep(rl, prompt) { + const answer = (await rl.question(`${prompt} [y/N] `)).trim().toLowerCase(); + if (answer !== "y" && answer !== "yes") { + log("Aborted."); + return false; + } + + return true; +} + +async function step1_PrepareValidationBaseline(rl) { + log("Step 1: Prepare validation baseline", "step"); + log("This creates a frozen pre-migration copy of the current step-template JSON files."); + log("The copied files are written to step-templates-orig/ at the repo root."); + log("The copy is unpacked so we can later compare the rebuilt source-first output to the original repo state."); + log("step-templates-orig/ is git-ignored so these temporary baseline files cannot be accidentally committed."); + + if (fs.existsSync(backupRoot)) { + log("step-templates-orig/ already exists and will be replaced.", "note"); + } + + if (!(await confirmStep(rl, "Prepare the validation baseline?"))) { + return false; + } + + if (fs.existsSync(backupRoot)) { + fs.rmSync(backupRoot, { recursive: true, force: true }); + } + + fs.cpSync(legacyRoot, backupRoot, { recursive: true }); + + for (const fileName of listTemplateJsonFiles(backupRoot)) { + const templateName = fileName.replace(/\.json$/i, ""); + const template = JSON.parse(fs.readFileSync(path.join(backupRoot, fileName), "utf8")); + const unpackedSidecars = []; + + for (const definition of scriptDefinitions) { + if (!definition.shouldExtract(template)) { + continue; + } + + const extension = definition.getExtension(template); + const sidecarPath = path.join(backupRoot, `${templateName}.${definition.legacyBaseName}${extension}`); + const value = template.Properties[definition.propertyName]; + fs.writeFileSync(sidecarPath, value, "utf8"); + unpackedSidecars.push(path.basename(sidecarPath)); + } + + if (unpackedSidecars.length > 0) { + log(`UNPACK ${fileName} -> ${unpackedSidecars.join(", ")}`, "action"); + continue; + } + + log(`UNPACK ${fileName} -> ${colorize(ansiDim, "no sidecars")}`, "action"); + } + + log("Created step-templates-orig/", "success"); + log("Unpacked reference sidecars in step-templates-orig/", "success"); + return true; +} + +async function step2_MoveJsonTemplatesIntoSourceTree(rl) { + log("Step 2: Move JSON templates into the source tree with history", "step"); + log("Each step-template JSON file is moved with git history into src/step-templates/