Add source-first migration tooling for step templates

.gitattributes

@@ -1,5 +1,11 @@
-# Set the default behavior, in case people don't have core.autocrlf set.
 * text=auto
 
-# Denote all files that are truly binary and should not be modified.
+*.json text eol=lf
 *.png binary
+
+src/step-templates/*/scriptbody.ps1 -text
+src/step-templates/*/predeploy.ps1 -text
+src/step-templates/*/deploy.ps1 -text
+src/step-templates/*/postdeploy.ps1 -text
+src/step-templates/*/scriptbody.sh -text
+src/step-templates/*/scriptbody.py -text

.gitignore

@@ -9,9 +9,8 @@ junitresults.xml
 scriptcs_packages/
 scriptcs_packages.config
 
-step-templates/*.ps1
-step-templates/*.sh
-step-templates/*.py
+/step-templates/
+/step-templates-orig/
 diff-output/
 /.vs
 !.vscode

gulpfile.babel.js

@@ -34,14 +34,41 @@ import http from "http";
 import https from "https";
 import jsonlint from "gulp-jsonlint";
 import path from "path";
-import { spawn } from "child_process";
+import { execFileSync, spawn } from "child_process";
 
 const sass = gulpSass(dartSass);
 const clientDir = "app";
 const serverDir = "server";
 
 const buildDir = "build";
 const publishDir = "dist";
+const sourceStepTemplatesDir = "src/step-templates";
+const scriptDefinitions = [
+  {
+    sourceBaseName: "scriptbody",
+    sourceExtensions: [".ps1", ".sh", ".py"],
+    propertyName: "Octopus.Action.Script.ScriptBody",
+    legacyBaseName: "ScriptBody",
+  },
+  {
+    sourceBaseName: "predeploy",
+    sourceExtensions: [".ps1"],
+    propertyName: "Octopus.Action.CustomScripts.PreDeploy.ps1",
+    legacyBaseName: "PreDeploy",
+  },
+  {
+    sourceBaseName: "deploy",
+    sourceExtensions: [".ps1"],
+    propertyName: "Octopus.Action.CustomScripts.Deploy.ps1",
+    legacyBaseName: "Deploy",
+  },
+  {
+    sourceBaseName: "postdeploy",
+    sourceExtensions: [".ps1"],
+    propertyName: "Octopus.Action.CustomScripts.PostDeploy.ps1",
+    legacyBaseName: "PostDeploy",
+  },
+];
 
 const $ = gulpLoadPlugins({
   rename: {
@@ -52,6 +79,156 @@ const $ = gulpLoadPlugins({
 const reload = browserSync.reload;
 const argv = yargs.argv;
 
+function isDirectory(targetPath) {
+  return fs.existsSync(targetPath) && fs.statSync(targetPath).isDirectory();
+}
+
+function ensureDirectory(dirPath) {
+  fs.mkdirSync(dirPath, { recursive: true });
+}
+
+function listMigratedTemplates() {
+  if (!isDirectory(sourceStepTemplatesDir)) {
+    return [];
+  }
+
+  return fs
+    .readdirSync(sourceStepTemplatesDir)
+    .filter((entry) => !["logos", "tests"].includes(entry))
+    .filter((entry) => isDirectory(path.join(sourceStepTemplatesDir, entry)))
+    .filter((entry) => fs.existsSync(path.join(sourceStepTemplatesDir, entry, "metadata.json")))
+    .sort();
+}
+
+function getLegacyJsonPath(templateName) {
+  return path.join("step-templates", `${templateName}.json`);
+}
+
+function getSourceTemplateDirectory(templateName) {
+  return path.join(sourceStepTemplatesDir, templateName);
+}
+
+function getLegacySidecarFileName(templateName, sourceFileName, definition) {
+  const extension = path.extname(sourceFileName);
+
+  return `${templateName}.${definition.legacyBaseName}${extension}`;
+}
+
+function runPack(templateName) {
+  execFileSync(process.env.PWSH_PATH || "pwsh", ["-NoProfile", "-File", path.join("tools", "_pack.ps1"), "-SearchPattern", templateName], {
+    cwd: process.cwd(),
+    stdio: "inherit",
+  });
+}
+
+function runPackAll() {
+  execFileSync(process.env.PWSH_PATH || "pwsh", ["-NoProfile", "-File", path.join("tools", "_pack.ps1")], {
+    cwd: process.cwd(),
+    stdio: "inherit",
+  });
+}
+
+function cleanGeneratedSidecars(templateName) {
+  for (const definition of scriptDefinitions) {
+    for (const extension of definition.sourceExtensions) {
+      const sidecarPath = path.join("step-templates", getLegacySidecarFileName(templateName, `${definition.sourceBaseName}${extension}`, definition));
+      if (fs.existsSync(sidecarPath)) {
+        fs.rmSync(sidecarPath, { force: true });
+      }
+    }
+  }
+}
+
+function cleanGeneratedSidecarsForTemplates(templateNames) {
+  for (const templateName of templateNames) {
+    cleanGeneratedSidecars(templateName);
+  }
+}
+
+function materializeLegacyTemplate(templateName) {
+  const sourceDirectory = getSourceTemplateDirectory(templateName);
+  const metadataPath = path.join(sourceDirectory, "metadata.json");
+
+  if (!fs.existsSync(metadataPath)) {
+    return false;
+  }
+
+  ensureDirectory("step-templates");
+  fs.copyFileSync(metadataPath, getLegacyJsonPath(templateName));
+
+  for (const definition of scriptDefinitions) {
+    for (const extension of definition.sourceExtensions) {
+      const sourceFileName = `${definition.sourceBaseName}${extension}`;
+      const sourceFilePath = path.join(sourceDirectory, sourceFileName);
+
+      if (!fs.existsSync(sourceFilePath)) {
+        continue;
+      }
+
+      const legacySidecarPath = path.join("step-templates", getLegacySidecarFileName(templateName, sourceFileName, definition));
+      fs.copyFileSync(sourceFilePath, legacySidecarPath);
+      break;
+    }
+  }
+
+  return true;
+}
+
+function generateMigratedTemplate(templateName) {
+  if (!materializeLegacyTemplate(templateName)) {
+    return false;
+  }
+
+  try {
+    runPack(templateName);
+  } finally {
+    cleanGeneratedSidecars(templateName);
+  }
+
+  return true;
+}
+
+function generateAllMigratedTemplates() {
+  const templateNames = listMigratedTemplates();
+  const materializedTemplateNames = templateNames.filter((templateName) => materializeLegacyTemplate(templateName));
+
+  if (materializedTemplateNames.length === 0) {
+    return false;
+  }
+
+  try {
+    runPackAll();
+  } finally {
+    cleanGeneratedSidecarsForTemplates(materializedTemplateNames);
+  }
+
+  return true;
+}
+
+function getChangedSourcePathType(changedPath) {
+  const absolutePath = path.resolve(changedPath);
+  const relativePath = path.relative(path.resolve(sourceStepTemplatesDir), absolutePath);
+
+  if (relativePath.startsWith("..")) {
+    return { type: "outside" };
+  }
+
+  const [firstSegment] = relativePath.split(path.sep).filter(Boolean);
+  if (!firstSegment) {
+    return { type: "all" };
+  }
+
+  if (firstSegment === "logos") {
+    return { type: "logos" };
+  }
+
+  if (firstSegment === "tests") {
+    return { type: "tests" };
+  }
+
+  return { type: "template", templateName: firstSegment };
+}
+
 function openBrowser(url) {
   if (process.env.CI) {
     return;
@@ -123,6 +300,11 @@ function lint(files, options = {}) {
 
 gulp.task("lint:client", lint(`${clientDir}/**/*.jsx`));
 gulp.task("lint:server", lint(`./${serverDir}/server.js`));
+gulp.task("prepare:step-templates", (done) => {
+  generateAllMigratedTemplates();
+  done();
+});
+
 gulp.task("lint:step-templates", () => {
   return gulp
     .src("./step-templates/*.json")
@@ -132,25 +314,24 @@ gulp.task("lint:step-templates", () => {
     .pipe(jsonlint.reporter());
 });
 
-gulp.task(
-  "tests",
-  gulp.series("lint:step-templates", () => {
-    return (
-      gulp
-        .src("./spec/*-tests.js")
-        // gulp-jasmine works on filepaths so you can't have any plugins before it
-        .pipe(
-          jasmine({
-            includeStackTrace: false,
-            reporter: [new jasmineReporters.JUnitXmlReporter(), process.env.TEAMCITY_VERSION ? new jasmineReporters.TeamCityReporter() : new jasmineTerminalReporter()],
-          })
-        )
-        .on("error", function () {
-          process.exit(1);
+gulp.task("test:step-templates", () => {
+  return (
+    gulp
+      .src("./spec/*-tests.js")
+      // gulp-jasmine works on filepaths so you can't have any plugins before it
+      .pipe(
+        jasmine({
+          includeStackTrace: false,
+          reporter: [new jasmineReporters.JUnitXmlReporter(), process.env.TEAMCITY_VERSION ? new jasmineReporters.TeamCityReporter() : new jasmineTerminalReporter()],
         })
-    );
-  })
-);
+      )
+      .on("error", function () {
+        process.exit(1);
+      })
+  );
+});
+
+gulp.task("tests", gulp.series("prepare:step-templates", "lint:step-templates", "test:step-templates"));
 
 function humanize(categoryId) {
   switch (categoryId) {
@@ -363,7 +544,9 @@ function provideMissingData() {
     template.Category = humanize(categoryId);
 
     if (!template.Logo) {
-      var logo = fs.readFileSync("./step-templates/logos/" + categoryId + ".png");
+      var sourceLogoPath = "./src/step-templates/logos/" + categoryId + ".png";
+      var legacyLogoPath = "./step-templates/logos/" + categoryId + ".png";
+      var logo = fs.readFileSync(fs.existsSync(sourceLogoPath) ? sourceLogoPath : legacyLogoPath);
       template.Logo = Buffer.from(logo).toString("base64");
     }
 
@@ -504,6 +687,26 @@ gulp.task(
     gulp.watch(`${clientDir}/**/*.jsx`, gulp.series("scripts", "copy:app", reloadServer));
     gulp.watch(`${clientDir}/content/styles/**/*.scss`, gulp.series("styles:client"));
     gulp.watch("step-templates/*.json", gulp.series("step-templates:data"));
+    gulp.watch(`${sourceStepTemplatesDir}/**/*`).on("all", (eventName, changedPath) => {
+      const change = changedPath ? getChangedSourcePathType(changedPath) : { type: "all" };
+
+      if (change.type === "template") {
+        generateMigratedTemplate(change.templateName);
+      } else if (change.type === "logos" || change.type === "all") {
+        generateAllMigratedTemplates();
+      } else if (change.type === "outside" || change.type === "tests") {
+        return;
+      }
+
+      gulp.series("step-templates:data")((error) => {
+        if (error) {
+          log.error(error);
+          return;
+        }
+
+        reload();
+      });
+    });
 
     gulp.watch(`${buildDir}/**/*.*`).on("change", reload);
   })

spec/logos-validation-tests.js

@@ -2,9 +2,9 @@ var fs = require("fs");
 
 describe("logos", function () {
   it("logos have valid details", function (done) {
-    var filenameCounter = 0;
-    var stepTemplateCount = 0;
-    var dirname = "./step-templates/logos";
+    var sourceDirname = "./src/step-templates/logos";
+    var legacyDirname = "./step-templates/logos";
+    var dirname = fs.existsSync(sourceDirname) ? sourceDirname : legacyDirname;
 
     fs.readdir(dirname, function (err, filenames) {
       if (err) {

spec/step-template-validation-tests.js

@@ -109,9 +109,8 @@ describe("step-templates", function () {
       }
 
       var otherThings = results.filter(function (file) {
-        var pesterFile = file.endsWith(".ScriptBody.ps1");
         var jsonFile = file.endsWith(".json");
-        return !pesterFile && !jsonFile && file !== "logos" && file !== "tests";
+        return !jsonFile && file !== "logos" && file !== "tests";
       });
       expect(otherThings).toEqual([]);
       done();