[CODE-119] Complete refactor of parser + build improvements + validate improvements + error improvements (#32)

* Refactored a bunch of code related to the parser WIP

* Refactored entire parser flow

* Added source map cache and btree parsing for yaml source maps

* Refactored yaml btree into a separate class

* Added source map integration into the code

* Fixed compile bugs and added descriptive comments on how the yaml b-tree works

* Improved styling for code snippet, misc clean-up and added code snippet to type not found error

* Improved error messages

* Added support for plugin validation error messages with source maps and code snippets. Moved ajv error message to be a separate function so that it can be re-used.

* Fixed codify.json file

* Fixed the debugger finally!! Small tweaks and improvements

* Removed tsc from start:dev to make development faster. Added additional folders to eslintignore

* Fixed bug with re-create. Fixed yaml parser bug.

Author: kevinwang5658

.eslintignore

@@ -1 +1,3 @@
 /dist
+/node_modules
+/tmp

bin/dev.js

@@ -1,4 +1,4 @@
-#!/usr/bin/env -S node --loader ts-node/esm --no-warnings=ExperimentalWarning
+#!/usr/bin/env tsx --no-warnings=ExperimentalWarning
 
 import { execute } from '@oclif/core'
 

package.json

@@ -11,13 +11,16 @@
     "ajv": "^8.12.0",
     "ajv-formats": "^3.0.1",
     "chalk": "^5.3.0",
-    "codify-schemas": "1.0.39",
+    "codify-schemas": "1.0.42",
     "debug": "^4.3.4",
     "ink": "^4.4.1",
     "parse-json": "^8.1.0",
     "react": "^18.3.1",
     "semver": "^7.5.4",
-    "supports-color": "^9.4.0"
+    "supports-color": "^9.4.0",
+    "json-source-map": "^0.6.1",
+    "js-yaml": "^4.1.0",
+    "js-yaml-source-map": "^0.2.2"
   },
   "description": "Codify is a set up as code tool for developers",
   "devDependencies": {
@@ -32,6 +35,8 @@
     "@types/node": "^18",
     "@types/react": "^18.3.1",
     "@types/semver": "^7.5.4",
+    "@types/js-yaml": "^4.0.9",
+    "@types/strip-ansi": "^5.2.1",
     "chai": "^4",
     "chai-as-promised": "^7.1.1",
     "eslint": "^8.51.0",
@@ -42,6 +47,7 @@
     "mock-fs": "^5.2.0",
     "oclif": "^4.5.7",
     "shx": "^0.3.3",
+    "strip-ansi": "^7.1.0",
     "tsx": "^4.7.3",
     "typescript": "^5",
     "vitest": "^1.4.0"
@@ -85,14 +91,14 @@
   "repository": "kevinwang5658/codify",
   "scripts": {
     "build": "shx rm -rf dist && tsc -b",
-    "lint": "eslint . --ext .ts",
+    "lint": "tsc && eslint . --ext .ts",
     "postpack": "shx rm -f oclif.manifest.json",
     "pkg": "oclif pack macos -r .",
     "posttest": "npm run lint",
     "prepack": "npm run build && oclif manifest && oclif readme",
     "test": "mocha --forbid-only \"test/**/*.test.ts\"",
     "version": "oclif readme && git add README.md",
-    "start:dev": "tsc && node ./bin/run.js",
+    "start:dev": "./bin/dev.js",
     "start:vm": "npm run build && npm run pack:macos && npm run start:vm"
   },
   "version": "0.0.0",

src/commands/apply/index.ts

@@ -1,10 +1,9 @@
 import { Args, Flags } from '@oclif/core'
-import { ResourceOperation } from 'codify-schemas';
 import * as path from 'node:path';
 
+import { BaseCommand } from '../../common/base-command.js';
 import { ApplyOrchestrator } from '../../orchestrators/apply.js';
 import { PlanOrchestrator } from '../../orchestrators/plan.js';
-import { BaseCommand } from '../../common/base-command.js';
 
 export default class Apply extends BaseCommand {
   static args = {

src/common/base-command.ts

@@ -4,8 +4,9 @@ import chalk from 'chalk';
 import { SudoRequestData } from 'codify-schemas';
 import createDebug from 'debug';
 
-import { ctx, Event } from '../events/context.js';
+import { Event, ctx } from '../events/context.js';
 import { Reporter, ReporterFactory, ReporterType } from '../ui/reporters/reporter.js';
+import { prettyPrintError } from './errors.js';
 
 export abstract class BaseCommand extends Command {
 
@@ -53,7 +54,7 @@ export abstract class BaseCommand extends Command {
   }
 
   protected async catch(err: Error): Promise<void> {
-    console.log(chalk.red(err.message));
+    prettyPrintError(err);
     process.exit(1);
   }
 

src/common/errors.test.ts

@@ -0,0 +1,76 @@
+import { describe, expect, it } from 'vitest';
+import { ajv } from '../utils/ajv';
+import { AjvValidationError } from './errors';
+import { ResourceSchema } from 'codify-schemas';
+import { SourceMapCache } from '../parser/source-maps';
+import { JsonParser } from '../parser/json/json-parser';
+import { FileType } from '../parser/entities';
+import stripAnsi from 'strip-ansi';
+
+describe('AjvValidationError tests', () => {
+  it('Can properly format a AJV error message without source maps', () => {
+    const validator = ajv.compile(ResourceSchema);
+    const content = {
+      // missing type
+      "name": "something",
+      "dependsOn": "supposed to be an array"
+    };
+
+    const isValid = validator(content)
+    expect(isValid).to.be.false;
+
+    const error = new AjvValidationError(
+      'resource is not valid',
+      validator.errors
+    );
+
+    console.log(error.formattedMessage())
+  })
+
+  it('Can properly format a AJV error message with source maps', () => {
+    const contents = `
+[
+  {
+    "type": "resourceType",
+    "name": "something",
+    "dependsOn": []
+  },
+  {
+    "type": "resourceType",
+    "name": "something",
+    "dependsOn": "supposed to be an array"
+  }
+]`;
+
+    const sourceMaps = new SourceMapCache()
+    const result = new JsonParser().parse({
+      fileType: FileType.JSON,
+      filePath: '/test/path/to/test.json',
+      contents
+    }, sourceMaps);
+
+    const validator = ajv.compile(ResourceSchema);
+    const isValid = validator(result[1].contents)
+    expect(isValid).to.be.false;
+
+    const error = new AjvValidationError(
+      'resource is not valid',
+      validator.errors,
+      '/test/path/to/test.json#/1',
+      sourceMaps
+    );
+
+    console.log(error.formattedMessage())
+    expect(stripAnsi(error.formattedMessage())).to.eq(
+`Validation error: resource is not valid
+
+"/dependsOn"  must be array
+      
+   7|   {
+   8|     "type": "resourceType",
+   9|     "name": "something",
+> 10|     "dependsOn": "supposed to be an array"
+  11|   }
+  12| ]`)
+  })
+})

src/common/errors.ts

@@ -1,20 +1,117 @@
+import { ErrorObject } from 'ajv';
+import chalk from 'chalk';
+
+import { ResourceConfig } from '../entities/resource-config.js';
+import { SourceMapCache } from '../parser/source-maps.js';
+import { formatAjvErrors } from '../utils/ajv.js';
 import { RemoveErrorMethods } from './types.js';
 
-export class InternalError extends Error {
+export abstract class CodifyError extends Error {
+  abstract formattedMessage(): string
+}
+
+export class InternalError extends CodifyError {
   name = 'InternalError'
+
+  formattedMessage(): string {
+    return `Internal error: ${this.message}`;
+  }
 }
 
+export class AjvValidationError extends CodifyError {
+  validationErrors: ErrorObject[];
+  sourceMapKey?: string;
+  sourceMaps?: SourceMapCache;
+  
+  constructor(
+    message: string,
+    validationErrors: ErrorObject[],
+    sourceMapKey?: string,
+    sourceMaps?: SourceMapCache,
+  ) {
+    super(message);
+    this.validationErrors = validationErrors;
+    this.sourceMapKey = sourceMapKey;
+    this.sourceMaps = sourceMaps;
+  }
 
-export class SyntaxError extends Error {
-  name = 'ConfigFileSyntaxError'
+  formattedMessage(): string {
+    let errorMessage = `Validation error: ${this.message}.\n\n`;
+    errorMessage += formatAjvErrors(this.validationErrors, this.sourceMapKey, this.sourceMaps)
+    return errorMessage;
+  }
+}
 
-  message!: string;
-  fileName!: string;
-  lineNumber!: string;
+export type PluginValidationErrorParams = Array<{
+  customErrorMessage?: string,
+  resource: ResourceConfig,
+  schemaErrors: ErrorObject[],
+}>
 
-  constructor(props: RemoveErrorMethods<SyntaxError>) {
-    super(props.message)
-    Object.assign(this, props);
+export class PluginValidationError extends CodifyError {
+  resourceErrors: PluginValidationErrorParams
+  sourceMaps?: SourceMapCache
+
+  constructor(
+    params: PluginValidationErrorParams,
+    sourceMaps?: SourceMapCache,
+  ) {
+    super('Validation error: the following parameters are not supported.\n\n');
+    this.resourceErrors = params
+    this.sourceMaps = sourceMaps;
+  }
+
+  formattedMessage(): string {
+    let errorMessage = `${this.message}`;
+
+    for (const resourceError of this.resourceErrors) {
+      const { customErrorMessage, resource, schemaErrors } = resourceError;
+
+      errorMessage += `Resource "${resource.id}" has invalid parameters.\n`
+      errorMessage += formatAjvErrors(schemaErrors, resource.sourceMapKey, this.sourceMaps)
+
+      if (customErrorMessage) {
+        let childMessage = `${schemaErrors.length + 1}. ${customErrorMessage}\n`
+
+        if (resource.sourceMapKey && this.sourceMaps) {
+          childMessage += `${this.sourceMaps.getCodeSnippet(resource.sourceMapKey)}\n`;
+        }
+
+        errorMessage += childMessage.split(/\n/)
+          .map((l) => `  ${l}`)
+          .join('\n')
+      }
+    }
+
+    return errorMessage;
+  }
+}
+
+export class TypeNotFoundError extends CodifyError {
+  invalidConfigs: ResourceConfig[];
+  sourceMaps?: SourceMapCache;
+
+  constructor(invalidConfigs: ResourceConfig[], sourceMaps?: SourceMapCache) {
+    super('Validation error: invalid type found. Resource type was not found in any plugins.')
+
+    this.invalidConfigs = invalidConfigs;
+    this.sourceMaps = sourceMaps;
+  }
+
+  formattedMessage(): string {
+    let errorMessage = `${this.message}\n\n`
+
+    for (const invalidConfig of this.invalidConfigs) {
+      if (!invalidConfig.sourceMapKey || !this.sourceMaps) {
+        errorMessage += `type ${invalidConfig.type} is not valid.`
+        continue;
+      }
+
+      const codeSnippet = this.sourceMaps?.getCodeSnippet(SourceMapCache.combineKeys(invalidConfig.sourceMapKey!, 'type'))
+      errorMessage += `Type "${invalidConfig.type}" is not valid\n${codeSnippet}`
+    }
+
+    return errorMessage;
   }
 }
 
@@ -31,12 +128,28 @@ export class InvalidResourceError extends Error {
   }
 }
 
-export class JsonFileParseError extends Error {
+export class SyntaxError extends CodifyError {
   name = 'JsonFileParseError'
   fileName!: string;
 
-  constructor(props: RemoveErrorMethods<JsonFileParseError>) {
+  constructor(props: RemoveErrorMethods<SyntaxError>) {
     super(props.message)
     Object.assign(this, props);
   }
+
+  formattedMessage(): string {
+    return `Syntax error: found in codify.json: ${this.message}`
+  }
+}
+
+export function prettyPrintError(error: unknown): void {
+  if (error instanceof CodifyError) {
+    return console.error(chalk.red(error.formattedMessage()));
+  }
+
+  if (error instanceof Error) {
+    return console.error(chalk.red(error.message));
+  }
+
+  console.error(chalk.red(String(error)));
 }

src/common/orchestrator.ts

@@ -1,5 +1,5 @@
 import { Project } from '../entities/project.js';
-import { ctx, SubProcessName } from '../events/context.js';
+import { SubProcessName, ctx } from '../events/context.js';
 import { DependencyMap, PluginManager } from '../plugins/plugin-manager.js';
 
 export const CommonOrchestrator = {

src/entities/config.ts

@@ -1,8 +1,10 @@
-import { ConfigClass } from '../parser/language-definition.js';
+export enum ConfigType {
+  PROJECT = 'project',
+  RESOURCE = 'resource',
+}
+
 
 export interface ConfigBlock {
-  configClass: ConfigClass;
+  configClass: ConfigType;
   type: string;
-
-  validateConfig(config: unknown): never | void;
 }