From 658f17983565b92136bb474f4a12c271143005e1 Mon Sep 17 00:00:00 2001 From: dittnamn Date: Thu, 30 Jul 2026 20:36:12 +0200 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=8E=A8=20Changed=20and=20improved=20t?= =?UTF-8?q?he=20slug=20generation=20and=20validation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ref towards https://github.com/TryGhost/Ghost/issues/3224 etc. In line with the changes in https://github.com/TryGhost/SDK/pull/1011 a change in the validator is necessary to allow unicode slugs to be used. The validator now allows all Unicode letters and numbers, along with spaces, underscores and dashes that can be used as slug separators. Along with the changes in the slugify() function, equivalent changes were made to the security.safe() function to allow it to use the same parameters as slugify() and to make sure the tests will still work with the new transliteration library. However, the security.safe() function is barely used inside Ghost, and could easily be replaced to use slugify() directly. A TODO notice has been added, so that it could potentially be removed in the future. --- packages/security/lib/string.js | 16 ++++++++++--- packages/security/test/string.test.js | 29 ++++++++++++++++++----- packages/validator/lib/validator.js | 4 +++- packages/validator/test/internals.test.js | 15 +++++++++++- 4 files changed, 53 insertions(+), 11 deletions(-) diff --git a/packages/security/lib/string.js b/packages/security/lib/string.js index 3dc1865ea..ce4866409 100644 --- a/packages/security/lib/string.js +++ b/packages/security/lib/string.js @@ -1,9 +1,19 @@ const slugify = require('@tryghost/string').slugify; +// @TODO: the safe() function can possibly be removed from here if Ghost uses slugify() directly instead + /** - * @param {string} string - * @param {{importing?: boolean}} [options] + * @param {string} string - the string we want to slugify + * @param {object} [options] - filter options + * @param {bool} [options.importing] - don't perform optional cleanup, e.g. removing extra dashes + * @param {bool} [options.unicodeSlugs] - don't perform optional transliteration, e.g. keep smörgåsbord as it is instead of turning it into smorgasbord + * @param {string} [options.slugSeparator] - separator to be used for the slugs, can be ` `, `_` or `-`, defaults to `-` + * @returns {string} - a slugified string */ module.exports.safe = function safe(string, options = {}) { - return slugify(string, { requiredChangesOnly: options.importing === true }); + return slugify(string, { + requiredChangesOnly: options.importing === true, + unicodeSlugs: options.unicodeSlugs === true, + slugSeparator: options.slugSeparator + }); }; diff --git a/packages/security/test/string.test.js b/packages/security/test/string.test.js index 0f572be34..0b339af59 100644 --- a/packages/security/test/string.test.js +++ b/packages/security/test/string.test.js @@ -15,7 +15,7 @@ describe('Lib: Security - String', function () { assert.equal(result, ''); }); - it('should remove non ascii characters', function () { + it('should remove non-letter characters', function () { const result = security.string.safe('howtowin✓', options); assert.equal(result, 'howtowin'); }); @@ -37,9 +37,9 @@ describe('Lib: Security - String', function () { // note: This is missing the soft-hyphen char that isn't much-liked by linters/browsers/etc, // it passed the test before it was removed const result = security.string.safe( - '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿', + '!"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~¡¢£¤¥¦§¨©ª«¬®¯°±²_³´µ¶·¸¹º»¼½¾¿', ); - assert.equal(result, '_-c-y-ss-c-a-r-deg-23up-1o-1-41-23-4'); + assert.equal(result, 'a-2_3u-1o-1-41-23-4'); }); it('should replace all of the foreign chars in ascii', function () { @@ -48,7 +48,7 @@ describe('Lib: Security - String', function () { ); assert.equal( result, - 'aaaaaaaeceeeeiiiidnoooooxouuuuythssaaaaaaaeceeeeiiiidnooooo-ouuuuythy', + 'aaaaaaae-ceeeeiiiidnooooo-ouuuuythssaaaaaaaeceeeeiiiidnooooo-ouuuuythy', ); }); @@ -83,16 +83,33 @@ describe('Lib: Security - String', function () { }); it('should properly handle unicode punctuation conversion', function () { + // note: the previous unidecode transformation handled this differently than anyascii, so this is + // a compromise that's "good enough" and gives the most optimal results for most languages + // result using unidecode was: nijian-wei-iganaika-zai-du-que-ren-sitekudasai-zai-du-miip-misitekudasai const result = security.string.safe( 'に間違いがないか、再度確認してください。再読み込みしてください。', options, ); assert.equal( result, - 'nijian-wei-iganaika-zai-du-que-ren-sitekudasai-zai-du-miip-misitekudasai', + 'ni-jian-weiiganaika-zai-du-que-renshitekudasai-zai-dumi-yumishitekudasai', ); }); + it('should not transliterate the slugs if the unicodeSlugs flag is passed', function () { + let result = security.string.safe('Ett smörgåsbord från Sydkorea: 스뫼르고스보르드', { + unicodeSlugs: true, + }); + assert.equal(result, 'ett-smörgåsbord-från-sydkorea-스뫼르고스보르드'); + }); + + it('should not replace existing dashes and underscores when the slugSeparator is set to spaces', function () { + let result = security.string.safe('Herr./Klaus-Jürgen_44', { + slugSeparator: ' ', + }); + assert.equal(result, 'herr klaus-jurgen_44'); + }); + it('should not lose or convert dashes if options are passed with truthy importing flag', function () { let result = security.string.safe('-slug-with-starting-ending-and---multiple-dashes-', { importing: true, @@ -104,7 +121,7 @@ describe('Lib: Security - String', function () { let result = security.string.safe("-slug-&with-✓-invalid-characters-に'", { importing: true, }); - assert.equal(result, '-slug--with--invalid-characters-ni'); + assert.equal(result, '-slug--with---invalid-characters-ni'); }); }); }); diff --git a/packages/validator/lib/validator.js b/packages/validator/lib/validator.js index c6650c185..20e9b9411 100644 --- a/packages/validator/lib/validator.js +++ b/packages/validator/lib/validator.js @@ -47,7 +47,9 @@ validators.isEmptyOrURL = function isEmptyOrURL(str) { validators.isSlug = function isSlug(str) { assertString(str); - return validators.matches(str, /^[a-z0-9\-_]+$/); + // Note that combining marks aren't included here (\p{M}). The slugs should always be normalized using + // NFC before being used, so if a slug contains a combining mark when getting here, it's invalid. + return validators.matches(str, /^[\p{L}\p{N} _-]+$/u); }; validators.isEmail = function isEmail(str, options = { legacy: true }) { diff --git a/packages/validator/test/internals.test.js b/packages/validator/test/internals.test.js index 2dc13b2a6..6f44ba6d0 100644 --- a/packages/validator/test/internals.test.js +++ b/packages/validator/test/internals.test.js @@ -46,7 +46,20 @@ describe('Validator internals', function () { it('isSlug validates slug format', function () { assert.equal(validator.isSlug('a-valid_slug-1'), true); - assert.equal(validator.isSlug('not valid slug'), false); + assert.equal(validator.isSlug('not? valid slug'), false); + }); + + it('isSlug rejects slugs with NFD combining marks, in this case U+0301', function () { + assert.equal(validator.isSlug('café'.normalize('NFC')), true); + assert.equal(validator.isSlug('café'.normalize('NFD')), false); + }); + + it('isSlug accepts foreign characters, in this case Japanese', function () { + assert.equal(validator.isSlug('に間違いがないか_再度確認してください_再読み込みしてください'), true); + }); + + it('isSlug rejects non-letter characters, like emojis', function () { + assert.equal(validator.isSlug('im-so-happy-today-🙃'), false); }); it('custom validators enforce string input', function () { From ac2ad0bc1150adbf80f87e6c3f1bee2fd9b962e5 Mon Sep 17 00:00:00 2001 From: dittnamn Date: Fri, 31 Jul 2026 15:35:23 +0200 Subject: [PATCH 2/4] Updated validator and test to match the changes in slugify() Combining marks are now permitted to be used in a valid slug, but to avoid misuse, marks in the beginning of a slug or more than three to a letter in other places means the slug is invalid. --- packages/validator/lib/validator.js | 8 +++++--- packages/validator/test/internals.test.js | 13 ++++++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/validator/lib/validator.js b/packages/validator/lib/validator.js index 20e9b9411..dd71fd54c 100644 --- a/packages/validator/lib/validator.js +++ b/packages/validator/lib/validator.js @@ -47,9 +47,11 @@ validators.isEmptyOrURL = function isEmptyOrURL(str) { validators.isSlug = function isSlug(str) { assertString(str); - // Note that combining marks aren't included here (\p{M}). The slugs should always be normalized using - // NFC before being used, so if a slug contains a combining mark when getting here, it's invalid. - return validators.matches(str, /^[\p{L}\p{N} _-]+$/u); + // The slugs should always be normalized with NFC before being used, but some languages rely on + // combining marks to create letters. To avoid misuse, the slugify() function only generates slugs + // with a natural number of combining marks. Marks in the beginning of a slug means they're invalid, + // and in the rest of the slug a maximum of three combining marks is permitted to each letter. + return validators.matches(str, /^(?!\p{M})(?!.*[\p{Mn}\p{Mc}]{4,})[\p{L}\p{N}\p{Mn}\p{Mc} _-]+$/u); }; validators.isEmail = function isEmail(str, options = { legacy: true }) { diff --git a/packages/validator/test/internals.test.js b/packages/validator/test/internals.test.js index 6f44ba6d0..6d1dd4f02 100644 --- a/packages/validator/test/internals.test.js +++ b/packages/validator/test/internals.test.js @@ -49,9 +49,16 @@ describe('Validator internals', function () { assert.equal(validator.isSlug('not? valid slug'), false); }); - it('isSlug rejects slugs with NFD combining marks, in this case U+0301', function () { - assert.equal(validator.isSlug('café'.normalize('NFC')), true); - assert.equal(validator.isSlug('café'.normalize('NFD')), false); + it('isSlug rejects slugs with too many combining marks, but accept some of them', function () { + // note that this might break some text editors, so it might need to be removed + assert.equal(validator.isSlug('ghost-blo̵̯͝ǵ'), true); + assert.equal(validator.isSlug('G̸̛̦̼̜̱̹̦̲̩̰̀̓̆̇̔̎̒̎h̸͕̹̤̿͌́͊͋̈̂͗̕o̶̠͑̍s̷̝̭̰̳̖̣͉̈́̌̐́̈́̒͂̚t̴̩̦̫̟̲̘̆̔̑̅͘̕͠͝͠ ̶̜̺͚̆̈ͅb̸̰͕͔͈̤̾̉͒̂̎ͅl̵̳͚̘̯̀̎o̵̯͝ǵ̴̨̛͍̞͙̲̦̗̖͍̂̈́͆͝'), false); + }); + + it('isSlug rejects slugs with combining marks in the beginning of a text', function () { + // note that this might break some text editors, so it might need to be removed + assert.equal(validator.isSlug('ผีในวัฒนธรรมไทย'), true); + assert.equal(validator.isSlug('\u0303\u0301\u0302ผีในวัฒนธรรมไทย'), false); }); it('isSlug accepts foreign characters, in this case Japanese', function () { From fb4d3de0eeda7f7a79f606a5e5b1ec653d8918f1 Mon Sep 17 00:00:00 2001 From: dittnamn Date: Fri, 31 Jul 2026 16:24:03 +0200 Subject: [PATCH 3/4] Updated validator to disallow leading combining marks after a separator --- packages/validator/lib/validator.js | 7 +++++-- packages/validator/test/internals.test.js | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/validator/lib/validator.js b/packages/validator/lib/validator.js index dd71fd54c..a40433057 100644 --- a/packages/validator/lib/validator.js +++ b/packages/validator/lib/validator.js @@ -49,9 +49,12 @@ validators.isSlug = function isSlug(str) { assertString(str); // The slugs should always be normalized with NFC before being used, but some languages rely on // combining marks to create letters. To avoid misuse, the slugify() function only generates slugs - // with a natural number of combining marks. Marks in the beginning of a slug means they're invalid, + // with a natural number of combining marks. Marks in the beginning of a word means they're invalid, // and in the rest of the slug a maximum of three combining marks is permitted to each letter. - return validators.matches(str, /^(?!\p{M})(?!.*[\p{Mn}\p{Mc}]{4,})[\p{L}\p{N}\p{Mn}\p{Mc} _-]+$/u); + return validators.matches( + str, + /^(?!\p{M})(?!.*[\p{Mn}\p{Mc}]{4,})[\p{L}\p{N}\p{Mn}\p{Mc} _-]+$/u + ); }; validators.isEmail = function isEmail(str, options = { legacy: true }) { diff --git a/packages/validator/test/internals.test.js b/packages/validator/test/internals.test.js index 6d1dd4f02..a5318d648 100644 --- a/packages/validator/test/internals.test.js +++ b/packages/validator/test/internals.test.js @@ -55,10 +55,11 @@ describe('Validator internals', function () { assert.equal(validator.isSlug('G̸̛̦̼̜̱̹̦̲̩̰̀̓̆̇̔̎̒̎h̸͕̹̤̿͌́͊͋̈̂͗̕o̶̠͑̍s̷̝̭̰̳̖̣͉̈́̌̐́̈́̒͂̚t̴̩̦̫̟̲̘̆̔̑̅͘̕͠͝͠ ̶̜̺͚̆̈ͅb̸̰͕͔͈̤̾̉͒̂̎ͅl̵̳͚̘̯̀̎o̵̯͝ǵ̴̨̛͍̞͙̲̦̗̖͍̂̈́͆͝'), false); }); - it('isSlug rejects slugs with combining marks in the beginning of a text', function () { + it('isSlug rejects slugs with combining marks in the beginning of a text or a word', function () { // note that this might break some text editors, so it might need to be removed assert.equal(validator.isSlug('ผีในวัฒนธรรมไทย'), true); assert.equal(validator.isSlug('\u0303\u0301\u0302ผีในวัฒนธรรมไทย'), false); + assert.equal(validator.isSlug('ghost-\u0301\u0302blog'), false); }); it('isSlug accepts foreign characters, in this case Japanese', function () { From f0582c0f254a2a5c514a752907b16594ace0d94b Mon Sep 17 00:00:00 2001 From: dittnamn Date: Fri, 31 Jul 2026 17:50:10 +0200 Subject: [PATCH 4/4] Updated and simplified validator regex to do as intended along with extra testing --- packages/validator/lib/validator.js | 5 +---- packages/validator/test/internals.test.js | 7 +++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/validator/lib/validator.js b/packages/validator/lib/validator.js index a40433057..99431978c 100644 --- a/packages/validator/lib/validator.js +++ b/packages/validator/lib/validator.js @@ -51,10 +51,7 @@ validators.isSlug = function isSlug(str) { // combining marks to create letters. To avoid misuse, the slugify() function only generates slugs // with a natural number of combining marks. Marks in the beginning of a word means they're invalid, // and in the rest of the slug a maximum of three combining marks is permitted to each letter. - return validators.matches( - str, - /^(?!\p{M})(?!.*[\p{Mn}\p{Mc}]{4,})[\p{L}\p{N}\p{Mn}\p{Mc} _-]+$/u - ); + return validators.matches(str, /^(?:[\p{L}\p{N}][\p{Mn}\p{Mc}]{0,3}|[ _-])+$/u); }; validators.isEmail = function isEmail(str, options = { legacy: true }) { diff --git a/packages/validator/test/internals.test.js b/packages/validator/test/internals.test.js index a5318d648..41c9f98cf 100644 --- a/packages/validator/test/internals.test.js +++ b/packages/validator/test/internals.test.js @@ -49,6 +49,13 @@ describe('Validator internals', function () { assert.equal(validator.isSlug('not? valid slug'), false); }); + it('isSlug accepts slugs generated without optional changes', function () { + assert.equal(validator.isSlug('-separator-in-beginning'), true); + assert.equal(validator.isSlug('-separator-in-beginning-and-end-'), true); + assert.equal(validator.isSlug('multiple---separators-in--use'), true); + assert.equal(validator.isSlug('different__kinds-of separator___in-the-same-text'), true); + }); + it('isSlug rejects slugs with too many combining marks, but accept some of them', function () { // note that this might break some text editors, so it might need to be removed assert.equal(validator.isSlug('ghost-blo̵̯͝ǵ'), true);