🎨 Added an option to use unicode slugs in URL:s

apps/admin-x-settings/src/components/settings/advanced/labs/beta-features.tsx

@@ -3,26 +3,62 @@ import LabItem from './lab-item';
 import NiceModal from '@ebay/nice-modal-react';
 import React, {useState} from 'react';
 import YamlFileEditorModal from './yaml-file-editor-modal';
-import {ActionList, Button, Dropzone} from '@tryghost/shade/components';
+import {ActionList, Button, Dropzone, Select, SelectContent, SelectItem, SelectTrigger, SelectValue} from '@tryghost/shade/components';
 import {Inline, Stack} from '@tryghost/shade/primitives';
 import {downloadRedirects, useUploadRedirects} from '@tryghost/admin-x-framework/api/redirects';
 import {downloadRoutes, useUploadRoutes} from '@tryghost/admin-x-framework/api/routes';
-import {getSettingValue} from '@tryghost/admin-x-framework/api/settings';
+import {getSettingValue, useEditSettings} from '@tryghost/admin-x-framework/api/settings';
 import {toast} from 'sonner';
 import {useGlobalData} from '../../../providers/global-data-provider';
 import {useHandleError} from '@tryghost/admin-x-framework/hooks';
 
 const IS_AUTOMATIONS_BETA_ACTIVE = true;
 
+const SLUG_SEPARATORS = [
+    {
+        value: '-',
+        label: 'Dashes [-]',
+        hint: 'The default mode (e.g. /example-ghost-post/).'
+    },
+    {
+        value: '_',
+        label: 'Underscores [_]',
+        hint: 'A mode with better readability and clearer distinction (e.g. /example_ghost_post/).'
+    },
+    {
+        value: ' ',
+        label: 'Spaces [ ]',
+        hint: 'A natural mode, but might look foreign in URL:s (eg. /example ghost post/). Also, many browsers will show spaces as %20 in the URL:s.'
+    }
+];
+
+const renderSlugSeparatorOptions = () => {
+    return SLUG_SEPARATORS.map(option => (
+        <SelectItem key={option.value} value={option.value}>
+            <span className='flex flex-col'>
+                <span>{option.label}</span>
+                <span className='text-sm text-muted-foreground'>
+                    {option.hint}
+                </span>
+            </span>
+        </SelectItem>
+    ));
+};
+
+const getSlugSeparatorOptionLabel = (value: string) =>
+    SLUG_SEPARATORS.find(option => option.value === value)?.label;
+
 const BetaFeatures: React.FC = () => {
     const {settings} = useGlobalData();
     const {mutateAsync: uploadRedirects} = useUploadRedirects();
     const {mutateAsync: uploadRoutes} = useUploadRoutes();
+    const {mutateAsync: editSettings} = useEditSettings();
     const handleError = useHandleError();
     const [redirectsUploading, setRedirectsUploading] = useState<boolean>(false);
     const [routesUploading, setRoutesUploading] = useState<boolean>(false);
     const labs = JSON.parse(getSettingValue<string>(settings, 'labs') || '{}');
     const isAutomationsEnabled = !!labs.automations;
+    const slugSeparator = getSettingValue<string>(settings, 'slug_separator') || '-';
 
     const openRedirectsEditor = () => {
         NiceModal.show(YamlFileEditorModal, {
@@ -73,6 +109,25 @@ const BetaFeatures: React.FC = () => {
                 action={<FeatureToggle flag="editorExcerpt" />}
                 detail={<>Adds the excerpt input below the post title in the editor</>}
                 title='Show post excerpt inline' />
+            <LabItem
+                action={<div className='flex w-full max-w-none min-w-[160px] flex-col items-end gap-3 md:w-2/3 md:max-w-[320px] md:flex-1'>
+                    <FeatureToggle flag="unicodeSlugs" />
+                    <Select
+                        disabled={!labs.unicodeSlugs}
+                        value={slugSeparator ?? '-'}
+                        onValueChange={async (value) => {
+                                await editSettings([{
+                                    key: 'slug_separator', value
+                                }]);
+                        }}>
+                        <SelectTrigger aria-label='Use Unicode letters and numbers in URL slugs instead of transliterating them' className='w-full border-transparent bg-muted hover:bg-muted'>
+                            <SelectValue>{getSlugSeparatorOptionLabel(slugSeparator)}</SelectValue>
+                        </SelectTrigger>
+                        <SelectContent>{renderSlugSeparatorOptions()}</SelectContent>
+                    </Select>
+                </div>}
+                detail={<>Use Unicode letters and numbers in URL slugs instead of transliterating them (e.g /smörgåsbord/ instead of /smorgasbord/), which may add benefits for SEO. You can also select another slug separator to adjust the look of the URL:s.</>}
+                title='International slugs' />
             <LabItem
                 action={<FeatureToggle flag="additionalPaymentMethods" />}
                 detail={<>Enable support for CashApp, iDEAL, Bancontact, and others. <a className='text-green' href="https://ghost.org/help/payment-methods" rel="noopener noreferrer" target="_blank">Learn more &rarr;</a></>}

apps/admin-x-settings/src/typings.d.ts

@@ -1,5 +1,8 @@
 declare module '@tryghost/limit-service'
 declare module '@tryghost/nql'
+declare module '@tryghost/string' {
+    export function slugify(string: string, options?: {requiredChangesOnly?: boolean, unicodeSlugs?: boolean, slugSeparator?: string}): string;
+}
 
 declare module '*.svg' {
     // eslint-disable-next-line @typescript-eslint/no-require-imports

apps/admin/src/vite-env.d.ts

@@ -3,6 +3,6 @@
 declare module '@tryghost/limit-service'
 declare module '@tryghost/nql'
 declare module '@tryghost/string' {
-    export function slugify(string: string, options?: {requiredChangesOnly?: boolean}): string;
+    export function slugify(string: string, options?: {requiredChangesOnly?: boolean, unicodeSlugs?: boolean, slugSeparator?: string}): string;
 }
 declare module '@tryghost/koenig-lexical'

apps/ember-admin/app/components/tags/tag-form.js

@@ -136,7 +136,7 @@ export default class TagForm extends Component {
 
         // Generate slug based on name for new tag when empty
         if (property === 'name' && tag.isNew && !this.hasChangedSlug) {
-            let slugValue = slugify(newValue);
+            let slugValue = slugify(newValue, {unicodeSlugs: this.feature.unicodeSlugs, slugSeparator: (this.feature.unicodeSlugs ? this.settings.slugSeparator : undefined)});
             if (/^#/.test(newValue)) {
                 slugValue = 'hash-' + slugValue;
             }

apps/ember-admin/app/controllers/lexical-editor.js

@@ -956,7 +956,7 @@ export default class LexicalEditorController extends Controller {
 
         // Update the slug unless the slug looks to be a custom slug or the title is a default/has been cleared out
         if (
-            (currentSlug && slugify(currentTitle) !== currentSlug)
+            (currentSlug && slugify(currentTitle, {unicodeSlugs: this.feature.unicodeSlugs, slugSeparator: (this.feature.unicodeSlugs ? this.settings.slugSeparator : undefined)}) !== currentSlug)
             && !(currentTitle === DEFAULT_TITLE || currentTitle?.endsWith(DUPLICATED_POST_TITLE_SUFFIX))
         ) {
             return;

apps/ember-admin/app/models/setting.js

@@ -132,6 +132,11 @@ export default Model.extend(ValidationEngine, {
      */
     transistor: attr('boolean'),
 
+    /**
+     * Settings for Labs options
+     */
+    slugSeparator: attr('string'),
+
     // HACK - not a real model attribute but a workaround for Ember Data not
     //        exposing meta from save responses
     _meta: attr()

apps/ember-admin/app/services/feature.js

@@ -82,6 +82,7 @@ export default class FeatureService extends Service {
     @feature('adminUIRefresh') adminUIRefresh;
     @feature('lexicalIndicators') lexicalIndicators;
     @feature('editorExcerpt') editorExcerpt;
+    @feature('unicodeSlugs') unicodeSlugs;
     @feature('memberDetailsReact') memberDetailsReact;
     @feature('previewByTier') previewByTier;
     @feature('paywallImprovements') paywallImprovements;

apps/ember-admin/app/services/slug-generator.js

@@ -9,6 +9,8 @@ const {resolve} = RSVP;
 export default class SlugGeneratorService extends Service {
     @service ghostPaths;
     @service ajax;
+    @service feature;
+    @service settings;
 
     generateSlug(slugType, textToSlugify, modelId) {
         let url;
@@ -18,14 +20,19 @@ export default class SlugGeneratorService extends Service {
         }
 
         // We already do a partial slugify at the client side to prevent issues with Pro returning a 404 page because of invalid (encoded) characters (a newline, %0A, for example)
-        let name = encodeURIComponent(slugify(textToSlugify));
+        let name = encodeURIComponent(slugify(textToSlugify, {unicodeSlugs: this.feature.unicodeSlugs, slugSeparator: (this.feature.unicodeSlugs ? this.settings.slugSeparator : undefined)}));
         if (modelId) {
             url = this.get('ghostPaths.url').api('slugs', slugType, name, modelId);
         } else {
             url = this.get('ghostPaths.url').api('slugs', slugType, name);
         }
 
-        return this.ajax.request(url).then((response) => {
+        return this.ajax.request(url, {
+            data: {
+                unicodeSlugs: this.feature.unicodeSlugs,
+                slugSeparator: (this.feature.unicodeSlugs ? this.settings.slugSeparator : undefined)
+            }
+        }).then((response) => {
             let [firstSlug] = response.slugs;
             let {slug} = firstSlug;
 

ghost/core/core/frontend/helpers/navigation.js

@@ -7,6 +7,8 @@ const errors = require('@tryghost/errors');
 const tpl = require('@tryghost/tpl');
 const {slugify} = require('@tryghost/string');
 const _ = require('lodash');
+const labs = require('../../shared/labs');
+const {settingsCache} = require('../services/proxy');
 
 const messages = {
     invalidData: 'navigation data is not an object or is a function',
@@ -75,7 +77,7 @@ module.exports = function navigation(options) {
         const out = {};
         out.current = _isCurrentUrl(e.url, currentUrl);
         out.label = e.label;
-        out.slug = slugify(e.label);
+        out.slug = slugify(e.label, {unicodeSlugs: labs.isSet('unicodeSlugs'), slugSeparator: (labs.isSet('unicodeSlugs') ? settingsCache.get('slug_separator') : undefined)});
         out.url = e.url;
         return out;
     });

ghost/core/core/frontend/services/data/entry-lookup.js

@@ -48,6 +48,11 @@ function entryLookup(postUrl, routerOptions, locals, {giftToken} = {}) {
         options.context.giftToken = giftToken;
     }
 
+    // In case this is a slug we normalize it to make sure it matches the form that the database uses
+    if (params.slug) {
+        params.slug = params.slug.normalize('NFC');
+    }
+
     return (api[routerOptions.query.controller] || api[routerOptions.query.resource])
         .read(_.extend(_.pick(params, 'slug', 'id'), options))
         .then(function then(result) {

ghost/core/core/frontend/services/data/fetch-data.js

@@ -89,6 +89,11 @@ async function fetchData(pathOptions, routerOptions, locals) {
         postQuery.options.limit = pathOptions.limit;
     }
 
+    // In case this is a slug we normalize it to make sure it matches the form that the database uses
+    if (pathOptions.slug) {
+        pathOptions.slug = pathOptions.slug.normalize('NFC');
+    }
+
     // CASE: always fetch post entries
     // The filter can in theory contain a "%s" e.g. filter="primary_tag:%s"
     promises.push(processQuery(postQuery, pathOptions.slug, locals));

ghost/core/core/frontend/services/routing/controllers/channel.js

@@ -1,7 +1,6 @@
 const debug = require('@tryghost/debug')('services:routing:controllers:channel');
 const tpl = require('@tryghost/tpl');
 const errors = require('@tryghost/errors');
-const security = require('@tryghost/security');
 const themeEngine = require('../../theme-engine');
 const dataService = require('../../data');
 const renderer = require('../../rendering');
@@ -25,7 +24,7 @@ module.exports = function channelController(req, res, next) {
 
     const pathOptions = {
         page: req.params.page !== undefined ? req.params.page : 1,
-        slug: req.params.slug ? security.string.safe(req.params.slug) : undefined
+        slug: req.params.slug
     };
 
     if (pathOptions.page) {

ghost/core/core/frontend/services/routing/controllers/collection.js

@@ -2,7 +2,6 @@ const _ = require('lodash');
 const debug = require('@tryghost/debug')('services:routing:controllers:collection');
 const tpl = require('@tryghost/tpl');
 const errors = require('@tryghost/errors');
-const security = require('@tryghost/security');
 const {routerManager} = require('../');
 const themeEngine = require('../../theme-engine');
 const renderer = require('../../rendering');
@@ -24,7 +23,7 @@ module.exports = function collectionController(req, res, next) {
 
     const pathOptions = {
         page: req.params.page !== undefined ? req.params.page : 1,
-        slug: req.params.slug ? security.string.safe(req.params.slug) : undefined
+        slug: req.params.slug
     };
 
     if (pathOptions.page) {

ghost/core/core/frontend/services/routing/controllers/rss.js

@@ -1,7 +1,6 @@
 const _ = require('lodash');
 const debug = require('@tryghost/debug')('services:routing:controllers:rss');
 const url = require('url');
-const security = require('@tryghost/security');
 const settingsCache = require('../../../../shared/settings-cache');
 const rssService = require('../../rss');
 const renderer = require('../../rendering');
@@ -29,7 +28,7 @@ module.exports = function rssController(req, res, next) {
 
     const pathOptions = {
         page: 1, // required for fetchData
-        slug: req.params.slug ? security.string.safe(req.params.slug) : undefined
+        slug: req.params.slug
     };
 
     // CASE: Ghost is using an rss cache - normalize the URL for use as a key

ghost/core/core/server/api/endpoints/slugs.js

@@ -22,7 +22,9 @@ const controller = {
         options: [
             'include',
             'type',
-            'id'
+            'id',
+            'unicodeSlugs',
+            'slugSeparator'
         ],
         data: [
             'name'
@@ -36,6 +38,12 @@ const controller = {
                 },
                 id: {
                     required: false
+                },
+                unicodeSlugs: {
+                    required: false
+                },
+                slugSeparator: {
+                    required: false
                 }
             },
             data: {
@@ -45,7 +53,7 @@ const controller = {
             }
         },
         async query(frame) {
-            const slug = await models.Base.Model.generateSlug(allowedTypes[frame.options.type], frame.data.name, {status: 'all', modelId: frame.options.id});
+            const slug = await models.Base.Model.generateSlug(allowedTypes[frame.options.type], frame.data.name, {status: 'all', modelId: frame.options.id, unicodeSlugs: frame.options.unicodeSlugs, slugSeparator: frame.options.slugSeparator});
             if (!slug) {
                 throw new errors.InternalServerError({
                     message: tpl(messages.couldNotGenerateSlug)

ghost/core/core/server/api/endpoints/utils/serializers/input/settings.js

@@ -88,6 +88,7 @@ const EDITABLE_SETTINGS = [
     'explore_ping',
     'explore_ping_growth',
     'indexnow_api_key',
+    'slug_separator',
     'transistor',
     'transistor_portal_enabled',
     'transistor_portal_heading',

ghost/core/core/server/api/endpoints/utils/serializers/input/utils/settings-key-group-mapper.js

@@ -40,6 +40,7 @@ const keyGroupMapping = {
     twitter_image: 'site',
     twitter_title: 'site',
     twitter_description: 'site',
+    slug_separator: 'site',
     active_theme: 'theme',
     is_private: 'private',
     password: 'private',

ghost/core/core/server/api/endpoints/utils/serializers/input/utils/settings-key-type-mapper.js

@@ -32,6 +32,7 @@ const keyTypeMapping = {
     twitter_image: 'string',
     twitter_title: 'string',
     twitter_description: 'string',
+    slug_separator: 'string',
     active_theme: 'string',
     password: 'string',
     public_hash: 'string',

ghost/core/core/server/data/exporter/export-filename.js

@@ -1,7 +1,7 @@
 const _ = require('lodash');
 const logging = require('@tryghost/logging');
 const errors = require('@tryghost/errors');
-const security = require('@tryghost/security');
+const {slugify} = require('@tryghost/string');
 const models = require('../../models');
 const path = require('path');
 
@@ -22,7 +22,7 @@ const exportFileName = async function exportFileName(options) {
         const settingsTitle = await models.Settings.findOne({key: 'title'}, _.merge({}, modelOptions, _.pick(options, 'transacting')));
 
         if (settingsTitle) {
-            title = security.string.safe(settingsTitle.get('value')) + '.';
+            title = slugify(settingsTitle.get('value')) + '.';
         }
 
         return title + 'ghost.' + datetime + '.json';

ghost/core/core/server/data/schema/default-settings/default-settings.json

@@ -265,6 +265,15 @@
                 }
             },
             "type": "string"
+        },
+        "slug_separator": {
+            "defaultValue": "-",
+            "validations": {
+                "isLength": {
+                    "max": 1
+                }
+            },
+            "type": "string"
         }
     },
     "theme": {

ghost/core/core/server/models/base/plugins/generate-slug.js

@@ -1,5 +1,5 @@
 const _ = require('lodash');
-const security = require('@tryghost/security');
+const {slugify} = require('@tryghost/string');
 
 const urlUtils = require('../../../../shared/url-utils').default;
 
@@ -70,7 +70,7 @@ module.exports = function (Bookshelf) {
                 });
             };
 
-            slug = security.string.safe(base, options);
+            slug = slugify(base, { requiredChangesOnly: options.importing, unicodeSlugs: options.unicodeSlugs, slugSeparator: options.slugSeparator });
 
             // the slug may never be longer than the allowed limit of 191 chars, but should also
             // take the counter into count. We reduce a too long slug to 185 so we're always on the
@@ -125,4 +125,6 @@ module.exports = function (Bookshelf) {
  * @property {boolean} [importing] Set to true to don't cut the slug on import
  * @property {boolean} [shortSlug] If it's a user, let's try to cut it down (unless this is a human request)
  * @property {boolean} [skipDuplicateChecks] Don't append unique identifiers when the slug is not unique (this prevents any database queries)
+ * @property {boolean} [unicodeSlugs] Don't perform optional transliteration, e.g. keep smörgåsbord as it is instead of turning it into smorgasbord
+ * @property {string} [slugSeparator] Separator to be used for the slugs, can be ` `, `_` or `-`, defaults to `-`
  */