feat(gitlab): two-way sync between Huly and GitLab

desktop/package.json

@@ -186,6 +186,9 @@
     "@hcengineering/github": "workspace:^0.7.0",
     "@hcengineering/github-assets": "workspace:^0.7.0",
     "@hcengineering/github-resources": "workspace:^0.7.0",
+    "@hcengineering/gitlab": "workspace:^0.7.0",
+    "@hcengineering/gitlab-assets": "workspace:^0.7.0",
+    "@hcengineering/gitlab-resources": "workspace:^0.7.0",
     "@hcengineering/desktop-preferences": "workspace:^0.7.0",
     "@hcengineering/desktop-preferences-assets": "workspace:^0.7.0",
     "@hcengineering/desktop-preferences-resources": "workspace:^0.7.0",

desktop/src/ui/platform.ts

@@ -226,6 +226,9 @@ import { ipcMainExposed } from './typesUtils'
 import github, { githubId } from '@hcengineering/github'
 import '@hcengineering/github-assets'
 import * as GithubLang from '@hcengineering/github-assets/lang'
+import gitlab, { gitlabId } from '@hcengineering/gitlab'
+import '@hcengineering/gitlab-assets'
+import * as GitlabLang from '@hcengineering/gitlab-assets/lang'
 import { preferenceId } from '@hcengineering/preference'
 import { uiId } from '@hcengineering/ui/src/plugin'
 
@@ -276,6 +279,7 @@ function configureI18n (): void {
   addStringsLoader(documentId, DocumentLang.loadLang)
   addStringsLoader(timeId, TimeLang.loadLang)
   addStringsLoader(githubId, GithubLang.loadLang)
+  addStringsLoader(gitlabId, GitlabLang.loadLang)
   addStringsLoader(documentsId, ControlledDocumentsLang.loadLang)
   addStringsLoader(productsId, ProductsLang.loadLang)
   addStringsLoader(questionsId, QuestionsLang.loadLang)
@@ -363,6 +367,7 @@ export async function configurePlatform (onWorkbenchConnect?: () => Promise<void
   setMetadata(github.metadata.GithubApplication, config.GITHUB_APP ?? '')
   setMetadata(github.metadata.GithubClientID, config.GITHUB_CLIENTID ?? '')
   setMetadata(github.metadata.GithubURL, config.GITHUB_URL ?? '')
+  setMetadata(gitlab.metadata.GitlabURL, config.GITLAB_URL ?? '')
 
   setMetadata(communication.metadata.Enabled, config.COMMUNICATION_API_ENABLED === 'true')
 
@@ -469,6 +474,7 @@ export async function configurePlatform (onWorkbenchConnect?: () => Promise<void
   addLocation(recorderId, async () => await import('@hcengineering/recorder-resources'))
   addLocation(presenceId, async () => await import('@hcengineering/presence-resources'))
   addLocation(githubId, async () => await import(/* webpackChunkName: "github" */ '@hcengineering/github-resources'))
+  addLocation(gitlabId, async () => await import(/* webpackChunkName: "gitlab" */ '@hcengineering/gitlab-resources'))
   addLocation(
     desktopPreferencesId,
     async () =>

desktop/src/ui/types.ts

@@ -40,6 +40,7 @@ export interface Config {
   GITHUB_APP: string
   GITHUB_CLIENTID: string
   GITHUB_URL: string
+  GITLAB_URL?: string
   GMAIL_URL: string
   INITIAL_URL: string
   LINK_PREVIEW_URL?: string

dev/docker-compose.gitlab.yaml

@@ -0,0 +1,36 @@
+# Dev overlay to run the GitLab integration service.
+#
+# Usage from the dev/ directory:
+#   docker compose -f docker-compose.yaml -f docker-compose.gitlab.yaml up -d
+#
+# Build the image first: rush docker:build --to @hcengineering/pod-gitlab
+# front already points GITLAB_URL at http://huly.local:3600 in docker-compose.yaml.
+# GitLab.com cannot reach huly.local: set GITLAB_WEBHOOK_BASE_URL to a tunnel URL (smee, ngrok) to receive webhooks.
+#
+services:
+  gitlab:
+    image: platformcollective/gitlab
+    extra_hosts:
+      - 'huly.local:host-gateway'
+    restart: unless-stopped
+    ports:
+      - 3600:3600
+    environment:
+      - PORT=3600
+      - SERVER_SECRET=secret
+      - ACCOUNTS_URL=http://huly.local:3000
+      - FRONT_URL=http://huly.local:8087
+      - COLLABORATOR_URL=ws://huly.local:3078
+      # Blob storage for merge request diffs (same as the other services); unset = no diffs
+      - STORAGE_CONFIG=${STORAGE_CONFIG}
+      # OAuth callback; register the same URL in the GitLab application. Empty = {FRONT_URL}/gitlab
+      - GITLAB_REDIRECT_URI=${GITLAB_REDIRECT_URI:-}
+      - WEBHOOK_BASE_URL=${GITLAB_WEBHOOK_BASE_URL:-http://huly.local:3600}
+      - WEBHOOK_SECRET=${GITLAB_WEBHOOK_SECRET:-secret}
+      # Dev only: lets a self-managed GitLab on http://localhost or *.local be used
+      - GITLAB_ALLOW_INSECURE_HOSTS=true
+      - GITLAB_READONLY=${GITLAB_READONLY:-false}
+      # Part B: comma list of trusted GitLab hosts (private addresses allowed); per-request timeout
+      - GITLAB_ALLOWED_HOSTS=${GITLAB_ALLOWED_HOSTS:-}
+      - GITLAB_REQUEST_TIMEOUT_MS=${GITLAB_REQUEST_TIMEOUT_MS:-30000}
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318/v1/traces

dev/docker-compose.yaml

@@ -282,6 +282,7 @@ services:
       - COLLABORATOR_URL=ws://huly.local:3078
       - STORAGE_CONFIG=${STORAGE_CONFIG}
       - GITHUB_URL=http://huly.local:3500
+      - GITLAB_URL=http://huly.local:3600
       - PRINT_URL=http://huly.local:4005
       - SIGN_URL=http://huly.local:4006
       #      - ANALYTICS_COLLECTOR_URL=http://huly.local:4017

dev/prod/package.json

@@ -132,6 +132,9 @@
     "@hcengineering/github": "workspace:^0.7.0",
     "@hcengineering/github-assets": "workspace:^0.7.0",
     "@hcengineering/github-resources": "workspace:^0.7.0",
+    "@hcengineering/gitlab": "workspace:^0.7.0",
+    "@hcengineering/gitlab-assets": "workspace:^0.7.0",
+    "@hcengineering/gitlab-resources": "workspace:^0.7.0",
     "@hcengineering/gmail": "workspace:^0.7.0",
     "@hcengineering/gmail-assets": "workspace:^0.7.0",
     "@hcengineering/gmail-resources": "workspace:^0.7.0",

dev/prod/public/config.json

@@ -12,6 +12,7 @@
   "GITHUB_APP": "uberflow-dev",
   "GITHUB_CLIENTID": "Iv1.43f9cac43bd68617",
   "GITHUB_URL": "http://huly.local:3500",
+  "GITLAB_URL": "http://huly.local:3600",
   "LAST_NAME_FIRST": "true",
   "PRINT_URL": "http://huly.local:4005",
   "SIGN_URL": "http://huly.local:4006",

dev/prod/src/platform.ts

@@ -75,6 +75,7 @@ import workbench, { workbenchId } from '@hcengineering/workbench'
 import { mailId } from '@hcengineering/mail'
 import { chatId } from '@hcengineering/chat'
 import github, { githubId } from '@hcengineering/github'
+import gitlab, { gitlabId } from '@hcengineering/gitlab'
 import { bitrixId } from '@hcengineering/bitrix'
 import { inboxId } from '@hcengineering/inbox'
 import { achievementId } from '@hcengineering/achievement'
@@ -191,6 +192,8 @@ import '@hcengineering/mail-assets'
 import * as MailLang from '@hcengineering/mail-assets/lang'
 import '@hcengineering/github-assets'
 import * as GithubLang from '@hcengineering/github-assets/lang'
+import '@hcengineering/gitlab-assets'
+import * as GitlabLang from '@hcengineering/gitlab-assets/lang'
 import '@hcengineering/achievement-assets'
 import * as AchievementLang from '@hcengineering/achievement-assets/lang'
 import '@hcengineering/communication-assets'
@@ -235,6 +238,7 @@ export interface Config {
   GITHUB_APP?: string
   GITHUB_CLIENTID?: string
   GITHUB_URL: string
+  GITLAB_URL?: string
   LOVE_ENDPOINT?: string
   LIVEKIT_WS?: string
   SIGN_URL?: string
@@ -384,6 +388,7 @@ function configureI18n(): void {
   addStringsLoader(documentId, DocumentLang.loadLang)
   addStringsLoader(timeId, TimeLang.loadLang)
   addStringsLoader(githubId, GithubLang.loadLang)
+  addStringsLoader(gitlabId, GitlabLang.loadLang)
   addStringsLoader(documentsId, ControlledDocumentsLang.loadLang)
   addStringsLoader(productsId, ProductsLang.loadLang)
   addStringsLoader(questionsId, QuestionsLang.loadLang)
@@ -523,6 +528,7 @@ export async function configurePlatform() {
   setMetadata(github.metadata.GithubApplication, config.GITHUB_APP ?? '')
   setMetadata(github.metadata.GithubClientID, config.GITHUB_CLIENTID ?? '')
   setMetadata(github.metadata.GithubURL, config.GITHUB_URL)
+  setMetadata(gitlab.metadata.GitlabURL, config.GITLAB_URL ?? '')
 
   setMetadata(rekoni.metadata.RekoniUrl, config.REKONI_URL)
 
@@ -558,6 +564,7 @@ export async function configurePlatform() {
       [loginId, login.component.LoginApp],
       [onboardId, onboard.component.OnboardApp],
       [githubId, github.component.ConnectApp],
+      [gitlabId, gitlab.component.ConnectApp],
       [calendarId, calendar.component.ConnectApp],
       [guestId, guest.component.GuestApp],
       [globalProfileRoute, globalProfile.component.GlobalProfileApp],
@@ -643,6 +650,7 @@ export async function configurePlatform() {
     async () => await import(/* webpackChunkName: "document" */ '@hcengineering/document-resources')
   )
   addLocation(githubId, async () => await import(/* webpackChunkName: "github" */ '@hcengineering/github-resources'))
+  addLocation(gitlabId, async () => await import(/* webpackChunkName: "gitlab" */ '@hcengineering/gitlab-resources'))
   addLocation(
     questionsId,
     async () => await import(/* webpackChunkName: "training" */ '@hcengineering/questions-resources')

foundations/core/packages/core/src/classes.ts

@@ -1010,6 +1010,7 @@ export interface WorkspaceMemberInfo {
 export enum SocialIdType {
   EMAIL = 'email',
   GITHUB = 'github',
+  GITLAB = 'gitlab',
   GOOGLE = 'google',
   PHONE = 'phone',
   OIDC = 'oidc',

models/all/package.json

@@ -105,6 +105,8 @@
     "@hcengineering/model-server-document": "workspace:^0.7.0",
     "@hcengineering/model-github": "workspace:^0.7.0",
     "@hcengineering/server-github-model": "workspace:^0.7.0",
+    "@hcengineering/server-gitlab-model": "workspace:^0.7.0",
+    "@hcengineering/model-gitlab": "workspace:^0.7.0",
     "@hcengineering/model-love": "workspace:^0.7.0",
     "@hcengineering/model-server-love": "workspace:^0.7.0",
     "@hcengineering/model-print": "workspace:^0.7.0",

models/all/src/index.ts

@@ -89,6 +89,8 @@ import { serverDocumentId, createModel as serverDocumentModel } from '@hcenginee
 
 import github, { githubId, createModel as githubModel } from '@hcengineering/model-github'
 import { serverGithubId, createModel as serverGithubModel } from '@hcengineering/server-github-model'
+import { serverGitlabId, createModel as serverGitlabModel } from '@hcengineering/server-gitlab-model'
+import gitlab, { gitlabId, createModel as gitlabModel } from '@hcengineering/model-gitlab'
 
 import { analyticsCollectorId, createModel as analyticsCollectorModel } from '@hcengineering/model-analytics-collector'
 import { exportId, createModel as exportModel } from '@hcengineering/model-export'
@@ -387,6 +389,18 @@ export default function buildModel (): Builder {
         classFilter: defaultFilter
       }
     ],
+    [
+      gitlabModel,
+      gitlabId,
+      {
+        label: gitlab.string.ConfigLabel,
+        description: gitlab.string.ConfigDescription,
+        enabled: false,
+        beta: true,
+        icon: gitlab.icon.Gitlab,
+        classFilter: defaultFilter
+      }
+    ],
     [
       loveModel,
       loveId,
@@ -561,6 +575,7 @@ export default function buildModel (): Builder {
     [serverActivityModel, serverActivityId],
     [serverDocumentModel, serverDocumentId],
     [serverGithubModel, serverGithubId],
+    [serverGitlabModel, serverGitlabId],
     [serverLoveModel, serverLoveId],
     [serverTimeModel, serverTimeId],
     [serverGuestModel, serverGuestId],

packages/safe-fetch/src/__tests__/ranges.test.ts

@@ -36,6 +36,7 @@ describe('isBlockedAddress', () => {
     '64:ff9b::8.8.8.8',
     '64:ff9b:1::a00:1',
     '64:ff9b:1:ffff::808:808',
+    '100::1',
     '::ffff:0:a00:1',
     '2002:0a00:0001::',
     '2002:0808:0808::',

packages/safe-fetch/src/ranges.ts

@@ -30,6 +30,7 @@ export const BLOCKED_IPV6_RANGES: readonly string[] = [
   '::1/128', // loopback
   '64:ff9b::/96', // NAT64 well-known prefix, blocked as a whole
   '64:ff9b:1::/48', // NAT64 local-use prefix (RFC 8215), blocked as a whole
+  '100::/64', // discard-only (RFC 6666)
   '2001::/32', // Teredo, blocked as a whole
   '2002::/16', // 6to4, blocked as a whole
   'fc00::/7', // unique local

plugins/diffview-resources/src/components/FileDiffView.svelte

@@ -40,7 +40,7 @@
   let hideDiffLabel = getHideDiffLabel(file)
 
   function getHideDiffLabel(file: DiffFile): IntlString | undefined {
-    if (diffRenderLimit >= 0 && file.stats.addedLines + file.stats.addedLines > diffRenderLimit) {
+    if (diffRenderLimit >= 0 && file.stats.addedLines + file.stats.deletedLines > diffRenderLimit) {
       return diffview.string.LargeDiffsAreHidden
     }
     if (file.diffType === 'delete') {

plugins/diffview-resources/src/index.ts

@@ -19,6 +19,8 @@ import DiffView from './components/DiffView.svelte'
 import Highlight from './components/Highlight.svelte'
 import InlineDiffView from './components/InlineDiffView.svelte'
 
+export { parseDiff, type ParseDiffOptions } from './parser'
+
 export default async (): Promise<Resources> => ({
   component: {
     DiffView,

plugins/diffview-resources/src/parser.ts

@@ -36,8 +36,19 @@ const diff2htmlConfig: Diff2HtmlConfig = {
 /**
  * @public
  */
-export function parseDiff (diff: Diff): DiffFile[] {
-  const files = parse(diff, diff2htmlConfig)
+export interface ParseDiffOptions {
+  /** Files with more changed lines come back without hunks (isTooBig); null: no limit. Default 10000 */
+  maxChanges?: number | null
+}
+
+/**
+ * @public
+ */
+export function parseDiff (diff: Diff, options: ParseDiffOptions = {}): DiffFile[] {
+  // diff2html reads an undefined diffMaxChanges as no limit
+  const diffMaxChanges =
+    options.maxChanges === null ? undefined : (options.maxChanges ?? diff2htmlConfig.diffMaxChanges)
+  const files = parse(diff, { ...diff2htmlConfig, diffMaxChanges })
   return files.map(mapFile)
 }
 

pods/front/src/__start.ts

@@ -32,6 +32,7 @@ startFront(metricsContext, {
   INTERCOM_APP_ID: process.env.INTERCOM_APP_ID ?? '',
   INTERCOM_API_URL: process.env.INTERCOM_API_URL ?? '',
   GITHUB_URL: process.env.GITHUB_URL ?? '',
+  GITLAB_URL: process.env.GITLAB_URL ?? '',
   LIVEKIT_WS: process.env.LIVEKIT_WS ?? '',
   LOVE_ENDPOINT: process.env.LOVE_ENDPOINT ?? '',
   SIGN_URL: process.env.SIGN_URL ?? '',

rush.json

@@ -2176,6 +2176,46 @@
       "projectFolder": "services/github/server-github-model",
       "shouldPublish": false
     },
+    {
+      "packageName": "@hcengineering/gitlab",
+      "projectFolder": "services/gitlab/gitlab",
+      "shouldPublish": false
+    },
+    {
+      "packageName": "@hcengineering/gitlab-assets",
+      "projectFolder": "services/gitlab/gitlab-assets",
+      "shouldPublish": false
+    },
+    {
+      "packageName": "@hcengineering/pod-gitlab",
+      "projectFolder": "services/gitlab/pod-gitlab",
+      "shouldPublish": false
+    },
+    {
+      "packageName": "@hcengineering/model-gitlab",
+      "projectFolder": "services/gitlab/model-gitlab",
+      "shouldPublish": false
+    },
+    {
+      "packageName": "@hcengineering/gitlab-resources",
+      "projectFolder": "services/gitlab/gitlab-resources",
+      "shouldPublish": false
+    },
+    {
+      "packageName": "@hcengineering/server-gitlab",
+      "projectFolder": "services/gitlab/server-gitlab",
+      "shouldPublish": false
+    },
+    {
+      "packageName": "@hcengineering/server-gitlab-resources",
+      "projectFolder": "services/gitlab/server-gitlab-resources",
+      "shouldPublish": false
+    },
+    {
+      "packageName": "@hcengineering/server-gitlab-model",
+      "projectFolder": "services/gitlab/server-gitlab-model",
+      "shouldPublish": false
+    },
     {
       "packageName": "@hcengineering/desktop-preferences",
       "projectFolder": "plugins/desktop-preferences",

server/account/src/__tests__/operations.test.ts

@@ -61,7 +61,8 @@ import {
   leaveWorkspace,
   checkJoin,
   mergeSpecifiedPersons,
-  canMergeSpecifiedPersons
+  canMergeSpecifiedPersons,
+  ensurePerson
 } from '../operations'
 import { accountPlugin } from '../plugin'
 
@@ -3831,3 +3832,30 @@ describe('merge specified persons', () => {
     })
   })
 })
+
+describe('ensurePerson', () => {
+  const mockCtx = {
+    error: jest.fn(),
+    info: jest.fn(),
+    warn: jest.fn()
+  } as unknown as MeasureContext
+
+  test('should let the gitlab service ensure a person without a workspace role check', async () => {
+    ;(decodeTokenVerbose as jest.Mock).mockReturnValue({ account: 'svc', extra: { service: 'gitlab' } })
+    const roleSpy = jest.spyOn(utils, 'getWorkspaceRole')
+    const mockDb = {
+      socialId: { findOne: jest.fn(async () => ({ _id: 'sid-1' as PersonId, personUuid: 'p-1' as PersonUuid })) }
+    } as unknown as AccountDB
+
+    const result = await ensurePerson(mockCtx, mockDb, null, 'test-token', {
+      socialType: SocialIdType.GITLAB,
+      socialValue: '42@gitlab.com',
+      firstName: 'Alice',
+      lastName: ''
+    })
+
+    expect(result).toEqual({ uuid: 'p-1', socialId: 'sid-1' })
+    expect(roleSpy).not.toHaveBeenCalled()
+    roleSpy.mockRestore()
+  })
+})