fix(server-nestjs): realign e2e tests with DSO orchestration docs

apps/server-nestjs/src/config/gitlab.config.spec.ts

@@ -17,8 +17,8 @@ describe('gitlabConfig', () => {
       url: 'https://gitlab.internal',
       internalUrl: 'https://gitlab.internal:8080',
       secretExposeInternalUrl: true,
-      mirrorTokenExpirationDays: 180,
-      mirrorTokenRotationThresholdDays: 90,
+      mirrorTokenExpirationDays: 365,
+      mirrorTokenRotationThresholdDays: 330,
       projectRootDir: 'forge-test/projects',
     })
   })

apps/server-nestjs/src/config/gitlab.config.ts

@@ -6,8 +6,8 @@ const gitlabFeatureSchema = z.object({
   GITLAB_TOKEN: z.string().min(1),
   GITLAB_URL: z.string().url(),
   GITLAB_INTERNAL_URL: z.string().url().optional(),
-  GITLAB_MIRROR_TOKEN_EXPIRATION_DAYS: z.coerce.number().int().positive().default(180),
-  GITLAB_MIRROR_TOKEN_ROTATION_THRESHOLD_DAYS: z.coerce.number().int().positive().default(90),
+  GITLAB_MIRROR_TOKEN_EXPIRATION_DAYS: z.coerce.number().int().positive().default(365),
+  GITLAB_MIRROR_TOKEN_ROTATION_THRESHOLD_DAYS: z.coerce.number().int().positive().default(330),
   GITLAB__SECRET_EXPOSE_INTERNAL_URL: truthySchema.default('false').transform(v => v === 'true' || v === '1'),
   PROJECTS_ROOT_DIR: z.string().min(1),
 }).transform(raw => ({

apps/server-nestjs/src/modules/gitlab/gitlab-client.service.ts

@@ -241,6 +241,7 @@ export class GitlabClientService {
         path: repoName,
         namespaceId: parentGroup.id,
         defaultBranch: 'main',
+        ciConfigPath: '.gitlab-ci-dso.yml',
       })
       this.logger.log(`Created a GitLab project repository (path=${fullPath}, repoId=${created.id})`)
       return created
@@ -375,6 +376,9 @@ export class GitlabClientService {
     this.logger.log(`Creating a GitLab user (email=${user.email}, username=${user.username})`)
     return await this.client.Users.create({
       ...user,
+      canCreateGroup: false,
+      forceRandomPassword: true,
+      projectsLimit: 0,
       skipConfirmation: true,
     }) as UserSchema
   }
@@ -465,8 +469,8 @@ export class GitlabClientService {
     const group = await this.getProjectGroup(projectSlug)
     if (!group) throw new Error('Unable to retrieve gitlab project group')
     return find(
-      this.offsetPaginate<{ name: string }>(
-        opts => this.client.GroupAccessTokens.all(group.id, opts) as unknown as Promise<{ data: { name: string }[], paginationInfo: OffsetPagination }>,
+      this.offsetPaginate<{ name: string, id: number }>(
+        opts => this.client.GroupAccessTokens.all(group.id, opts) as unknown as Promise<{ data: { name: string, id: number }[], paginationInfo: OffsetPagination }>,
       ),
       token => token.name === `${projectSlug}-bot`,
     )
@@ -487,6 +491,24 @@ export class GitlabClientService {
     return this.createProjectToken(projectSlug, tokenName, ['write_repository', 'read_repository', 'read_api'])
   }
 
+  async revokeProjectToken(projectSlug: string, tokenId: number): Promise<void> {
+    const group = await this.getProjectGroup(projectSlug)
+    if (!group) throw new Error('Unable to retrieve gitlab project group')
+    this.logger.log(`Revoking a GitLab group access token (projectSlug=${projectSlug}, tokenId=${tokenId})`)
+    await this.client.GroupAccessTokens.revoke(group.id, tokenId)
+  }
+
+  async validateProjectToken(group: CondensedGroupSchemaWith<'id'>, token: string): Promise<boolean> {
+    try {
+      const res = await fetch(`${this.config.internalUrl ?? this.config.url}/api/v4/groups/${group.id}`, {
+        headers: { 'PRIVATE-TOKEN': token },
+      })
+      return res.ok
+    } catch {
+      return false
+    }
+  }
+
   async getOrCreateMirrorPipelineTriggerToken(projectSlug: string): Promise<PipelineTriggerTokenSchema> {
     const mirrorRepo = await this.upsertProjectMirrorRepo(projectSlug)
     this.logger.verbose(`Resolving a GitLab pipeline trigger token (projectSlug=${projectSlug}, repoId=${mirrorRepo.id})`)

apps/server-nestjs/src/modules/gitlab/gitlab.service.ts

@@ -1,7 +1,6 @@
 import type { CondensedGroupSchema, MemberSchema, ProjectSchema } from '@gitbeaker/core'
 import type { ConfigType } from '@nestjs/config'
 import type { RequiredPluginResult } from '../plugin/plugin.utils'
-import type { VaultSecret } from '../vault/vault-client.service'
 import type { ProjectWithDetails } from './gitlab-datastore.service'
 import { specificallyEnabled } from '@cpn-console/hooks'
 import { AccessLevel } from '@gitbeaker/core'
@@ -32,7 +31,6 @@ import {
 } from './gitlab.constants'
 import {
   adminRoleFlag,
-  daysAgoFromNow,
   generateAccessLevelMapping,
   generateAdminRoleMapping,
   generateName,
@@ -84,8 +82,12 @@ export class GitlabService {
     const span = trace.getActiveSpan()
     span?.setAttribute('project.slug', project.slug)
     this.logger.log(`Handling a project delete event for ${project.slug}`)
-    await this.ensureProjectGroup(project)
-    this.logger.log(`GitLab sync completed for project ${project.slug}`)
+    const group = await this.gitlab.getGroupByPath(`${this.gitlabConfig.projectRootDir}/${project.slug}`)
+    if (group) {
+      await this.gitlab.deleteGroup(group)
+      this.logger.log(`Deleted GitLab project group (${project.slug})`)
+    }
+    this.logger.log(`GitLab cleanup completed for project ${project.slug}`)
   }
 
   // @Cron(CronExpression.EVERY_HOUR)
@@ -484,14 +486,19 @@ export class GitlabService {
   private async getOrRotateMirrorCreds(projectSlug: string) {
     const span = trace.getActiveSpan()
     span?.setAttribute('project.slug', projectSlug)
-    const vaultSecret = await this.vault.readTechnReadOnlyCreds(projectSlug)
-    if (!vaultSecret) return this.createMirrorAccessToken(projectSlug)
-
-    const isExpiring = this.isMirrorCredsExpiring(vaultSecret)
-    span?.setAttribute('mirror.creds.expiring', isExpiring)
-    if (!isExpiring) {
-      span?.setAttribute('mirror.creds.rotated', false)
-      return vaultSecret.data as { MIRROR_USER: string, MIRROR_TOKEN: string }
+    const currentToken = await this.gitlab.getProjectToken(projectSlug)
+    if (currentToken) {
+      const vaultSecret = await this.vault.readTechnReadOnlyCreds(projectSlug)
+      if (vaultSecret?.data?.MIRROR_TOKEN) {
+        const group = await this.gitlab.getOrCreateProjectSubGroup(projectSlug)
+        const isValid = await this.gitlab.validateProjectToken(group, vaultSecret.data.MIRROR_TOKEN)
+        if (isValid) {
+          span?.setAttribute('mirror.creds.rotated', false)
+          return vaultSecret.data
+        }
+        this.logger.warn(`Mirror token invalid, revoking (projectSlug=${projectSlug}, tokenId=${currentToken.id})`)
+        await this.gitlab.revokeProjectToken(projectSlug, currentToken.id).catch(() => {})
+      }
     }
     return this.createMirrorAccessToken(projectSlug)
   }
@@ -511,12 +518,6 @@ export class GitlabService {
     return creds
   }
 
-  private isMirrorCredsExpiring(vaultSecret: VaultSecret): boolean {
-    if (!vaultSecret?.metadata?.created_time) return false
-    const createdTime = new Date(vaultSecret.metadata.created_time)
-    return daysAgoFromNow(createdTime) > this.gitlabConfig.mirrorTokenRotationThresholdDays
-  }
-
   private getExternalRepoHost(externalRepoUrl: string | null | undefined): string | undefined {
     if (!externalRepoUrl) return undefined
     try {

apps/server-nestjs/src/modules/keycloak/keycloak.service.ts

@@ -49,7 +49,11 @@ export class KeycloakService {
     const span = trace.getActiveSpan()
     span?.setAttribute('project.slug', project.slug)
     this.logger.log(`Handling a project delete event for ${project.slug}`)
-    await this.purgeOrphanGroups([project])
+    const projectGroup = await this.keycloak.getGroupByPath(`/${project.slug}`)
+    if (projectGroup?.id) {
+      await this.keycloak.deleteGroup(projectGroup.id)
+      this.logger.log(`Deleted Keycloak project group (${project.slug})`)
+    }
     this.logger.log(`Keycloak cleanup completed for project ${project.slug}`)
   }
 

apps/server-nestjs/src/modules/sonarqube/sonarqube-datastore.service.ts

@@ -18,6 +18,11 @@ export const projectSelect = {
       value: true,
     },
   },
+  owner: {
+    select: {
+      email: true,
+    },
+  },
 } satisfies Prisma.ProjectSelect
 
 export type ProjectWithDetails = Prisma.ProjectGetPayload<{

apps/server-nestjs/src/modules/sonarqube/sonarqube-testing.utils.ts

@@ -30,6 +30,7 @@ export function makeProjectWithDetails(overrides: Partial<ProjectWithDetails> =
     slug: faker.internet.domainWord(),
     repositories: [],
     plugins: [],
+    owner: { email: faker.internet.email() },
     ...overrides,
   } satisfies ProjectWithDetails
 }

apps/server-nestjs/src/modules/sonarqube/sonarqube.service.ts

@@ -143,7 +143,7 @@ export class SonarqubeService implements OnModuleInit {
     span?.setAttribute('project.slug', project.slug)
     const rolePaths = await this.getProjectRoleGroupPaths(project)
     await Promise.all([
-      this.ensureUser(project.slug, project.slug),
+      this.ensureUser(project),
       this.ensureProjectSonarGroups(rolePaths),
       this.ensureProjectRepositories(project, rolePaths),
     ])
@@ -197,28 +197,31 @@ export class SonarqubeService implements OnModuleInit {
   }
 
   @StartActiveSpan()
-  private async ensureUser(username: string, projectSlug: string): Promise<void> {
-    const existingSecret = await this.vault.readSonarqubeUser(projectSlug)
-    const user = await this.findUser(username)
+  private async ensureUser(project: ProjectWithDetails): Promise<void> {
+    const { slug, owner } = project
+    const existingSecret = await this.vault.readSonarqubeUser(slug)
+    const user = await this.findUser(slug)
     let newSecret: SonarqubeUserSecret | undefined
 
+    const userEmail = owner.email
+
     if (!user) {
-      this.logger.log(`Creating SonarQube user (login=${username})`)
+      this.logger.log(`Creating SonarQube user (login=${slug}, email=${userEmail})`)
       const password = generateRandomPassword(30)
-      await this.client.createUser({ email: `${projectSlug}@${projectSlug}`, local: 'true', login: username, name: username, password })
-      const token = await this.rotateToken(username)
-      newSecret = { SONAR_USERNAME: username, SONAR_PASSWORD: password, SONAR_TOKEN: token }
+      await this.client.createUser({ email: userEmail, local: 'true', login: slug, name: slug, password })
+      const token = await this.rotateToken(slug)
+      newSecret = { SONAR_USERNAME: slug, SONAR_PASSWORD: password, SONAR_TOKEN: token }
     } else if (existingSecret) {
-      this.logger.verbose(`SonarQube user already exists with vault credentials (login=${username})`)
+      this.logger.verbose(`SonarQube user already exists with vault credentials (login=${slug})`)
     } else {
-      this.logger.warn(`SonarQube user exists but vault secret is missing, rotating token (login=${username})`)
-      const token = await this.rotateToken(username)
-      newSecret = { SONAR_USERNAME: username, SONAR_PASSWORD: 'not initialized', SONAR_TOKEN: token }
+      this.logger.warn(`SonarQube user exists but vault secret is missing, rotating token (login=${slug})`)
+      const token = await this.rotateToken(slug)
+      newSecret = { SONAR_USERNAME: slug, SONAR_PASSWORD: 'not initialized', SONAR_TOKEN: token }
     }
 
     if (newSecret) {
-      await this.vault.writeSonarqubeUser(projectSlug, newSecret)
-      this.logger.log(`Stored SonarQube credentials in vault (slug=${projectSlug})`)
+      await this.vault.writeSonarqubeUser(slug, newSecret)
+      this.logger.log(`Stored SonarQube credentials in vault (slug=${slug})`)
     }
   }
 

apps/server-nestjs/src/modules/vault/vault-client.service.ts

@@ -76,6 +76,11 @@ export interface SonarqubeUserSecret {
   SONAR_TOKEN: string
 }
 
+export interface MirrorUserSecret {
+  MIRROR_USER: string
+  MIRROR_TOKEN: string
+}
+
 export interface VaultMetadata {
   created_time: string
   custom_metadata: Record<string, any> | null
@@ -204,7 +209,7 @@ export class VaultClientService {
   }
 
   @StartActiveSpan()
-  async readTechnReadOnlyCreds(projectSlug: string): Promise<VaultSecret | null> {
+  async readTechnReadOnlyCreds(projectSlug: string): Promise<VaultSecret<MirrorUserSecret> | null> {
     const vaultPath = generateTechReadOnlyCredPath(this.baseConfig.projectsRootDir, projectSlug)
     const span = trace.getActiveSpan()
     span?.setAttribute('project.slug', projectSlug)

apps/server-nestjs/test/argocd.e2e-spec.ts

@@ -25,9 +25,9 @@ const canRunArgoCDE2E
 
 const describeWithArgoCD = describe.runIf(canRunArgoCDE2E)
 
-describeWithArgoCD('ArgoCDController (e2e)', {}, () => {
+describeWithArgoCD('ArgoCDService (e2e)', () => {
   let moduleRef: TestingModule
-  let argocdController: ArgoCDService
+  let argocdService: ArgoCDService
   let gitlab: GitlabClientService
   let gitlabClient: Gitlab
   let vault: VaultClientService
@@ -60,7 +60,7 @@ describeWithArgoCD('ArgoCDController (e2e)', {}, () => {
 
     await moduleRef.init()
 
-    argocdController = moduleRef.get<ArgoCDService>(ArgoCDService)
+    argocdService = moduleRef.get<ArgoCDService>(ArgoCDService)
     gitlab = moduleRef.get<GitlabClientService>(GitlabClientService)
     gitlabClient = moduleRef.get<Gitlab>(GITLAB_REST_CLIENT)
     vault = moduleRef.get<VaultClientService>(VaultClientService)
@@ -212,7 +212,7 @@ describeWithArgoCD('ArgoCDController (e2e)', {}, () => {
 
     vaultProjectValuesPath = `${config.projectsRootDir}/${testProjectId}`
     await vault.write({ e2e: true }, vaultProjectValuesPath)
-  })
+  }, 144000)
 
   afterAll(async () => {
     if (vaultProjectValuesPath) {
@@ -252,7 +252,7 @@ describeWithArgoCD('ArgoCDController (e2e)', {}, () => {
     const staleAction = await gitlab.generateCreateOrUpdateAction(infraProject, 'main', staleFilePath, 'stale: true\n')
     await gitlab.maybeCreateCommit(infraProject, 'ci: :robot_face: Seed stale values', staleAction ? [staleAction] : [])
 
-    await argocdController.handleUpsert(project)
+    await argocdService.handleUpsert(project)
 
     const expectedFilePath = `${project.name}/${clusterLabel}/${envDevName}/values.yaml`
     const file = await gitlabClient.RepositoryFiles.show(infraRepoId, expectedFilePath, 'main')
@@ -294,7 +294,7 @@ describeWithArgoCD('ArgoCDController (e2e)', {}, () => {
       select: projectSelect,
     })
 
-    await argocdController.handleUpsert(after)
+    await argocdService.handleUpsert(after)
 
     const updatedDev = await gitlabClient.RepositoryFiles.show(infraRepoId, devFilePath, 'main')
     const devRaw = Buffer.from(updatedDev.content, 'base64').toString('utf8')

apps/server-nestjs/test/gitlab.e2e-spec.ts

@@ -25,10 +25,10 @@ const canRunGitlabE2E
 
 const describeWithGitLab = describe.runIf(canRunGitlabE2E)
 
-describeWithGitLab('GitlabController (e2e)', {}, () => {
+describeWithGitLab('GitlabService (e2e)', () => {
   let moduleRef: TestingModule
-  let gitlabController: GitlabService
-  let gitlabService: GitlabClientService
+  let gitlabService: GitlabService
+  let gitlabClientService: GitlabClientService
   let gitlabClient: Gitlab
   let vaultService: VaultClientService
   let prisma: PrismaService
@@ -46,8 +46,8 @@ describeWithGitLab('GitlabController (e2e)', {}, () => {
 
     await moduleRef.init()
 
-    gitlabController = moduleRef.get<GitlabService>(GitlabService)
-    gitlabService = moduleRef.get<GitlabClientService>(GitlabClientService)
+    gitlabService = moduleRef.get<GitlabService>(GitlabService)
+    gitlabClientService = moduleRef.get<GitlabClientService>(GitlabClientService)
     gitlabClient = moduleRef.get<Gitlab>(GITLAB_REST_CLIENT)
     vaultService = moduleRef.get<VaultClientService>(VaultClientService)
     prisma = moduleRef.get<PrismaService>(PrismaService)
@@ -109,9 +109,9 @@ describeWithGitLab('GitlabController (e2e)', {}, () => {
     // Clean GitLab group
     if (testProjectSlug && config.projectsRootDir) {
       const fullPath = `${config.projectsRootDir}/${testProjectSlug}`
-      const group = await gitlabService.getGroupByPath(fullPath)
+      const group = await gitlabClientService.getGroupByPath(fullPath)
       if (group) {
-        await gitlabService.deleteGroup(group).catch(() => {})
+        await gitlabClientService.deleteGroup(group).catch(() => {})
       }
     }
 
@@ -146,7 +146,7 @@ describeWithGitLab('GitlabController (e2e)', {}, () => {
     })
 
     // Act
-    await gitlabController.handleUpsert(project)
+    await gitlabService.handleUpsert(project)
 
     // Assert
     const groupPath = `${config.projectsRootDir}/${testProjectSlug}`
@@ -155,11 +155,11 @@ describeWithGitLab('GitlabController (e2e)', {}, () => {
       name: z.string(),
       full_path: z.string(),
       web_url: z.string(),
-    }).parse(await gitlabService.getGroupByPath(groupPath))
+    }).parse(await gitlabClientService.getGroupByPath(groupPath))
     expect(group.full_path).toBe(groupPath)
 
     // Check membership
-    const members = await gitlabService.getGroupMembers(group)
+    const members = await gitlabClientService.getGroupMembers(group)
     const isMember = members.some(m => m.id === ownerUser.id)
     expect(isMember).toBe(true)
 
@@ -219,18 +219,31 @@ describeWithGitLab('GitlabController (e2e)', {}, () => {
         select: projectSelect,
       })
 
-      await gitlabController.handleUpsert(project)
+      await gitlabService.handleUpsert(project)
 
       const groupPath = `${config.projectsRootDir}/${testProjectSlug}`
       const group = z.object({
         id: z.number(),
         name: z.string(),
         web_url: z.string(),
-      }).parse(await gitlabService.getGroupByPath(groupPath))
+      }).parse(await gitlabClientService.getGroupByPath(groupPath))
 
-      const members = await gitlabService.getGroupMembers(group)
+      const members = await gitlabClientService.getGroupMembers(group)
       const isNewMemberPresent = members.some(m => m.id === newUserGitlabId)
       expect(isNewMemberPresent).toBe(true)
     }, 72000)
   })
+
+  it('should remove project group from GitLab on delete', async () => {
+    const project = await prisma.project.findUniqueOrThrow({
+      where: { id: testProjectId },
+      select: projectSelect,
+    })
+
+    await gitlabService.handleDelete(project)
+
+    const groupPath = `${config.projectsRootDir}/${testProjectSlug}`
+    const group = await gitlabClientService.getGroupByPath(groupPath)
+    expect(group).toBeUndefined()
+  }, 72000)
 })