feat: update manifests for gateway api (#943)

* feat: update manifests for gateway api

* fix: namespace assignment of resources

* feat: rewrite cloudtty api for being more robust

* test: updated tests and added module for tty implementation

* fix: load kubeconfig

* fix: object deletion

* fix: await on return

* fix: js binding obscurities

* chore: clean up unused variable

* chore: removed unused code

* chore: removed unused files

* fix: namespaced roles on multiple team assignments

* Apply suggestion from @ferruhcihan

Co-authored-by: Ferruh <63190600+ferruhcihan@users.noreply.github.com>

---------

Co-authored-by: svcAPLBot <174728082+svcAPLBot@users.noreply.github.com>
Co-authored-by: Ferruh <63190600+ferruhcihan@users.noreply.github.com>

Author: 3 people

src/api/v1/cloudtty.ts

@@ -23,6 +23,7 @@ export const connectCloudtty = async (req: OpenApiRequestExt, res: Response): Pr
 export const deleteCloudtty = async (req: OpenApiRequestExt, res: Response): Promise<void> => {
   const sessionUser = req.user
   debug(`deleteCloudtty - ${sessionUser.email} - ${sessionUser.sub}`)
-  await req.otomi.deleteCloudtty(sessionUser)
+  const { teamId } = req.query as { teamId: string }
+  await req.otomi.deleteCloudtty(teamId, sessionUser)
   res.json({})
 }

src/api/v2/cloudtty.ts

@@ -23,6 +23,7 @@ export const connectAplCloudtty = async (req: OpenApiRequestExt, res: Response):
 export const deleteAplCloudtty = async (req: OpenApiRequestExt, res: Response): Promise<void> => {
   const sessionUser = req.user
   debug(`deleteCloudtty - ${sessionUser.email} - ${sessionUser.sub}`)
-  await req.otomi.deleteCloudtty(sessionUser)
+  const { teamId } = req.query as { teamId: string }
+  await req.otomi.deleteCloudtty(teamId, sessionUser)
   res.json({})
 }

src/k8s_operations.ts

@@ -1,65 +1,9 @@
-import {
-  CoreV1Api,
-  CustomObjectsApi,
-  KubeConfig,
-  KubernetesObject,
-  KubernetesObjectApi,
-  RbacAuthorizationV1Api,
-  VersionApi,
-} from '@kubernetes/client-node'
+import { CoreV1Api, CustomObjectsApi, KubeConfig, VersionApi } from '@kubernetes/client-node'
 import Debug from 'debug'
-import * as fs from 'fs'
-import * as yaml from 'js-yaml'
-import { promisify } from 'util'
 import { AplBuildResponse, AplServiceResponse, AplWorkloadResponse, SealedSecretManifestResponse } from './otomi-models'
 
 const debug = Debug('otomi:api:k8sOperations')
 
-/**
- * Replicate the functionality of `kubectl apply`.  That is, create the resources defined in the `specFile` if they do
- * not exist, patch them if they do exist.
- *
- * @param specPath File system path to a YAML Kubernetes spec.
- * @return Array of resources created
- */
-export async function apply(specPath: string): Promise<KubernetesObject[]> {
-  const kc = new KubeConfig()
-  kc.loadFromDefault()
-  const client = KubernetesObjectApi.makeApiClient(kc) as any
-  const fsReadFileP = promisify(fs.readFile)
-  const specString = await fsReadFileP(specPath, 'utf8')
-  const specs: any = yaml.loadAll(specString)
-  const validSpecs = specs.filter((s) => s && s.kind && s.metadata)
-  const created: KubernetesObject[] = []
-  for (const spec of validSpecs) {
-    // this is to convince the old version of TypeScript that metadata exists even though we already filtered specs
-    // without metadata out
-    spec.metadata = spec.metadata || {}
-    spec.metadata.annotations = spec.metadata.annotations || {}
-    delete spec.metadata.annotations['kubectl.kubernetes.io/last-applied-configuration']
-    spec.metadata.annotations['kubectl.kubernetes.io/last-applied-configuration'] = JSON.stringify(spec)
-    try {
-      // try to get the resource, if it does not exist an error will be thrown and we will end up in the catch
-      // block.
-      await client.read(spec)
-      // we got the resource, so it exists, so patch it
-      //
-      // Note that this could fail if the spec refers to a custom resource. For custom resources you may need
-      // to specify a different patch merge strategy in the content-type header.
-      //
-      // See: https://github.com/kubernetes/kubernetes/issues/97423
-      const response = await client.patch(spec)
-      created.push(response.body)
-    } catch (e) {
-      // we did not get the resource, so it does not exist, so create it
-      const response = await client.create(spec)
-      created.push(response.body)
-    }
-  }
-  debug(`Cloudtty is created!`)
-  return created
-}
-
 export async function watchPodUntilRunning(namespace: string, podName: string) {
   let isRunning = false
   const kc = new KubeConfig()
@@ -101,63 +45,6 @@ export async function checkPodExists(namespace: string, podName: string): Promis
   }
 }
 
-export async function k8sdelete({
-  sub,
-  isPlatformAdmin,
-  userTeams,
-}: {
-  sub: string
-  isPlatformAdmin: boolean
-  userTeams: string[]
-}): Promise<void> {
-  const kc = new KubeConfig()
-  kc.loadFromDefault()
-  const k8sApi = kc.makeApiClient(CoreV1Api)
-  const customObjectsApi = kc.makeApiClient(CustomObjectsApi)
-  const rbacAuthorizationV1Api = kc.makeApiClient(RbacAuthorizationV1Api)
-  const resourceName = sub
-  const namespace = 'team-admin'
-  try {
-    const apiVersion = 'v1beta1'
-    const apiGroupAuthz = 'security.istio.io'
-    const apiGroupVS = 'networking.istio.io'
-    const pluralAuth = 'authorizationpolicies'
-    const pluralVS = 'virtualservices'
-
-    await customObjectsApi.deleteNamespacedCustomObject({
-      group: apiGroupAuthz,
-      version: apiVersion,
-      namespace,
-      plural: pluralAuth,
-      name: `tty-${resourceName}`,
-    })
-
-    await k8sApi.deleteNamespacedServiceAccount({ name: `tty-${resourceName}`, namespace })
-    await k8sApi.deleteNamespacedPod({ name: `tty-${resourceName}`, namespace })
-    if (!isPlatformAdmin) {
-      for (const team of userTeams!) {
-        await rbacAuthorizationV1Api.deleteNamespacedRoleBinding({
-          name: `tty-${team}-${resourceName}-rolebinding`,
-          namespace: team,
-        })
-      }
-    } else {
-      await rbacAuthorizationV1Api.deleteClusterRoleBinding({ name: 'tty-admin-clusterrolebinding' })
-    }
-    await k8sApi.deleteNamespacedService({ name: `tty-${resourceName}`, namespace })
-
-    await customObjectsApi.deleteNamespacedCustomObject({
-      group: apiGroupVS,
-      version: apiVersion,
-      namespace,
-      plural: pluralVS,
-      name: `tty-${resourceName}`,
-    })
-  } catch (error) {
-    debug(`Failed to delete resources for ${resourceName} in namespace ${namespace}.`)
-  }
-}
-
 export async function getKubernetesVersion() {
   if (process.env.NODE_ENV === 'development') return 'x.x.x'
 

src/otomi-stack.test.ts

@@ -12,6 +12,14 @@ import { loadSpec } from './app'
 import { PublicUrlExists, ValidationError } from './error'
 import { Git } from './git'
 
+jest.mock('./tty', () => ({
+  __esModule: true,
+  default: jest.fn().mockImplementation(() => ({
+    createTty: jest.fn(),
+    deleteTty: jest.fn(),
+  })),
+}))
+
 jest.mock('src/middleware', () => ({
   ...jest.requireActual('src/middleware'),
   getSessionStack: jest.fn(),

src/otomi-stack.ts

@@ -3,8 +3,7 @@ import Debug from 'debug'
 
 import { getRegions, ObjectStorageKeyRegions, Region, ResourcePage } from '@linode/api-v4'
 import { existsSync, rmSync } from 'fs'
-import { pathExists, unlink } from 'fs-extra'
-import { readdir, readFile, writeFile } from 'fs/promises'
+import { readFile } from 'fs/promises'
 import { generate as generatePassword } from 'generate-password'
 import { cloneDeep, filter, get, isEmpty, map, merge, omit, pick, set, unset } from 'lodash'
 import { getAppList, getAppSchema, getSecretPaths } from 'src/app'
@@ -117,13 +116,11 @@ import { getAIModels } from './ai/aiModelHandler'
 import { DatabaseCR } from './ai/DatabaseCR'
 import { getResourceFilePath, getSecretFilePath } from './fileStore/file-map'
 import {
-  apply,
   checkPodExists,
   getCloudttyActiveTime,
   getKubernetesVersion,
   getSecretValues,
   getTeamSecretsFromK8s,
-  k8sdelete,
   watchPodUntilRunning,
 } from './k8s_operations'
 import {
@@ -146,6 +143,7 @@ import {
   sparseCloneChart,
   validateGitUrl,
 } from './utils/workloadUtils'
+import CloudTty from './tty'
 
 interface ExcludedApp extends App {
   managed: boolean
@@ -208,12 +206,20 @@ export default class OtomiStack {
   isLoaded = false
   git: Git
   fileStore: FileStore
+  private cloudTty: CloudTty
 
   constructor(editor?: string, sessionId?: string) {
     this.editor = editor
     this.sessionId = sessionId ?? 'main'
   }
 
+  getCloudTty() {
+    if (!this.cloudTty) {
+      this.cloudTty = new CloudTty()
+    }
+    return this.cloudTty
+  }
+
   getAppList() {
     let apps = getAppList()
     apps = apps.filter((item) => item !== 'ingress-nginx')
@@ -1524,11 +1530,12 @@ export default class OtomiStack {
   }
 
   async connectCloudtty(teamId: string, sessionUser: SessionUser): Promise<Cloudtty> {
+    const isAdmin = sessionUser.isPlatformAdmin
+    const targetNamespace = isAdmin ? 'team-admin' : `team-${teamId}`
     if (!sessionUser.sub) {
       debug('No user sub found, cannot connect to shell.')
       throw new OtomiError(500, 'No user sub found, cannot connect to shell.')
     }
-    const userTeams = sessionUser.teams.map((teamName) => `team-${teamName}`)
     const variables = {
       FQDN: '',
       SUB: sessionUser.sub,
@@ -1545,62 +1552,20 @@ export default class OtomiStack {
     }
 
     // if cloudtty shell does not exists then check if the pod is running and return it
-    if (await checkPodExists('team-admin', `tty-${sessionUser.sub}`)) {
+    if (await checkPodExists(targetNamespace, `tty-${sessionUser.sub}`)) {
       return { iFrameUrl: `https://tty.${variables.FQDN}/${sessionUser.sub}` }
     }
 
-    if (await pathExists('/tmp/ttyd.yaml')) await unlink('/tmp/ttyd.yaml')
-
-    //if user is admin then read the manifests from ./dist/src/ttyManifests/adminTtyManifests
-    const files = sessionUser.isPlatformAdmin
-      ? await readdir('./dist/src/ttyManifests/adminTtyManifests', 'utf-8')
-      : await readdir('./dist/src/ttyManifests', 'utf-8')
-    const filteredFiles = files.filter((file) => file.startsWith('tty'))
-    const variableKeys = Object.keys(variables)
-
-    const podContentAddTargetTeam = (fileContent) => {
-      const regex = new RegExp(`\\$TARGET_TEAM`, 'g')
-      return fileContent.replace(regex, teamId)
-    }
-
-    // iterates over the rolebinding file and replace the $TARGET_TEAM with the team name for teams
-    const rolebindingContentsForUsers = (fileContent) => {
-      const rolebindingArray: string[] = []
-      userTeams?.forEach((team: string) => {
-        const regex = new RegExp(`\\$TARGET_TEAM`, 'g')
-        const rolebindingForTeam: string = fileContent.replace(regex, team)
-        rolebindingArray.push(rolebindingForTeam)
-      })
-      return rolebindingArray.join('\n')
-    }
-
-    const fileContents = await Promise.all(
-      filteredFiles.map(async (file) => {
-        let fileContent = sessionUser.isPlatformAdmin
-          ? await readFile(`./dist/src/ttyManifests/adminTtyManifests/${file}`, 'utf-8')
-          : await readFile(`./dist/src/ttyManifests/${file}`, 'utf-8')
-        variableKeys.forEach((key) => {
-          const regex = new RegExp(`\\$${key}`, 'g')
-          fileContent = fileContent.replace(regex, variables[key] as string)
-        })
-        if (file === 'tty_02_Pod.yaml') fileContent = podContentAddTargetTeam(fileContent)
-        if (!sessionUser.isPlatformAdmin && file === 'tty_03_Rolebinding.yaml') {
-          fileContent = rolebindingContentsForUsers(fileContent)
-        }
-        return fileContent
-      }),
-    )
-    await writeFile('/tmp/ttyd.yaml', fileContents, 'utf-8')
-    await apply('/tmp/ttyd.yaml')
-    await watchPodUntilRunning('team-admin', `tty-${sessionUser.sub}`)
+    await this.getCloudTty().createTty(teamId, sessionUser, variables.FQDN)
+    await watchPodUntilRunning(targetNamespace, `tty-${sessionUser.sub}`)
 
     // check the pod every 30 minutes and terminate it after 2 hours of inactivity
     const ISACTIVE_INTERVAL = 30 * 60 * 1000
     const TERMINATE_TIMEOUT = 2 * 60 * 60 * 1000
     const intervalId = setInterval(() => {
-      getCloudttyActiveTime('team-admin', `tty-${sessionUser.sub}`).then((activeTime: number) => {
+      getCloudttyActiveTime(targetNamespace, `tty-${sessionUser.sub}`).then(async (activeTime: number) => {
         if (activeTime > TERMINATE_TIMEOUT) {
-          this.deleteCloudtty(sessionUser)
+          await this.getCloudTty().deleteTty(teamId, sessionUser)
           clearInterval(intervalId)
           debug(`Cloudtty terminated after ${TERMINATE_TIMEOUT / (60 * 60 * 1000)} hours of inactivity`)
         }
@@ -1610,16 +1575,8 @@ export default class OtomiStack {
     return { iFrameUrl: `https://tty.${variables.FQDN}/${sessionUser.sub}` }
   }
 
-  async deleteCloudtty(sessionUser: SessionUser): Promise<void> {
-    const { sub, isPlatformAdmin, teams } = sessionUser as { sub: string; isPlatformAdmin: boolean; teams: string[] }
-    const userTeams = teams.map((teamName) => `team-${teamName}`)
-    try {
-      if (await checkPodExists('team-admin', `tty-${sessionUser.sub}`)) {
-        await k8sdelete({ sub, isPlatformAdmin, userTeams })
-      }
-    } catch (error) {
-      debug('Failed to delete cloudtty')
-    }
+  async deleteCloudtty(teamId: string, sessionUser: SessionUser): Promise<void> {
+    await this.getCloudTty().deleteTty(teamId, sessionUser)
   }
 
   private async fetchCatalog(