Migrate from http/1.1 and websockets to http/2 (gardener#972)

* migrate to http/2

* fixed bugs in SessionPool

* PR feedback IV

* handle stream errors

* add kind for list objects and reduce log severity for tolerable cases

* support for vscode-jest-runner

* improved jest script

* Separate session pool for informers

.dockerignore

@@ -15,23 +15,35 @@
 
 # include workspace packages/logger
 !packages/logger/package.json
+!packages/logger/jest.setup.js
 !packages/logger/lib
 !packages/logger/__tests__
+!packages/logger/__fixtures__
+!packages/logger/__mocks__
 
 # include workspace packages/request
 !packages/request/package.json
+!packages/request/jest.setup.js
 !packages/request/lib
 !packages/request/__tests__
+!packages/request/__fixtures__
+!packages/request/__mocks__
 
 # include workspace packages/kube-config
 !packages/kube-config/package.json
+!packages/kube-config/jest.setup.js
 !packages/kube-config/lib
 !packages/kube-config/__tests__
+!packages/kube-config/__fixtures__
+!packages/kube-config/__mocks__
 
 # include workspace packages/kube-client
 !packages/kube-client/package.json
+!packages/kube-client/jest.setup.js
 !packages/kube-client/lib
 !packages/kube-client/__tests__
+!packages/kube-client/__fixtures__
+!packages/kube-client/__mocks__
 
 # include workspace backend
 !backend/package.json
@@ -61,6 +73,12 @@
 !charts/gardener-dashboard
 !charts/identity
 !charts/__tests__
+!charts/__fixtures__
+!charts/__mocks__
+
+# include workspace packages/test-utils
+!packages/test-utils/package.json
+!packages/test-utils/lib
 
 # include metadata files
 !VERSION

.vscode/extensions.json

@@ -1,6 +1,7 @@
 {
   "recommendations": [
     "arcanis.vscode-zipfs",
-    "dbaeumer.vscode-eslint"
+    "dbaeumer.vscode-eslint",
+    "firsttris.vscode-jest-runner"
   ]
 }

.vscode/settings.json

@@ -32,7 +32,21 @@
   },
   "eslint.nodePath": ".yarn/sdks",
   "files.exclude": {
-    "**/.nyc_output": true,
-    "**/node_modules": true
-  }
+    ".pnp.*": true,
+    ".yarn": true,
+    "yarn.lock": true,
+    ".vscode": true,
+    ".pre-commit-config.yaml": true,
+    ".secrets.baseline": true,
+    ".docforge": true,
+    ".github": true,
+    ".reuse": true,
+    "logo": true,
+    "LICENSES": true,
+    "**/node_modules": true,
+    "**/coverage": true,
+    "**/dist": true
+  },
+  "jestrunner.enableYarnPnpSupport": true,
+  "jestrunner.jestCommand": "scripts/jest"
 }

.yarnrc.yml

@@ -43,4 +43,8 @@ plugins:
   - path: .yarn/plugins/@yarnpkg/plugin-version.cjs
     spec: "@yarnpkg/plugin-version"
 
-yarnPath: .yarn/releases/yarn-2.3.3.cjs
+yarnPath: .yarn/releases/yarn-2.4.1.cjs
+
+logFilters:
+  - code: "YN0005"
+    level: "discard"

Dockerfile

@@ -3,41 +3,51 @@
 # SPDX-License-Identifier: Apache-2.0
 
 #### Builder ####
-FROM  eu.gcr.io/gardener-project/3rd/node:14-alpine3.12 as builder
+FROM eu.gcr.io/gardener-project/3rd/node:15-alpine3.13 as builder
 
 WORKDIR /usr/src/app
 
 COPY . .
 
 # validate zero-installs project and disable network
-RUN yarn config set enableNetwork false \
-    && yarn install --immutable --immutable-cache
+RUN yarn config set enableNetwork false
+RUN yarn install --immutable --immutable-cache
 
 # run lint in all workspaces
-RUN yarn workspaces foreach --all run lint
+RUN yarn workspace @gardener-dashboard/logger      run lint
+RUN yarn workspace @gardener-dashboard/request     run lint
+RUN yarn workspace @gardener-dashboard/kube-config run lint
+RUN yarn workspace @gardener-dashboard/kube-client run lint
+RUN yarn workspace @gardener-dashboard/backend     run lint
+RUN yarn workspace @gardener-dashboard/frontend    run lint
 
 # run test in all workspaces
-RUN yarn workspaces foreach --all run test-coverage
+RUN yarn workspace @gardener-dashboard/logger      run test-coverage
+RUN yarn workspace @gardener-dashboard/request     run test-coverage
+RUN yarn workspace @gardener-dashboard/kube-config run test-coverage
+RUN yarn workspace @gardener-dashboard/kube-client run test-coverage
+RUN yarn workspace @gardener-dashboard/backend     run test-coverage
+RUN yarn workspace @gardener-dashboard/frontend    run test-coverage
 
 # bump backend and frontend version
-RUN yarn workspaces foreach --include "*/(frontend|backend)" version "$(cat VERSION)"
+RUN yarn workspace @gardener-dashboard/backend     version "$(cat VERSION)"
+RUN yarn workspace @gardener-dashboard/frontend    version "$(cat VERSION)"
 
 # run backend production install
-RUN yarn workspace @gardener-dashboard/backend prod-install --pack /usr/src/build
+RUN yarn workspace @gardener-dashboard/backend     prod-install --pack /usr/src/build
 
 # run frontend build
-RUN yarn workspace @gardener-dashboard/frontend run build
+RUN yarn workspace @gardener-dashboard/frontend    run build
 
 # copy files to production directory
 RUN cp -r frontend/dist /usr/src/build/public \
     && find /usr/src/build/.yarn -mindepth 1 -name cache -prune -o -exec rm -rf {} +
 
 #### Release ####
-FROM eu.gcr.io/gardener-project/3rd/alpine:3.12 as release
+FROM eu.gcr.io/gardener-project/3rd/alpine:3.13 as release
 
-RUN addgroup -g 1000 node \
-    && adduser -u 1000 -G node -s /bin/sh -D node \
-    && apk add --no-cache tini libstdc++
+RUN addgroup -g 1000 node && adduser -u 1000 -G node -s /bin/sh -D node
+RUN apk add --no-cache tini libstdc++
 
 WORKDIR /usr/src/app
 

backend/__fixtures__/projects.js

@@ -260,31 +260,32 @@ const mocks = {
       return Promise.resolve(item)
     }
   },
-  watch ({ phase = 'Ready', milliseconds } = {}) {
+  watch ({ phase = 'Ready', end = false, milliseconds } = {}) {
     return headers => {
       const [pathname] = split(headers[':path'], '?')
       const matchResult = matchList(pathname)
       if (matchResult === false) {
         return Promise.reject(createError(503))
       }
       const fieldSelector = parseFieldSelector(headers)
-      const stream = new PassThrough()
+      const stream = new PassThrough({ objectMode: true })
       process.nextTick(async () => {
         const items = filter(projects.list(), fieldSelector)
         for (const item of items) {
-          const chunk = JSON.stringify({ type: 'ADDED', object: item }) + '\n'
-          stream.write(chunk)
+          stream.write({ type: 'ADDED', object: item })
         }
         await delay(milliseconds)
         const initialItems = filter(items, ['status.phase', 'Initial'])
-        for (const item of initialItems) {
+        for (const oldItem of initialItems) {
+          const item = cloneDeep(oldItem)
           const resourceVersion = get(item, 'metadata.resourceVersion', '42')
           set(item, 'status.phase', phase)
           set(item, 'metadata.resourceVersion', (+resourceVersion + 1).toString())
-          const chunk = JSON.stringify({ type: 'MODIFIED', object: item }) + '\n'
-          stream.write(chunk)
+          stream.write({ type: 'MODIFIED', object: item })
+        }
+        if (end === true) {
+          stream.end()
         }
-        stream.end()
       })
       return stream
     }

backend/__fixtures__/serviceaccounts.js

@@ -91,7 +91,7 @@ const mocks = {
       return Promise.resolve(item)
     }
   },
-  watch () {
+  watch ({ end = false } = {}) {
     return headers => {
       const [pathname] = split(headers[':path'], '?')
       const matchResult = matchList(pathname)
@@ -101,17 +101,18 @@ const mocks = {
       const { params: { namespace } = {} } = matchResult
       const fieldSelector = parseFieldSelector(headers)
       const items = filter(serviceaccounts.list(namespace), fieldSelector)
-      const stream = new PassThrough()
+      const stream = new PassThrough({ objectMode: true })
       process.nextTick(() => {
         for (const item of items) {
           const event = {
             type: 'ADDED',
             object: cloneDeep(item)
           }
-          const chunk = JSON.stringify(event) + '\n'
-          stream.write(chunk)
+          stream.write(event)
+        }
+        if (end === true) {
+          stream.end()
         }
-        stream.end()
       })
       return stream
     }

backend/__fixtures__/terminals.js

@@ -146,7 +146,7 @@ const mocks = {
       return Promise.resolve(item)
     }
   },
-  watch () {
+  watch ({ end = false } = {}) {
     return headers => {
       const [pathname] = split(headers[':path'], '?')
       const matchResult = matchList(pathname)
@@ -156,17 +156,18 @@ const mocks = {
       const { params: { namespace } = {} } = matchResult
       const fieldSelector = parseFieldSelector(headers)
       const items = filter(terminals.list(namespace), fieldSelector)
-      const stream = new PassThrough()
+      const stream = new PassThrough({ objectMode: true })
       process.nextTick(() => {
         for (const item of items) {
           const event = {
             type: 'ADDED',
             object: cloneDeep(item)
           }
-          const chunk = JSON.stringify(event) + '\n'
-          stream.write(chunk)
+          stream.write(event)
+        }
+        if (end === true) {
+          stream.end()
         }
-        stream.end()
       })
       return stream
     }

backend/__mocks__/@gardener-dashboard/kube-client.js

@@ -6,54 +6,9 @@
 
 'use strict'
 
-const { EventEmitter } = require('events')
 const kubeClient = jest.requireActual('@gardener-dashboard/kube-client')
-const { WatchBuilder } = kubeClient
-const NEWLINE = 10
 
-WatchBuilder.create = jest.fn((resource, url, searchParams, name) => {
-  const mockWatch = new EventEmitter()
-  mockWatch.resourceName = resource.constructor.names.plural
-  WatchBuilder.setWaitFor(mockWatch)
-  mockWatch.disconnect = jest.fn()
-  const key = Reflect
-    .ownKeys(resource)
-    .find(key => typeof key === 'symbol' && key.description === 'http.client')
-  const httpClient = resource[key]
-  const options = {
-    method: 'get',
-    searchParams: new URLSearchParams(searchParams.toString())
-  }
-  options.searchParams.set('watch', true)
-  if (name) {
-    let fieldSelector = `metadata.name=${name}`
-    if (searchParams.has('fieldSelector')) {
-      fieldSelector += ',' + searchParams.get('fieldSelector')
-    }
-    options.searchParams.set('fieldSelector', fieldSelector)
-  }
-  const stream = httpClient.request(url, options)
-  process.nextTick(async () => {
-    let data
-    for await (const chunk of stream) {
-      data = Buffer.isBuffer(data)
-        ? Buffer.concat([data, chunk], data.length + chunk.length)
-        : chunk
-      let index
-      while ((index = data.indexOf(NEWLINE)) !== -1) {
-        const event = JSON.parse(data.slice(0, index))
-        mockWatch.emit('event', event)
-        data = data.slice(index + 1)
-      }
-    }
-    // flush
-    if (Buffer.isBuffer(data) && data.length) {
-      const event = JSON.parse(data)
-      mockWatch.emit('event', event)
-    }
-    mockWatch.disconnect()
-  })
-  return mockWatch
-})
-
-module.exports = kubeClient
+module.exports = {
+  ...kubeClient,
+  createDashboardClient: jest.fn().mockImplementation(kubeClient.createDashboardClient)
+}

backend/__mocks__/@gardener-dashboard/request.js

@@ -41,6 +41,10 @@ class MockClient {
     return this[defaults]
   }
 
+  stream (path, options) {
+    return this.request(path, options)
+  }
+
   request (path, { method = 'get', searchParams, headers = {}, json, body } = {}) {
     headers = {
       ...this.defaults.options.headers,

backend/jest.setup.js

@@ -6,6 +6,7 @@
 
 'use strict'
 
+require('abort-controller/polyfill')
 const http = require('http')
 const { Test } = require('supertest')
 const pEvent = require('p-event')
@@ -44,12 +45,10 @@ function createHttpAgent () {
   return agent
 }
 
-function createSocketAgent () {
+function createSocketAgent (cache) {
   const server = http.createServer()
-  const io = require('./lib/io')()
-  server.listen(0, '127.0.0.1', () => {
-    io.attach(server)
-  })
+  const io = require('./lib/io')(server, cache)
+  server.listen(0, '127.0.0.1')
 
   const agent = {
     io,
@@ -87,10 +86,10 @@ function createSocketAgent () {
   return agent
 }
 
-function createAgent (type = 'http') {
+function createAgent (type = 'http', cache) {
   switch (type) {
     case 'io':
-      return createSocketAgent()
+      return createSocketAgent(cache)
     default:
       return createHttpAgent()
   }
@@ -110,15 +109,23 @@ jest.mock('./lib/config/gardener', () => {
 })
 
 jest.mock('./lib/cache', () => {
+  const { find } = require('lodash')
   const fixtures = require('./__fixtures__')
   const originalCache = jest.requireActual('./lib/cache')
   const createTicketCache = jest.requireActual('./lib/cache/tickets')
   const { cache } = originalCache
-  cache.cloudprofiles.replace(fixtures.cloudprofiles.list())
-  cache.seeds.replace(fixtures.seeds.list())
-  cache.quotas.replace(fixtures.quotas.list())
-  cache.projects.replace(fixtures.projects.list())
-  cache.controllerregistrations.replace(fixtures.controllerregistrations.list())
+  const keys = ['cloudprofiles', 'seeds', 'quotas', 'projects', 'controllerregistrations']
+  for (const key of keys) {
+    cache.set(key, {
+      items: fixtures[key].list(),
+      list () {
+        return this.items
+      },
+      find (predicate) {
+        return find(this.list(), predicate)
+      }
+    })
+  }
   cache.ticketCache = createTicketCache()
   cache.resetTicketCache = () => (cache.ticketCache = createTicketCache())
   return originalCache