wip

.github/workflows/tests.yml

@@ -24,10 +24,10 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
+      # - name: Set up Python ${{ matrix.python-version }}
+      #   uses: actions/setup-python@v2
+      #   with:
+      #     python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip codecov flake8
@@ -37,4 +37,3 @@ jobs:
       - name: Run tests
         run: |
           docker-compose -f docker-compose.test.yml up --build --exit-code-from test
-

Pipfile

@@ -19,7 +19,7 @@ app-common-python = ">=0.2.5"
 boto3 = ">=1.16.13"
 botocore = ">=1.19.13"
 click = ">=0.7"
-connexion = {extras = ["swagger-ui"],version = "==2.14.1"}
+connexion = {extras = ["swagger-ui", "uvicorn"], version = "*"}
 flask = "==2.2.5"
 gitpython = ">=3.1.30"
 iso8601 = ">=0.1.12"
@@ -37,6 +37,7 @@ tornado = ">=6.0.3"
 watchtower = ">=1.0.0"
 werkzeug = ">=1.0.1"
 yarl = ">=1.6.2"
+a2wsgi = "*"
 
 [requires]
 python_version = "3.9"

docker-compose.test.yml

@@ -41,10 +41,15 @@ services:
     security_opt:
       - label=disable
     command: >
-      bash -c 'cd /vmaas && ./scripts/check_deps_versions.sh \
-                         && ./scripts/validate_dashboards.py ./monitoring/grafana/dashboards/ \
-                         && python3 -m vmaas.common.wait_for_services python3 -m vmaas.reposcan.database.upgrade \
-                         && ./run_tests.sh vmaas/common \
-                         && ./run_tests.sh vmaas/reposcan \
-                         && ./run_tests.sh vmaas/webapp \
-                         && bash <(curl -s https://codecov.io/bash)'
+      bash -c 'cd /vmaas && python3 -m vmaas.common.wait_for_services python3 -m vmaas.reposcan.database.upgrade \
+                         && ./run_tests.sh vmaas/reposcan'
+    # command: >
+    #   bash -c 'cd /vmaas && ./scripts/check_deps_versions.sh \
+    #                      && ./scripts/validate_dashboards.py ./monitoring/grafana/dashboards/ \
+    #                      && python3 -m vmaas.common.wait_for_services python3 -m vmaas.reposcan.database.upgrade \
+    #                      && ./run_tests.sh vmaas/common \
+    #                      && ./run_tests.sh vmaas/reposcan \
+    #                      && ./run_tests.sh vmaas/webapp \
+    #                      && bash <(curl -s https://codecov.io/bash)'
+    # command: >
+    #   bash -c 'cd /vmaas && tail -f /dev/null'

docker-compose.yml

@@ -52,72 +52,72 @@ services:
     depends_on:
       - vmaas_database
 
-  vmaas_webapp:
-    command: /vmaas/entrypoint.sh webapp
-    container_name: vmaas-webapp
-    hostname: vmaas-webapp
-    image: vmaas/app:latest
-    restart: unless-stopped
-    env_file:
-      - ./conf/common.env
-      - ./conf/webapp.env
-    ports:
-      - 8080:8000
-      - 9081:10000
-    depends_on:
-      - vmaas_reposcan
+  # vmaas_webapp:
+  #   command: /vmaas/entrypoint.sh webapp
+  #   container_name: vmaas-webapp
+  #   hostname: vmaas-webapp
+  #   image: vmaas/app:latest
+  #   restart: unless-stopped
+  #   env_file:
+  #     - ./conf/common.env
+  #     - ./conf/webapp.env
+  #   ports:
+  #     - 8080:8000
+  #     - 9081:10000
+  #   depends_on:
+  #     - vmaas_reposcan
 
-  vmaas_webapp_go:
-    command: /vmaas/entrypoint.sh webapp-go
-    container_name: vmaas-webapp-go
-    hostname: vmaas-webapp-go
-    image: vmaas/app:latest
-    restart: unless-stopped
-    env_file:
-      - ./conf/common.env
-      - ./conf/webapp_go.env
-    ports:
-      - 8000:8000
-      - 9088:10000
-    depends_on:
-      - vmaas_reposcan
-      - vmaas_webapp # to proxy requests
+  # vmaas_webapp_go:
+  #   command: /vmaas/entrypoint.sh webapp-go
+  #   container_name: vmaas-webapp-go
+  #   hostname: vmaas-webapp-go
+  #   image: vmaas/app:latest
+  #   restart: unless-stopped
+  #   env_file:
+  #     - ./conf/common.env
+  #     - ./conf/webapp_go.env
+  #   ports:
+  #     - 8000:8000
+  #     - 9088:10000
+  #   depends_on:
+  #     - vmaas_reposcan
+  #     - vmaas_webapp # to proxy requests
 
-  vmaas_prometheus:
-    container_name: vmaas-prometheus
-    image: prom/prometheus:v2.21.0
-    volumes:
-      - prometheus-data:/prometheus
-      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
-    security_opt:
-      - label=disable
-    command:
-      - '--config.file=/etc/prometheus/prometheus.yml'
-      - '--storage.tsdb.path=/prometheus'
-      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
-      - '--web.console.templates=/usr/share/prometheus/consoles'
-    ports:
-      - 9090:9090
-    depends_on:
-      - vmaas_reposcan
-      - vmaas_webapp
-    restart: unless-stopped
-    profiles:
-      - monitoring
+  # vmaas_prometheus:
+  #   container_name: vmaas-prometheus
+  #   image: prom/prometheus:v2.21.0
+  #   volumes:
+  #     - prometheus-data:/prometheus
+  #     - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+  #   security_opt:
+  #     - label=disable
+  #   command:
+  #     - '--config.file=/etc/prometheus/prometheus.yml'
+  #     - '--storage.tsdb.path=/prometheus'
+  #     - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+  #     - '--web.console.templates=/usr/share/prometheus/consoles'
+  #   ports:
+  #     - 9090:9090
+  #   depends_on:
+  #     - vmaas_reposcan
+  #     - vmaas_webapp
+  #   restart: unless-stopped
+  #   profiles:
+  #     - monitoring
 
-  vmaas_grafana:
-    container_name: vmaas-grafana
-    build:
-      context: ./
-      dockerfile: ./monitoring/grafana/Dockerfile
-    image: vmaas-grafana:latest
-    depends_on:
-      - vmaas_prometheus
-    ports:
-      - 3000:3000
-    restart: unless-stopped
-    profiles:
-      - monitoring
+  # vmaas_grafana:
+  #   container_name: vmaas-grafana
+  #   build:
+  #     context: ./
+  #     dockerfile: ./monitoring/grafana/Dockerfile
+  #   image: vmaas-grafana:latest
+  #   depends_on:
+  #     - vmaas_prometheus
+  #   ports:
+  #     - 3000:3000
+  #   restart: unless-stopped
+  #   profiles:
+  #     - monitoring
 
 volumes:
   vmaas-db-data:

vmaas/reposcan/dump.output

@@ -0,0 +1 @@
+./dump.output-1702291874.9758973

vmaas/reposcan/reposcan.py

@@ -17,6 +17,8 @@
 
 import connexion
 import git
+from a2wsgi import ASGIMiddleware
+from connexion.options import SwaggerUIOptions
 from flask import request
 from prometheus_client import generate_latest
 from tornado.ioloop import IOLoop, PeriodicCallback
@@ -926,10 +928,8 @@ def terminate(*_):
     for sig in KILL_SIGNALS:
         signal.signal(sig, terminate)
 
-    app = connexion.App(__name__, options={
-        'swagger_ui': True,
-        'openapi_spec_path': '/openapi.json'
-    })
+    app = connexion.App(__name__,
+                        swagger_ui_options=SwaggerUIOptions(swagger_ui=True))
 
     # Response validation is disabled due to returing streamed response in GET /pkgtree
     # https://github.com/zalando/connexion/pull/467 should fix it
@@ -953,4 +953,4 @@ def set_headers(response):  # pylint: disable=unused-variable
         response.headers["Access-Control-Allow-Headers"] = "Content-Type"
         return response
 
-    return app
+    return ASGIMiddleware(app)  # As of connexion >3.0 ASGI server should be used as default, otherwise wrapped by such middleware.

vmaas/reposcan/test/test_case.py

@@ -4,8 +4,10 @@
 import base64
 import json
 from http import HTTPStatus
+from flask import Flask
 import pytest
 from vmaas.reposcan import reposcan
+#  from tornado.wsgi import WSGIContainer
 
 URL_BASE = ""
 
@@ -40,7 +42,7 @@ def app(self):
         connexion_app = reposcan.create_app({reposcan.DEFAULT_PATH + "/v1": "reposcan.spec.yaml",
                                              reposcan.DEFAULT_PATH_API + "/v1": "reposcan.spec.yaml",
                                              "": "reposcan.healthz.spec.yaml"})
-        return connexion_app.app
+        return Flask(connexion_app)
 
     def fetch(self, path, **kwargs):
         """Fetch method for vulnerability API."""

vmaas/webapp/app.py

@@ -17,6 +17,7 @@
 from aiohttp import hdrs
 from aiohttp import web
 from aiohttp.client_exceptions import ClientConnectionError
+from connexion.options import SwaggerUIOptions
 from prometheus_client import generate_latest
 from jsonschema.exceptions import ValidationError
 from vmaas.webapp.cache import Cache
@@ -564,11 +565,12 @@ def format_error(detail, status):
         middlewares.append(gzip_middleware)
     middlewares.extend([error_handler, timing_middleware])
 
-    app = connexion.AioHttpApp(__name__, options={
-        'swagger_ui': True,
-        'openapi_spec_path': '/openapi.json',
-        'middlewares': middlewares
-    })
+    # app = connexion.AsyncApp(__name__, options={
+    #     'swagger_ui': True,
+    #     'openapi_spec_path': '/openapi.json',
+    #     'middlewares': middlewares
+    # })
+    app = connexion.AsyncApp(__name__, swagger_ui_options=SwaggerUIOptions(swagger_ui=True))
 
     def metrics(request, **kwargs):  # pylint: disable=unused-argument
         """Provide current prometheus metrics"""
@@ -580,16 +582,18 @@ def dummy_200(request, **kwargs):  # pylint: disable=unused-argument
         """Dummy endpoint returning 200"""
         return web.Response()
 
-    async def on_prepare(request, response):  # pylint: disable=unused-argument
-        """Hook for preparing new responses"""
-        response.headers["Access-Control-Allow-Origin"] = "*"
-        response.headers["Access-Control-Allow-Headers"] = "Content-Type"
-        response.headers['Access-Control-Allow-Methods'] = 'GET, OPTIONS, POST'
-
-    app.app.on_response_prepare.append(on_prepare)
-    app.app.router.add_get("/metrics", metrics)
-    app.app.router.add_get("/healthz", HealthHandler.get)
-    app.app.router.add_route("OPTIONS", "/api/v3/cves", dummy_200)
+    # async def on_prepare(request, response):  # pylint: disable=unused-argument
+    #     """Hook for preparing new responses"""
+    #     response.headers["Access-Control-Allow-Origin"] = "*"
+    #     response.headers["Access-Control-Allow-Headers"] = "Content-Type"
+    #     response.headers['Access-Control-Allow-Methods'] = 'GET, OPTIONS, POST'
+
+    # app.app.on_response_prepare.append(on_prepare)
+    app.add_url_rule("/", 'metrics', metrics)
+    app.add_url_rule("/", 'healthz', HealthHandler.get)
+    # app.app.router.add_get("/metrics", metrics)
+    # app.app.router.add_get("/healthz", HealthHandler.get)
+    # app.app.router.add_route("OPTIONS", "/api/v3/cves", dummy_200)
 
     for route, spec in specs.items():
         app.add_api(spec, resolver=connexion.RestyResolver('app'),

vmaas/webapp/main.py

@@ -1,7 +1,5 @@
 """Entry point for the application"""
-import asyncio
 
-from aiohttp import web
 from prometheus_client import start_http_server
 from vmaas.webapp.app import create_app, DEFAULT_PATH, DEFAULT_PATH_API
 from vmaas.webapp.app import init_refresh_timer
@@ -12,17 +10,18 @@
 
 LOGGER = get_logger(__name__)
 
+application = create_app({DEFAULT_PATH + "/v1": "webapp.v1.spec.yaml",
+                          DEFAULT_PATH + "/v2": "webapp.v2.spec.yaml",
+                          DEFAULT_PATH + "/v3": "webapp.v3.spec.yaml",
+                          DEFAULT_PATH_API + "/v1": "webapp.v1.spec.yaml",
+                          DEFAULT_PATH_API + "/v2": "webapp.v2.spec.yaml",
+                          DEFAULT_PATH_API + "/v3": "webapp.v3.spec.yaml"})
+
 if __name__ == '__main__':
     init_logging()
     # pylint: disable=invalid-name
-    application = create_app({DEFAULT_PATH + "/v1": "webapp.v1.spec.yaml",
-                              DEFAULT_PATH + "/v2": "webapp.v2.spec.yaml",
-                              DEFAULT_PATH + "/v3": "webapp.v3.spec.yaml",
-                              DEFAULT_PATH_API + "/v1": "webapp.v1.spec.yaml",
-                              DEFAULT_PATH_API + "/v2": "webapp.v2.spec.yaml",
-                              DEFAULT_PATH_API + "/v3": "webapp.v3.spec.yaml"})
     init_refresh_timer()
     cfg = Config()
 
     start_http_server(int(cfg.metrics_port))
-    web.run_app(application.app, port=cfg.public_port, access_log_format="%s %r (%a) %Tfs", loop=asyncio.get_event_loop())
+    application.run("main:application", port=cfg.public_port)