Merge pull request #5 from WSE-research/reduce-resource-requirements

Reduce resource requirements

.github/workflows/deploy-component.yml

@@ -47,20 +47,20 @@ jobs:
           password: ${{ secrets.SERVER_PASSWORD }}
           envs: DP_DOCKER_IMAGE_NAME,DP_DOCKER_CONTAINER_NAME,DP_DOCKER_CONTAINER_PORT,DP_DOCKER_ENV_FILE,WD_DOCKER_IMAGE_NAME,WD_DOCKER_CONTAINER_NAME,WD_DOCKER_CONTAINER_PORT,WD_DOCKER_ENV_FILE
           script: |
-            # DBpedia
-            echo "Stopping and deleting old container"
-            docker stop "$DP_DOCKER_CONTAINER_NAME" || true && docker rm "$DP_DOCKER_CONTAINER_NAME" || true
-            echo "Pulling image: $DP_DOCKER_IMAGE_NAME"
-            docker pull $DP_DOCKER_IMAGE_NAME:latest
-            echo "Starting container: $DP_DOCKER_CONTAINER_NAME"
-            docker run --restart=unless-stopped -d -p "$DP_DOCKER_CONTAINER_PORT":"$DP_DOCKER_CONTAINER_PORT" --env-file "./env_files/$DP_DOCKER_ENV_FILE" --name "$DP_DOCKER_CONTAINER_NAME" "$DP_DOCKER_IMAGE_NAME:latest"
+            #            # DBpedia
+            #            echo "Stopping and deleting old container"
+            #            docker stop "$DP_DOCKER_CONTAINER_NAME" || true && docker rm "$DP_DOCKER_CONTAINER_NAME" || true
+            #            echo "Pulling image: $DP_DOCKER_IMAGE_NAME"
+            #            docker pull $DP_DOCKER_IMAGE_NAME:latest
+            #            echo "Starting container: $DP_DOCKER_CONTAINER_NAME"
+            #            docker run --restart=unless-stopped --memory="18g" -d -p "$DP_DOCKER_CONTAINER_PORT":"$DP_DOCKER_CONTAINER_PORT" --env-file "./env_files/$DP_DOCKER_ENV_FILE" --name "$DP_DOCKER_CONTAINER_NAME" "$DP_DOCKER_IMAGE_NAME:latest"
             # Wikidata
             echo "Stopping and deleting old container"
             docker stop "$WD_DOCKER_CONTAINER_NAME" || true && docker rm "$WD_DOCKER_CONTAINER_NAME" || true
             echo "Pulling image: $WD_DOCKER_IMAGE_NAME"
             docker pull $WD_DOCKER_IMAGE_NAME:latest
             echo "Starting container: $WD_DOCKER_CONTAINER_NAME"
-            docker run --restart=unless-stopped -d -p "$WD_DOCKER_CONTAINER_PORT":"$WD_DOCKER_CONTAINER_PORT" --env-file "./env_files/$WD_DOCKER_ENV_FILE" --name "$WD_DOCKER_CONTAINER_NAME" "$WD_DOCKER_IMAGE_NAME:latest"
+            docker run --restart=unless-stopped --memory="20g" -d -p "$WD_DOCKER_CONTAINER_PORT":"$WD_DOCKER_CONTAINER_PORT" --env-file "./env_files/$WD_DOCKER_ENV_FILE" --name "$WD_DOCKER_CONTAINER_NAME" "$WD_DOCKER_IMAGE_NAME:latest"
       - name: "Deploy on SWE"
         if: ${{ env.TARGET == 'SWE' }} && steps.setup.outcome == 'success'
         uses: WSE-research/[email protected]

Dockerfile

@@ -2,16 +2,15 @@ FROM python:3.10
 
 COPY requirements.txt ./
 
-RUN pip install --upgrade pip
-RUN pip install -r requirements.txt
-RUN pip install gunicorn
-
 ADD KGQAn/word_embedding ./KGQAn/word_embedding
 ADD KGQAn/src ./KGQAn/src
 
 ADD KGQAn/requirements.txt ./KGQAn/requirements.txt
 RUN pip install -r KGQAn/requirements.txt
 
+RUN pip install --upgrade pip
+RUN pip install -r requirements.txt
+
 COPY component component
 COPY run.py boot.sh start_kgqan_services.sh ./
 

boot.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+export $(grep -v "^#" < .env)
 
 # check required parameters
 declare -a required_vars=(

docker-compose.yml

@@ -1,6 +1,7 @@
 services: 
   component:
     # for building from source
+    mem_limit: 20gb
     image: qanary-component-qb-python-kgqanwrapper:latest
     build: 
       context: .

run.py

@@ -6,7 +6,7 @@
 
 from component import app, healthendpoint, aboutendpoint
 
-logging.basicConfig(level=logging.WARNING)
+logging.basicConfig(level=logging.ERROR)
 # TODO: get logger from module
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)

start_kgqan_services.sh

@@ -66,16 +66,15 @@ mkdir -p "$kgqan_model_path"
 cp -r /KGQAn/data/output_pred21_8_30/* "$kgqan_model_path"
 echo "Copy commands finished"
 
-## TODO: re-enable when resource problem solved
-## start word_embedding_server
-#cd KGQAn/
-#python word_embedding/server.py &
-#
-## start kgqan_server
-#cd src
-#./run.sh server &
-#
-#cd ../../
+# start word_embedding_server
+cd KGQAn/
+python word_embedding/server.py &
+
+# start kgqan_server
+cd src
+./run.sh server &
+
+cd ../../
 
 # continue with CMD from Dockerfile
 $@