diff --git a/.asf.yaml b/.asf.yaml index dbecc8f2..4d7fc50d 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -1,3 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + github: enabled_merge_buttons: squash: true @@ -6,4 +25,4 @@ github: features: wiki: true issues: false - projects: false \ No newline at end of file + projects: false diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..f7f3f801 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,49 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +language: node_js +node_js: + - "12" + +git: + depth: 1 + +services: + - 'docker' + +jobs: + include: + - stage: pre-commit checks + install: + - rm -rf package-lock.json node_modules + - npm install -g @angular/cli + - npm install karma + - npm install -g protractor + - npm install yarn + script: + - make check-license + - make build-prod + - yarn test:coverage + - yarn lint + - stage: publish docker image + install: skip + deploy: + provider: script + script: make push + on: + branch: master + condition: $TRAVIS_EVENT_TYPE = cron diff --git a/Makefile b/Makefile index 35861caa..8b4fe9c2 100644 --- a/Makefile +++ b/Makefile @@ -30,13 +30,22 @@ endif # Image build parameters # This tag of the image must be changed when pushed to a public repository. -ifeq ($(TAG),) -TAG := yunikorn/yunikorn-web +ifeq ($(REGISTRY),) +REGISTRY := apache endif # Set the default web port, this must be the same as in the nginx/nginx.conf file. PORT=9889 +.PHONY: check-license +check-license: + @echo "checking license header" + @licRes=$$(grep -Lr --exclude-dir={node_modules,dist} --include=*.{sh,md,yaml,yml,js,ts,html,js,scss} "Licensed to the Apache Software Foundation" .) ; \ + if [ -n "$${licRes}" ]; then \ + echo "following files have incorrect license header:\n$${licRes}" ; \ + exit 1; \ + fi + # Local build and deploy with compose .PHONY: deploy-prod deploy-prod: @@ -50,7 +59,7 @@ start-dev: # Run the web interface from the production image .PHONY: run run: image - docker run -d -p ${PORT}:9889 ${TAG}:${VERSION} + docker run -d -p ${PORT}:9889 ${REGISTRY}/yunikorn:web-${VERSION} # Build the web interface in a production ready version .PHONY: build-prod @@ -58,9 +67,11 @@ build-prod: yarn install && yarn build:prod # Build an image based on the production ready version -image: build-prod +.PHONY: image +image: + @echo "building web UI docker image" @SHA=$$(git rev-parse --short=12 HEAD) ; \ - docker build -t ${TAG}:${VERSION} . \ + docker build -t ${REGISTRY}/yunikorn:web-${VERSION} . \ --label "GitRevision=$${SHA}" \ --label "Version=${VERSION}" \ --label "BuildTimeStamp=${DATE}" @@ -82,3 +93,9 @@ clean: rm -rf ./node_modules rm -rf ./out rm -rf ./out-tsc + +.PHONY: push_image +push_image: image + @echo "push docker images" + echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin + docker push ${REGISTRY}/yunikorn:web-${VERSION} diff --git a/README.md b/README.md index 8ff438ed..a99183c2 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,21 @@ + + # Yunikorn web UI YuniKorn web provides a web interface on top of the scheduler. It provides insight in the current and historic scheduler status. It depends on `yunikorn-core` which encapsulates all the actual scheduling logic. @@ -31,11 +49,11 @@ Image builds are geared towards a production build and will always build with th Run `make image` to build the docker image `yunikorn-web`. Run `make run` to build the image and deploy the container from the docker image `yunikorn-web`. -You can set `TAG` and `VERSION` in the commandline to build docker image with a specified version and tag. For example, +You can set `REGISTRY` and `VERSION` in the commandline to build docker image with a specified version and registry. For example, ``` -make image TAG=yunikorn/yunikorn-web VERSION=latest +make image REGISTRY=yunikorn VERSION=latest ``` -this command will build binary with version `latest` and the docker image tag is `yunikorn/yunikorn-web:latest`. +This command will build binary with version `web-latest` and the docker full image tag is `yunikorn/yunikorn:web-latest`. Run `make deploy-prod` to build and deploy the scheduler webapp using docker-compose. The project uses [multi-stage build](https://docs.docker.com/develop/develop-images/multistage-build/) feature of the docker and requires Docker 17.05 or higher. diff --git a/docker-compose.yml b/docker-compose.yml index 7a1170af..25179998 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + version: '3' services: yunikornwebapp: build: . - image: yunikorn/yunikorn-web + image: yunikorn/yunikorn:web-latest ports: - 9889:9889 diff --git a/docker_start.sh b/docker_start.sh index 1d690773..36e23a00 100755 --- a/docker_start.sh +++ b/docker_start.sh @@ -27,8 +27,8 @@ yarn install echo "[3/5] Building modules..." yarn build:prod -echo "[4/5] Building docker image yunikorn/yunikorn-web:latest..." -docker build -t yunikorn/yunikorn-web:latest -f ./nginx/Dockerfile . +echo "[4/5] Building docker image apache/yunikorn:web-latest..." +docker build -t apache/yunikorn:web-latest -f ./nginx/Dockerfile . -echo "[5/5] Starting docker container using image yunikorn/yunikorn-web:latest..." -docker run -d -p 9889:9889 yunikorn/yunikorn-web:latest +echo "[5/5] Starting docker container using image apache/yunikorn:web-latest..." +docker run -d -p 9889:9889 apache/yunikorn:web-latest diff --git a/docs/how-to-contribute.md b/docs/how-to-contribute.md index 7a354a99..5bc9a253 100644 --- a/docs/how-to-contribute.md +++ b/docs/how-to-contribute.md @@ -1,3 +1,20 @@ + # How do I contribute code? ### Find diff --git a/jsdb.json b/jsdb.json index 157bc48e..a2c3c9ec 100644 --- a/jsdb.json +++ b/jsdb.json @@ -23,53 +23,51 @@ "usedcapacity": "0" }, "nodes": null, - "queues": [ - { - "queuename": "root", - "status": "RUNNING", - "capacities": { - "capacity": "[memory:1000000 vcore:100000]", - "maxcapacity": "[memory:1000000 vcore:100000]", - "usedcapacity": "[memory:5908 vcore:4000]", - "absusedcapacity": "20" + "queues": { + "queuename": "root", + "status": "RUNNING", + "capacities": { + "capacity": "[memory:1000000 vcore:100000]", + "maxcapacity": "[memory:1000000 vcore:100000]", + "usedcapacity": "[memory:5908 vcore:4000]", + "absusedcapacity": "20" + }, + "queues": [ + { + "queuename": "advertisement", + "status": "RUNNING", + "capacities": { + "capacity": "[memory:500000 vcore:50000]", + "maxcapacity": "[memory:500000 vcore:50000]", + "usedcapacity": "[memory:5908 vcore:4000]", + "absusedcapacity": "20" + }, + "queues": null }, - "queues": [ - { - "queuename": "advertisement", - "status": "RUNNING", - "capacities": { - "capacity": "[memory:500000 vcore:50000]", - "maxcapacity": "[memory:500000 vcore:50000]", - "usedcapacity": "[memory:5908 vcore:4000]", - "absusedcapacity": "20" - }, - "queues": null + { + "queuename": "search", + "status": "RUNNING", + "capacities": { + "capacity": "[memory:400000 vcore:40000]", + "maxcapacity": "[memory:400000 vcore:40000]", + "usedcapacity": "[]", + "absusedcapacity": "20" }, - { - "queuename": "search", - "status": "RUNNING", - "capacities": { - "capacity": "[memory:400000 vcore:40000]", - "maxcapacity": "[memory:400000 vcore:40000]", - "usedcapacity": "[]", - "absusedcapacity": "20" - }, - "queues": null + "queues": null + }, + { + "queuename": "sandbox", + "status": "RUNNING", + "capacities": { + "capacity": "[memory:100000 vcore:10000]", + "maxcapacity": "[vcore:10000 memory:100000]", + "usedcapacity": "[]", + "absusedcapacity": "20" }, - { - "queuename": "sandbox", - "status": "RUNNING", - "capacities": { - "capacity": "[memory:100000 vcore:10000]", - "maxcapacity": "[vcore:10000 memory:100000]", - "usedcapacity": "[]", - "absusedcapacity": "20" - }, - "queues": null - } - ] - } - ] + "queues": null + } + ] + } }, "apps": [ { @@ -209,5 +207,345 @@ "timestamp": 1585238563807452000, "totalContainers": "2" } + ], + "nodes": [ + { + "partitionName": "[mycluster]default", + "nodesInfo": [ + { + "nodeID": "ip-10-97-251-125.us-west-2.compute.internal", + "hostName": "ip-10-97-251-125.us-west-2.compute.internal", + "rackName": "/rack-default", + "capacity": "[attachable-volumes-aws-ebs:25 ephemeral-storage:94477937300 hugepages-1Gi:0 hugepages-2Mi:0 memory:7463 pods:29 vcore:1900]", + "allocated": "[memory:3958 vcore:1710]", + "occupied": "[]", + "available": "[attachable-volumes-aws-ebs:25 ephemeral-storage:94477937300 hugepages-1Gi:0 hugepages-2Mi:0 memory:3505 pods:29 vcore:190]", + "allocations": [ + { + "allocationKey": "036954e8-78d7-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "d662b17b-7fdd-44b5-ae37-0f697277dc0f", + "resource": "[vcore:100]", + "priority": "", + "queueName": "root.kube-system", + "nodeId": "ip-10-97-251-125.us-west-2.compute.internal", + "applicationId": "kube-proxy-_1586267545875054833", + "partition": "default" + }, + { + "allocationKey": "03693499-78d7-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "4fbe06b7-6e37-4a61-bb33-2c629391cfd0", + "resource": "[vcore:10]", + "priority": "", + "queueName": "root.kube-system", + "nodeId": "ip-10-97-251-125.us-west-2.compute.internal", + "applicationId": "aws-node-_1586267545873980492", + "partition": "default" + }, + { + "allocationKey": "cbae18dd-78d6-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "b8f28638-17f9-428b-94a1-55b640dda5fa", + "resource": "[memory:1074 vcore:500]", + "priority": "", + "queueName": "root.dex-app-4dxkn4mx", + "nodeId": "ip-10-97-251-125.us-west-2.compute.internal", + "applicationId": "mean-heron-web-67fc57dbd8-_1586267452372775067", + "partition": "default" + }, + { + "allocationKey": "cbaf5077-78d6-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "efdbbe9a-7929-48c9-afd1-c55f99d2deac", + "resource": "[memory:1074 vcore:500]", + "priority": "", + "queueName": "root.dex-app-4dxkn4mx", + "nodeId": "ip-10-97-251-125.us-west-2.compute.internal", + "applicationId": "mean-heron-livy-856db4f4d8-_1586267452380963122", + "partition": "default" + }, + { + "allocationKey": "cbae6510-78d6-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "5b911654-0123-480f-86c3-c5ed56871776", + "resource": "[memory:1074 vcore:500]", + "priority": "", + "queueName": "root.dex-app-4dxkn4mx", + "nodeId": "ip-10-97-251-125.us-west-2.compute.internal", + "applicationId": "mean-heron-shs-6f455dd86f-_1586267452374806532", + "partition": "default" + }, + { + "allocationKey": "0f5b3f2d-78d7-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "fb4f65a8-e434-4ff8-9c7f-e5a1633e0c9e", + "resource": "[memory:525]", + "priority": "", + "queueName": "root.logging", + "nodeId": "ip-10-97-251-125.us-west-2.compute.internal", + "applicationId": "cdp-fluentd-_1586267565914703700", + "partition": "default" + }, + { + "allocationKey": "0f5b0ae5-78d7-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "4c33c27e-5b52-41f1-95c1-a81d480f02b9", + "resource": "[memory:1]", + "priority": "", + "queueName": "root.monitoring", + "nodeId": "ip-10-97-251-125.us-west-2.compute.internal", + "applicationId": "monitoring-prometheus-node-exporter-_1586267565913942067", + "partition": "default" + }, + { + "allocationKey": "0f5f56f1-78d7-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "65cf4028-5b36-46c2-ac2e-5dbfb00975e9", + "resource": "[memory:210 vcore:100]", + "priority": "", + "queueName": "root.dex", + "nodeId": "ip-10-97-251-125.us-west-2.compute.internal", + "applicationId": "fluentd-_1586267565942134087", + "partition": "default" + } + ], + "schedulable": true + }, + { + "nodeID": "ip-10-97-247-177.us-west-2.compute.internal", + "hostName": "ip-10-97-247-177.us-west-2.compute.internal", + "rackName": "/rack-default", + "capacity": "[attachable-volumes-aws-ebs:25 ephemeral-storage:94477937300 hugepages-1Gi:0 hugepages-2Mi:0 memory:6280 pods:29 vcore:1250]", + "allocated": "[]", + "occupied": "[memory:218 vcore:210]", + "available": "[attachable-volumes-aws-ebs:25 ephemeral-storage:94477937300 hugepages-1Gi:0 hugepages-2Mi:0 memory:6062 pods:29 vcore:1040]", + "allocations": null, + "schedulable": true + }, + { + "nodeID": "ip-10-97-241-166.us-west-2.compute.internal", + "hostName": "ip-10-97-241-166.us-west-2.compute.internal", + "rackName": "/rack-default", + "capacity": "[attachable-volumes-aws-ebs:25 ephemeral-storage:38601419508 hugepages-1Gi:0 hugepages-2Mi:0 memory:7843 pods:29 vcore:1998]", + "allocated": "[]", + "occupied": "[memory:919 vcore:310]", + "available": "[attachable-volumes-aws-ebs:25 ephemeral-storage:38601419508 hugepages-1Gi:0 hugepages-2Mi:0 memory:6924 pods:29 vcore:1688]", + "allocations": null, + "schedulable": true + }, + { + "nodeID": "ip-10-97-249-226.us-west-2.compute.internal", + "hostName": "ip-10-97-249-226.us-west-2.compute.internal", + "rackName": "/rack-default", + "capacity": "[attachable-volumes-aws-ebs:25 ephemeral-storage:94477937300 hugepages-1Gi:0 hugepages-2Mi:0 memory:7463 pods:29 vcore:1900]", + "allocated": "[memory:3958 vcore:1710]", + "occupied": "[]", + "available": "[attachable-volumes-aws-ebs:25 ephemeral-storage:94477937300 hugepages-1Gi:0 hugepages-2Mi:0 memory:3505 pods:29 vcore:190]", + "allocations": [ + { + "allocationKey": "cbb629c9-78d6-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "5434e58b-ccb4-4484-9536-312594908b94", + "resource": "[memory:1074 vcore:500]", + "priority": "", + "queueName": "root.dex-app-4dxkn4mx", + "nodeId": "ip-10-97-249-226.us-west-2.compute.internal", + "applicationId": "mean-heron-safari-5b5d7854b7-_1586267452426030808", + "partition": "default" + }, + { + "allocationKey": "cbad7dbe-78d6-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "274c8bba-2144-4d51-bb54-024a29d477f1", + "resource": "[memory:1074 vcore:500]", + "priority": "", + "queueName": "root.dex-app-4dxkn4mx", + "nodeId": "ip-10-97-249-226.us-west-2.compute.internal", + "applicationId": "mean-heron-airflowapi-bf86bdffd-_1586267452369027873", + "partition": "default" + }, + { + "allocationKey": "156690e4-78d7-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "1b1f8f09-0a32-4eaa-b3e4-8121ffd3fba6", + "resource": "[memory:525]", + "priority": "", + "queueName": "root.logging", + "nodeId": "ip-10-97-249-226.us-west-2.compute.internal", + "applicationId": "cdp-fluentd-_1586267576055729704", + "partition": "default" + }, + { + "allocationKey": "156689d4-78d7-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "5804da82-8131-402f-b2e0-8422c445127f", + "resource": "[memory:1]", + "priority": "", + "queueName": "root.monitoring", + "nodeId": "ip-10-97-249-226.us-west-2.compute.internal", + "applicationId": "monitoring-prometheus-node-exporter-_1586267576054574724", + "partition": "default" + }, + { + "allocationKey": "15694abd-78d7-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "4c128885-2ed6-4127-89e4-70a490c0265f", + "resource": "[memory:210 vcore:100]", + "priority": "", + "queueName": "root.dex", + "nodeId": "ip-10-97-249-226.us-west-2.compute.internal", + "applicationId": "fluentd-_1586267576073633162", + "partition": "default" + }, + { + "allocationKey": "0974c389-78d7-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "bf9f9c0b-78cf-4e82-976a-3af94f011e23", + "resource": "[vcore:100]", + "priority": "", + "queueName": "root.kube-system", + "nodeId": "ip-10-97-249-226.us-west-2.compute.internal", + "applicationId": "kube-proxy-_1586267556016295572", + "partition": "default" + }, + { + "allocationKey": "09749d93-78d7-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "564f2a71-4df8-48ec-9611-300ff260bb9a", + "resource": "[vcore:10]", + "priority": "", + "queueName": "root.kube-system", + "nodeId": "ip-10-97-249-226.us-west-2.compute.internal", + "applicationId": "aws-node-_1586267556015166954", + "partition": "default" + }, + { + "allocationKey": "cbb66256-78d6-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "a6901552-0762-4b2d-9aca-a88d0168369e", + "resource": "[memory:1074 vcore:500]", + "priority": "", + "queueName": "root.dex-app-4dxkn4mx", + "nodeId": "ip-10-97-249-226.us-west-2.compute.internal", + "applicationId": "mean-heron-redis-master-0_1586267452427210685", + "partition": "default" + } + ], + "schedulable": true + }, + { + "nodeID": "ip-10-97-240-9.us-west-2.compute.internal", + "hostName": "ip-10-97-240-9.us-west-2.compute.internal", + "rackName": "/rack-default", + "capacity": "[attachable-volumes-aws-ebs:25 ephemeral-storage:38601419508 hugepages-1Gi:0 hugepages-2Mi:0 memory:7843 pods:29 vcore:1998]", + "allocated": "[]", + "occupied": "[memory:1628 vcore:710]", + "available": "[attachable-volumes-aws-ebs:25 ephemeral-storage:38601419508 hugepages-1Gi:0 hugepages-2Mi:0 memory:6215 pods:29 vcore:1288]", + "allocations": null, + "schedulable": true + }, + { + "nodeID": "ip-10-97-250-244.us-west-2.compute.internal", + "hostName": "ip-10-97-250-244.us-west-2.compute.internal", + "rackName": "/rack-default", + "capacity": "[attachable-volumes-aws-ebs:25 hugepages-1Gi:0 hugepages-2Mi:0 memory:7463 pods:29 vcore:1900]", + "allocated": "[memory:3958 vcore:1710]", + "occupied": "[]", + "available": "[attachable-volumes-aws-ebs:25 hugepages-1Gi:0 hugepages-2Mi:0 memory:3505 pods:29 vcore:190]", + "allocations": [ + { + "allocationKey": "11946368-78d7-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "24cab7f8-49ef-499a-be5c-0ab91f0e9af7", + "resource": "[memory:525]", + "priority": "", + "queueName": "root.logging", + "nodeId": "ip-10-97-250-244.us-west-2.compute.internal", + "applicationId": "cdp-fluentd-_1586267569644646509", + "partition": "default" + }, + { + "allocationKey": "11946e82-78d7-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "ef9e99de-61b8-4b18-8272-6de7087e8c76", + "resource": "[memory:1]", + "priority": "", + "queueName": "root.monitoring", + "nodeId": "ip-10-97-250-244.us-west-2.compute.internal", + "applicationId": "monitoring-prometheus-node-exporter-_1586267569644085414", + "partition": "default" + }, + { + "allocationKey": "05ac28a1-78d7-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "2ac3e691-9a1f-4f5c-a9cc-d6afbc3a10b1", + "resource": "[vcore:10]", + "priority": "", + "queueName": "root.kube-system", + "nodeId": "ip-10-97-250-244.us-west-2.compute.internal", + "applicationId": "aws-node-_1586267549665450723", + "partition": "default" + }, + { + "allocationKey": "05ac2f5a-78d7-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "23bd81e1-1a8f-4321-b733-d7d47cb614df", + "resource": "[vcore:100]", + "priority": "", + "queueName": "root.kube-system", + "nodeId": "ip-10-97-250-244.us-west-2.compute.internal", + "applicationId": "kube-proxy-_1586267549665884159", + "partition": "default" + }, + { + "allocationKey": "cbadbb6b-78d6-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "1899ccb7-c4e2-4564-bb27-5bdbf7289ee9", + "resource": "[memory:1074 vcore:500]", + "priority": "", + "queueName": "root.dex-app-4dxkn4mx", + "nodeId": "ip-10-97-250-244.us-west-2.compute.internal", + "applicationId": "mean-heron-scheduler-76884b477b-_1586267452369749149", + "partition": "default" + }, + { + "allocationKey": "cbbb7fcf-78d6-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "d9aed216-6732-4502-b04e-60ac489cdb83", + "resource": "[memory:1074 vcore:500]", + "priority": "", + "queueName": "root.dex-app-4dxkn4mx", + "nodeId": "ip-10-97-250-244.us-west-2.compute.internal", + "applicationId": "mean-heron-worker-0_1586267452460954234", + "partition": "default" + }, + { + "allocationKey": "cbb4cf93-78d6-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "f0eabf29-96c0-429f-b20e-da5e69252cb1", + "resource": "[memory:1074 vcore:500]", + "priority": "", + "queueName": "root.dex-app-4dxkn4mx", + "nodeId": "ip-10-97-250-244.us-west-2.compute.internal", + "applicationId": "mean-heron-dex-app-api-7f9585765-_1586267452415423807", + "partition": "default" + }, + { + "allocationKey": "1197b3e0-78d7-11ea-a9f2-0aafc96ae9e0", + "allocationTags": null, + "uuid": "8157574d-ed6a-49fe-be92-2b42b11d8679", + "resource": "[memory:210 vcore:100]", + "priority": "", + "queueName": "root.dex", + "nodeId": "ip-10-97-250-244.us-west-2.compute.internal", + "applicationId": "fluentd-_1586267569666943847", + "partition": "default" + } + ], + "schedulable": true + } + ] + } ] } diff --git a/landmark/docker_images.yaml b/landmark/docker_images.yaml new file mode 100644 index 00000000..6d859e31 --- /dev/null +++ b/landmark/docker_images.yaml @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +docker_images: + yunikorn-web: __registry__/yunikorn:web- diff --git a/src/app/app.module.ts b/src/app/app.module.ts index f913ceec..64f9b0d9 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -22,6 +22,7 @@ import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; import { RouterModule, Routes } from '@angular/router'; import { HttpClientModule } from '@angular/common/http'; import { NgxSpinnerModule } from 'ngx-spinner'; +import { FormsModule } from '@angular/forms'; import { MatCardModule, MatTabsModule, @@ -32,7 +33,9 @@ import { MatSidenavModule, MatTableModule, MatPaginatorModule, - MatSortModule + MatSortModule, + MatInputModule, + MatTooltipModule } from '@angular/material'; import { envConfigFactory, EnvconfigService } from './services/envconfig/envconfig.service'; @@ -49,6 +52,7 @@ import { ContainerStatusComponent } from './components/container-status/containe import { ContainerHistoryComponent } from './components/container-history/container-history.component'; import { QueueRackComponent } from './components/queue-rack/queue-rack.component'; import { AppsViewComponent } from './components/apps-view/apps-view.component'; +import { NodesViewComponent } from './components/nodes-view/nodes-view.component'; const appRoutes: Routes = [ { @@ -76,6 +80,11 @@ const appRoutes: Routes = [ component: QueuesViewComponent, data: { breadcrumb: 'Queues' } }, + { + path: 'nodes', + component: NodesViewComponent, + data: { breadcrumb: 'Nodes' } + }, { path: '', pathMatch: 'full', @@ -104,13 +113,15 @@ const appRoutes: Routes = [ ContainerStatusComponent, ContainerHistoryComponent, QueueRackComponent, - AppsViewComponent + AppsViewComponent, + NodesViewComponent ], imports: [ BrowserModule, BrowserAnimationsModule, HttpClientModule, NgxSpinnerModule, + FormsModule, MatCardModule, MatTabsModule, MatSelectModule, @@ -121,6 +132,8 @@ const appRoutes: Routes = [ MatTableModule, MatPaginatorModule, MatSortModule, + MatInputModule, + MatTooltipModule, RouterModule.forRoot(appRoutes) ], providers: [ diff --git a/src/app/components/apps-view/apps-view.component.html b/src/app/components/apps-view/apps-view.component.html index ba198fe9..591f4f17 100644 --- a/src/app/components/apps-view/apps-view.component.html +++ b/src/app/components/apps-view/apps-view.component.html @@ -16,7 +16,26 @@ * limitations under the License. --> -
+
+ + + + +
@@ -25,7 +44,7 @@ {{ element['formattedSubmissionTime'] }} - {{ element[columnDef.colId] }} + {{ element[columnDef.colId] || 'n/a' }} @@ -60,23 +79,23 @@
-
-

App Allocations

+
+

Allocations

{{ columnDef.colName }} - {{ element[columnDef.colId] }} + {{ element[columnDef.colId] || 'n/a' }} @@ -97,7 +116,7 @@

App Allocations

diff --git a/src/app/components/apps-view/apps-view.component.scss b/src/app/components/apps-view/apps-view.component.scss index f36b265b..010c13b7 100644 --- a/src/app/components/apps-view/apps-view.component.scss +++ b/src/app/components/apps-view/apps-view.component.scss @@ -16,7 +16,7 @@ * limitations under the License. */ -.jobs-view { +.apps-view { width: 100%; height: 100%; padding: 25px; @@ -34,7 +34,6 @@ cursor: pointer; } &.selected-row { - // background: #bbbbbb; background: #303d54; .mat-cell { color: #fff; @@ -45,8 +44,9 @@ .mat-cell.indicator-icon { max-width: 40px; font-size: 18px; + margin-left: 10px; } - .job-details { + .app-allocations { margin-top: 40px; .mat-table { margin-top: 20px; @@ -59,4 +59,28 @@ width: 100%; text-align: center; } + .search-wrapper { + width: 500px; + input { + width: calc(100% - 22px); + color: #333; + } + .clear-btn { + outline: none; + border: none; + padding: 0 0 0 4px; + cursor: pointer; + background: transparent; + i { + font-size: 18px; + &:hover { + color: #f44336; + } + } + } + .search-icon { + margin-left: 4px; + font-size: 17px; + } + } } diff --git a/src/app/components/apps-view/apps-view.component.spec.ts b/src/app/components/apps-view/apps-view.component.spec.ts index b21ba570..7615ced7 100644 --- a/src/app/components/apps-view/apps-view.component.spec.ts +++ b/src/app/components/apps-view/apps-view.component.spec.ts @@ -21,11 +21,14 @@ import { NoopAnimationsModule } from '@angular/platform-browser/animations'; import { HAMMER_LOADER } from '@angular/platform-browser'; import { NgxSpinnerService } from 'ngx-spinner'; import { configureTestSuite } from 'ng-bullet'; +import { FormsModule } from '@angular/forms'; import { MatTableModule, MatPaginatorModule, MatDividerModule, - MatSortModule + MatSortModule, + MatInputModule, + MatTooltipModule } from '@angular/material'; import { AppsViewComponent } from './apps-view.component'; @@ -41,10 +44,13 @@ describe('AppsViewComponent', () => { declarations: [AppsViewComponent], imports: [ NoopAnimationsModule, + FormsModule, MatTableModule, MatPaginatorModule, MatDividerModule, - MatSortModule + MatSortModule, + MatInputModule, + MatTooltipModule ], providers: [ { provide: SchedulerService, useValue: MockSchedulerService }, diff --git a/src/app/components/apps-view/apps-view.component.ts b/src/app/components/apps-view/apps-view.component.ts index cd322aa5..aeea25d7 100644 --- a/src/app/components/apps-view/apps-view.component.ts +++ b/src/app/components/apps-view/apps-view.component.ts @@ -16,18 +16,16 @@ * limitations under the License. */ -import { Component, OnInit, ViewChild } from '@angular/core'; +import { Component, OnInit, ViewChild, ElementRef } from '@angular/core'; +import { MatPaginator, MatTableDataSource, MatSort } from '@angular/material'; +import { finalize, debounceTime, distinctUntilChanged } from 'rxjs/operators'; import { NgxSpinnerService } from 'ngx-spinner'; -import { finalize } from 'rxjs/operators'; -import { MatPaginator, MatTableDataSource, PageEvent, MatSort } from '@angular/material'; +import { fromEvent } from 'rxjs'; import { SchedulerService } from '@app/services/scheduler/scheduler.service'; -import { AppInfo, AppAllocation } from '@app/models/app-info.model'; - -export interface ColumnDef { - colId: string; - colName: string; -} +import { AppInfo } from '@app/models/app-info.model'; +import { AllocationInfo } from '@app/models/alloc-info.model'; +import { ColumnDef } from '@app/models/column-def.model'; @Component({ selector: 'app-applications-view', @@ -35,19 +33,22 @@ export interface ColumnDef { styleUrls: ['./apps-view.component.scss'] }) export class AppsViewComponent implements OnInit { - @ViewChild('jobsViewMatPaginator', { static: true }) appPaginator: MatPaginator; + @ViewChild('appsViewMatPaginator', { static: true }) appPaginator: MatPaginator; @ViewChild('allocationMatPaginator', { static: true }) allocPaginator: MatPaginator; @ViewChild(MatSort, { static: true }) appSort: MatSort; + @ViewChild('searchInput', { static: true }) searchInput: ElementRef; appDataSource = new MatTableDataSource([]); appColumnDef: ColumnDef[] = []; appColumnIds: string[] = []; - allocDataSource = new MatTableDataSource([]); + allocDataSource = new MatTableDataSource([]); allocColumnDef: ColumnDef[] = []; allocColumnIds: string[] = []; selectedRow: AppInfo | null = null; + initialAppData: AppInfo[] = []; + searchText = ''; constructor(private scheduler: SchedulerService, private spinner: NgxSpinnerService) {} @@ -59,9 +60,9 @@ export class AppsViewComponent implements OnInit { this.appColumnDef = [ { colId: 'applicationId', colName: 'Application ID' }, + { colId: 'queueName', colName: 'Queue Name' }, { colId: 'applicationState', colName: 'Application State' }, { colId: 'usedResource', colName: 'Used Resource' }, - { colId: 'queueName', colName: 'Queue Name' }, { colId: 'partition', colName: 'Partition' }, { colId: 'submissionTime', colName: 'Submission Time' } ]; @@ -89,14 +90,21 @@ export class AppsViewComponent implements OnInit { }) ) .subscribe(data => { + this.initialAppData = data; this.appDataSource.data = data; }); + + fromEvent(this.searchInput.nativeElement, 'keyup') + .pipe(debounceTime(500), distinctUntilChanged()) + .subscribe(() => { + this.onSearchAppData(); + }); } unselectAllRowsButOne(row: AppInfo) { - this.appDataSource.data.map(job => { - if (job !== row) { - job.isSelected = false; + this.appDataSource.data.map(app => { + if (app !== row) { + app.isSelected = false; } }); } @@ -114,7 +122,7 @@ export class AppsViewComponent implements OnInit { } } - onPaginatorChanged(page: PageEvent) { + removeRowSelection() { if (this.selectedRow) { this.selectedRow.isSelected = false; this.selectedRow = null; @@ -122,6 +130,10 @@ export class AppsViewComponent implements OnInit { } } + onPaginatorChanged() { + this.removeRowSelection(); + } + isAppDataSourceEmpty() { return this.appDataSource.data && this.appDataSource.data.length === 0; } @@ -129,4 +141,25 @@ export class AppsViewComponent implements OnInit { isAllocDataSourceEmpty() { return this.allocDataSource.data && this.allocDataSource.data.length === 0; } + + onClearSearch() { + this.searchText = ''; + this.removeRowSelection(); + this.appDataSource.data = this.initialAppData; + } + + onSearchAppData() { + const searchTerm = this.searchText.trim().toLowerCase(); + + if (searchTerm) { + this.removeRowSelection(); + this.appDataSource.data = this.initialAppData.filter( + data => + data.applicationId.toLowerCase().includes(searchTerm) || + data.queueName.toLowerCase().includes(searchTerm) + ); + } else { + this.onClearSearch(); + } + } } diff --git a/src/app/components/cluster-container/cluster-container.component.html b/src/app/components/cluster-container/cluster-container.component.html index deb03b42..446fbd0a 100644 --- a/src/app/components/cluster-container/cluster-container.component.html +++ b/src/app/components/cluster-container/cluster-container.component.html @@ -26,6 +26,9 @@ Queues + + Nodes + diff --git a/src/app/components/cluster-info/cluster-info.component.ts b/src/app/components/cluster-info/cluster-info.component.ts index 3aa696ce..f68a2abd 100644 --- a/src/app/components/cluster-info/cluster-info.component.ts +++ b/src/app/components/cluster-info/cluster-info.component.ts @@ -57,7 +57,7 @@ export class ClusterInfoComponent implements OnInit { }) ) .subscribe(data => { - this.updateJobStatusData(data); + this.updateAppStatusData(data); this.updateContainerStatusData(data); }); @@ -70,7 +70,7 @@ export class ClusterInfoComponent implements OnInit { }); } - updateJobStatusData(info: ClusterInfo) { + updateAppStatusData(info: ClusterInfo) { this.appStatusData = [ new DonutDataItem('Failed', +info.failedApplications, '#cc6164'), new DonutDataItem('Pending', +info.pendingApplications, '#facc54'), diff --git a/src/app/components/dashboard/dashboard.component.scss b/src/app/components/dashboard/dashboard.component.scss index 417bfaff..c9d72e03 100644 --- a/src/app/components/dashboard/dashboard.component.scss +++ b/src/app/components/dashboard/dashboard.component.scss @@ -17,6 +17,7 @@ */ .mat-card { + padding: 20px; margin-bottom: 20px; &:hover { cursor: pointer; diff --git a/src/app/components/nodes-view/nodes-view.component.html b/src/app/components/nodes-view/nodes-view.component.html new file mode 100644 index 00000000..c1701f3e --- /dev/null +++ b/src/app/components/nodes-view/nodes-view.component.html @@ -0,0 +1,101 @@ + + +
+
+ + + {{ columnDef.colName }} + {{ element[columnDef.colId] || 'n/a' }} + + + + + + + + + + + + + + +
No records found
+
+
+ + + + + + +
+ + +
+ +
+

Allocations

+ + +
+ + + {{ columnDef.colName }} + {{ element[columnDef.colId] || 'n/a' }} + + + + +
No records found
+
+
+ + + + + + +
+ + +
+
+
diff --git a/src/app/components/nodes-view/nodes-view.component.scss b/src/app/components/nodes-view/nodes-view.component.scss new file mode 100644 index 00000000..4f75dfed --- /dev/null +++ b/src/app/components/nodes-view/nodes-view.component.scss @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.nodes-view { + width: 100%; + height: 100%; + padding: 25px; + .mat-header-cell { + font-size: 15px; + font-weight: bold; + color: #666; + } + .mat-cell { + color: #333; + } + .mat-row { + &:hover { + background: #cccccc; + cursor: pointer; + } + &.selected-row { + background: #303d54; + .mat-cell { + color: #fff; + } + } + } + .mat-header-cell.indicator-icon, + .mat-cell.indicator-icon { + max-width: 40px; + font-size: 18px; + margin-left: 10px; + } + .node-allocations { + margin-top: 40px; + .mat-table { + margin-top: 20px; + } + } + .no-record { + font-size: 14px; + font-weight: bold; + color: #666; + width: 100%; + text-align: center; + } +} diff --git a/src/app/components/nodes-view/nodes-view.component.spec.ts b/src/app/components/nodes-view/nodes-view.component.spec.ts new file mode 100644 index 00000000..db69dca1 --- /dev/null +++ b/src/app/components/nodes-view/nodes-view.component.spec.ts @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; +import { NoopAnimationsModule } from '@angular/platform-browser/animations'; +import { NgxSpinnerService } from 'ngx-spinner'; +import { configureTestSuite } from 'ng-bullet'; + +import { NodesViewComponent } from './nodes-view.component'; +import { + MatTableModule, + MatPaginatorModule, + MatDividerModule, + MatSortModule +} from '@angular/material'; +import { SchedulerService } from '@app/services/scheduler/scheduler.service'; +import { HAMMER_LOADER } from '@angular/platform-browser'; +import { MockSchedulerService, MockNgxSpinnerService } from '@app/testing/mocks'; + +describe('NodesViewComponent', () => { + let component: NodesViewComponent; + let fixture: ComponentFixture; + + configureTestSuite(() => { + TestBed.configureTestingModule({ + declarations: [NodesViewComponent], + imports: [ + NoopAnimationsModule, + MatTableModule, + MatPaginatorModule, + MatDividerModule, + MatSortModule + ], + providers: [ + { provide: SchedulerService, useValue: MockSchedulerService }, + { provide: NgxSpinnerService, useValue: MockNgxSpinnerService }, + { provide: HAMMER_LOADER, useValue: () => new Promise(() => {}) } + ] + }); + }); + + beforeEach(() => { + fixture = TestBed.createComponent(NodesViewComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create the component', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/app/components/nodes-view/nodes-view.component.ts b/src/app/components/nodes-view/nodes-view.component.ts new file mode 100644 index 00000000..b189d0f6 --- /dev/null +++ b/src/app/components/nodes-view/nodes-view.component.ts @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Component, OnInit, ViewChild } from '@angular/core'; +import { MatPaginator, MatSort, MatTableDataSource, PageEvent } from '@angular/material'; +import { NgxSpinnerService } from 'ngx-spinner'; + +import { SchedulerService } from '@app/services/scheduler/scheduler.service'; +import { NodeInfo } from '@app/models/node-info.model'; +import { AllocationInfo } from '@app/models/alloc-info.model'; +import { ColumnDef } from '@app/models/column-def.model'; +import { finalize } from 'rxjs/operators'; + +@Component({ + selector: 'app-nodes-view', + templateUrl: './nodes-view.component.html', + styleUrls: ['./nodes-view.component.scss'] +}) +export class NodesViewComponent implements OnInit { + @ViewChild('nodesViewMatPaginator', { static: true }) nodePaginator: MatPaginator; + @ViewChild('allocationMatPaginator', { static: true }) allocPaginator: MatPaginator; + @ViewChild(MatSort, { static: true }) nodeSort: MatSort; + + nodeDataSource = new MatTableDataSource([]); + nodeColumnDef: ColumnDef[] = []; + nodeColumnIds: string[] = []; + + allocDataSource = new MatTableDataSource([]); + allocColumnDef: ColumnDef[] = []; + allocColumnIds: string[] = []; + + selectedRow: NodeInfo | null = null; + + constructor(private scheduler: SchedulerService, private spinner: NgxSpinnerService) {} + + ngOnInit() { + this.nodeDataSource.paginator = this.nodePaginator; + this.allocDataSource.paginator = this.allocPaginator; + this.nodeDataSource.sort = this.nodeSort; + + this.nodeColumnDef = [ + { colId: 'nodeId', colName: 'Node ID' }, + { colId: 'hostName', colName: 'Host Name' }, + { colId: 'rackName', colName: 'Rack Name' }, + { colId: 'partitionName', colName: 'Partition Name' }, + { colId: 'capacity', colName: 'Capacity' }, + { colId: 'allocated', colName: 'Allocated' }, + { colId: 'available', colName: 'Available' } + ]; + + this.nodeColumnIds = this.nodeColumnDef.map(col => col.colId).concat('indicatorIcon'); + + this.allocColumnDef = [ + { colId: 'allocationKey', colName: 'Allocation Key' }, + { colId: 'resource', colName: 'Resource' }, + { colId: 'queueName', colName: 'Queue Name' }, + { colId: 'priority', colName: 'Priority' }, + { colId: 'partition', colName: 'Partition' }, + { colId: 'nodeId', colName: 'Node ID' }, + { colId: 'applicationId', colName: 'Application ID' } + ]; + + this.allocColumnIds = this.allocColumnDef.map(col => col.colId); + + this.spinner.show(); + this.scheduler + .fetchNodeList() + .pipe( + finalize(() => { + this.spinner.hide(); + }) + ) + .subscribe(data => { + this.nodeDataSource.data = data; + }); + } + + unselectAllRowsButOne(row: NodeInfo) { + this.nodeDataSource.data.map(node => { + if (node !== row) { + node.isSelected = false; + } + }); + } + + toggleRowSelection(row: NodeInfo) { + this.unselectAllRowsButOne(row); + if (row.isSelected) { + this.selectedRow = null; + row.isSelected = false; + this.allocDataSource.data = []; + } else { + this.selectedRow = row; + row.isSelected = true; + this.allocDataSource.data = row.allocations; + } + } + + onPaginatorChanged(page: PageEvent) { + if (this.selectedRow) { + this.selectedRow.isSelected = false; + this.selectedRow = null; + this.allocDataSource.data = []; + } + } + + isNodeDataSourceEmpty() { + return this.nodeDataSource.data && this.nodeDataSource.data.length === 0; + } + + isAllocDataSourceEmpty() { + return this.allocDataSource.data && this.allocDataSource.data.length === 0; + } +} diff --git a/src/app/components/queue-rack/queue-rack.component.ts b/src/app/components/queue-rack/queue-rack.component.ts index 7bb4f9a5..299aa6ab 100644 --- a/src/app/components/queue-rack/queue-rack.component.ts +++ b/src/app/components/queue-rack/queue-rack.component.ts @@ -19,6 +19,7 @@ import { Component, OnInit, Input, Output, EventEmitter } from '@angular/core'; import { QueueInfo, ToggleQueueChildrenEvent } from '@app/models/queue-info.model'; +import { NOT_AVAILABLE } from '@app/utils/constants'; @Component({ selector: 'app-queue-rack', @@ -61,7 +62,7 @@ export class QueueRackComponent implements OnInit { } getQueueCapacityColor(queue: QueueInfo) { - const absUsedCapacity = +queue.absoluteUsedCapacity; + const absUsedCapacity = this.getMaxAbsValue(queue.absoluteUsedCapacity); if (absUsedCapacity > 60 && absUsedCapacity <= 75) { return '#60cea5'; } else if (absUsedCapacity > 75 && absUsedCapacity < 90) { @@ -73,7 +74,22 @@ export class QueueRackComponent implements OnInit { } getProgressBarValue(queue: QueueInfo) { - const absUsedCapacity = +queue.absoluteUsedCapacity; + const absUsedCapacity = this.getMaxAbsValue(queue.absoluteUsedCapacity); return Math.min(absUsedCapacity, 100); } + + getMaxAbsValue(absCapacities: string): number { + let max = 0; + if (absCapacities !== null) { + const splitted = absCapacities + .replace(NOT_AVAILABLE, '0') + .replace(/[^:0-9]/g, '') + .split(':'); + if (splitted.length !== 0) { + const capacities: number[] = splitted.map(x => +x); + max = Math.max(...capacities); + } + } + return max; + } } diff --git a/src/app/components/queues-view/queues-view.component.html b/src/app/components/queues-view/queues-view.component.html index 2d7c5089..bcd58382 100644 --- a/src/app/components/queues-view/queues-view.component.html +++ b/src/app/components/queues-view/queues-view.component.html @@ -73,7 +73,7 @@
Absolute Used Capacity:
-
{{ selectedQueue.absoluteUsedCapacity }}%
+
{{ selectedQueue.absoluteUsedCapacity }}
diff --git a/src/app/components/queues-view/queues-view.component.scss b/src/app/components/queues-view/queues-view.component.scss index 836ca3ec..b7ca493e 100644 --- a/src/app/components/queues-view/queues-view.component.scss +++ b/src/app/components/queues-view/queues-view.component.scss @@ -45,7 +45,7 @@ cursor: pointer; padding-right: 5px; &:hover { - color: #ef6162; + color: #f44336; } } .header { diff --git a/src/app/models/alloc-info.model.ts b/src/app/models/alloc-info.model.ts new file mode 100644 index 00000000..e81116dd --- /dev/null +++ b/src/app/models/alloc-info.model.ts @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export class AllocationInfo { + constructor( + public allocationKey: string, + public allocationTags: string, + public uuid: string, + public resource: string, + public priority: string, + public queueName: string, + public nodeId: string, + public applicationId: string, + public partition: string + ) {} +} diff --git a/src/app/models/app-info.model.ts b/src/app/models/app-info.model.ts index 3c07578c..a8aa90a0 100644 --- a/src/app/models/app-info.model.ts +++ b/src/app/models/app-info.model.ts @@ -17,6 +17,7 @@ */ import * as moment from 'moment'; +import { AllocationInfo } from './alloc-info.model'; export class AppInfo { isSelected = false; @@ -26,8 +27,8 @@ export class AppInfo { public partition: string, public queueName: string, public submissionTime: number, - public allocations: AppAllocation[] | null, - public applicationState: string + public applicationState: string, + public allocations: AllocationInfo[] | null, ) {} get formattedSubmissionTime() { @@ -35,21 +36,7 @@ export class AppInfo { return moment(millisecs).format('YYYY/MM/DD HH:mm:ss'); } - setAllocations(allocs: AppAllocation[]) { + setAllocations(allocs: AllocationInfo[]) { this.allocations = allocs; } } - -export class AppAllocation { - constructor( - public allocationKey: string, - public allocationTags: string, - public uuid: string, - public resource: string, - public priority: string, - public queueName: string, - public nodeId: string, - public applicationId: string, - public partition: string - ) {} -} diff --git a/src/app/models/column-def.model.ts b/src/app/models/column-def.model.ts new file mode 100644 index 00000000..8e739fee --- /dev/null +++ b/src/app/models/column-def.model.ts @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface ColumnDef { + colId: string; + colName: string; +} diff --git a/src/app/models/node-info.model.ts b/src/app/models/node-info.model.ts new file mode 100644 index 00000000..67d7c348 --- /dev/null +++ b/src/app/models/node-info.model.ts @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AllocationInfo } from './alloc-info.model'; + +export class NodeInfo { + isSelected = false; + constructor( + public nodeId: string, + public hostName: string, + public rackName: string, + public partitionName: string, + public capacity: string, + public allocated: string, + public occupied: string, + public available: string, + public allocations: AllocationInfo[] | null + ) {} + + setAllocations(allocs: AllocationInfo[]) { + this.allocations = allocs; + } +} diff --git a/src/app/services/scheduler/scheduler.service.ts b/src/app/services/scheduler/scheduler.service.ts index 62b6673a..186026fe 100644 --- a/src/app/services/scheduler/scheduler.service.ts +++ b/src/app/services/scheduler/scheduler.service.ts @@ -26,8 +26,11 @@ import { EnvconfigService } from '../envconfig/envconfig.service'; import { ClusterInfo } from '@app/models/cluster-info.model'; import { CommonUtil } from '@app/utils/common.util'; import { ResourceInfo } from '@app/models/resource-info.model'; -import { AppInfo, AppAllocation } from '@app/models/app-info.model'; +import { AppInfo } from '@app/models/app-info.model'; +import { AllocationInfo } from '@app/models/alloc-info.model'; import { HistoryInfo } from '@app/models/history-info.model'; +import { NodeInfo } from '@app/models/node-info.model'; +import { NOT_AVAILABLE } from '@app/utils/constants'; @Injectable({ providedIn: 'root' @@ -53,8 +56,8 @@ export class SchedulerService { return this.httpClient.get(queuesUrl).pipe( map((data: any) => { let rootQueue = new QueueInfo(); - if (data && data.queues && data.queues[0]) { - const rootQueueData = data.queues[0]; + if (data && data.queues) { + const rootQueueData = data.queues; rootQueue.queueName = rootQueueData.queuename; rootQueue.state = rootQueueData.status || 'RUNNING'; rootQueue.children = null; @@ -78,25 +81,25 @@ export class SchedulerService { const result = []; if (data && data.length > 0) { data.forEach(app => { - const jobInfo = new AppInfo( + const appInfo = new AppInfo( app['applicationID'], - this.formatCapacity(this.splitCapacity(app['usedResource'])), + this.formatCapacity(this.splitCapacity(app['usedResource'], NOT_AVAILABLE)), app['partition'], app['queueName'], app['submissionTime'], - null, - app['applicationState'] + app['applicationState'], + [] ); const allocations = app['allocations']; if (allocations && allocations.length > 0) { const appAllocations = []; allocations.forEach(alloc => { appAllocations.push( - new AppAllocation( + new AllocationInfo( alloc['allocationKey'], alloc['allocationTags'], alloc['uuid'], - this.formatCapacity(this.splitCapacity(alloc['resource'])), + this.formatCapacity(this.splitCapacity(alloc['resource'], NOT_AVAILABLE)), alloc['priority'], alloc['queueName'], alloc['nodeId'], @@ -105,9 +108,9 @@ export class SchedulerService { ) ); }); - jobInfo.setAllocations(appAllocations); + appInfo.setAllocations(appAllocations); } - result.push(jobInfo); + result.push(appInfo); }); } return result; @@ -153,6 +156,63 @@ export class SchedulerService { ); } + public fetchNodeList(): Observable { + const nodesUrl = `${this.envConfig.getSchedulerWebAddress()}/ws/v1/nodes`; + + return this.httpClient.get(nodesUrl).pipe( + map((data: any) => { + const result = []; + + if (data && data.length > 0) { + for (const info of data) { + const nodesInfoData = info.nodesInfo || []; + + nodesInfoData.forEach(node => { + const nodeInfo = new NodeInfo( + node['nodeID'], + node['hostName'], + node['rackName'], + info['partitionName'], + this.formatCapacity(this.splitCapacity(node['capacity'], NOT_AVAILABLE)), + this.formatCapacity(this.splitCapacity(node['allocated'], NOT_AVAILABLE)), + this.formatCapacity(this.splitCapacity(node['occupied'], NOT_AVAILABLE)), + this.formatCapacity(this.splitCapacity(node['available'], NOT_AVAILABLE)), + [] + ); + + const allocations = node['allocations']; + if (allocations && allocations.length > 0) { + const appAllocations = []; + + allocations.forEach(alloc => { + appAllocations.push( + new AllocationInfo( + alloc['allocationKey'], + alloc['allocationTags'], + alloc['uuid'], + this.formatCapacity(this.splitCapacity(alloc['resource'], NOT_AVAILABLE)), + alloc['priority'], + alloc['queueName'], + alloc['nodeId'], + alloc['applicationId'], + alloc['partition'] + ) + ); + }); + + nodeInfo.setAllocations(appAllocations); + } + + result.push(nodeInfo); + }); + } + } + + return result; + }) + ); + } + private generateQueuesTree(data: any, currentQueue: QueueInfo) { if (data && data.queues && data.queues.length > 0) { const chilrenQs = []; @@ -179,34 +239,35 @@ export class SchedulerService { const maxCap = data['capacities']['maxcapacity'] as string; const absUsedCapacity = data['capacities']['absusedcapacity'] as string; - const configCapResources = this.splitCapacity(configCap); - const usedCapResources = this.splitCapacity(usedCap); - const maxCapResources = this.splitCapacity(maxCap); + const configCapResources = this.splitCapacity(configCap, NOT_AVAILABLE); + const usedCapResources = this.splitCapacity(usedCap, NOT_AVAILABLE); + const maxCapResources = this.splitCapacity(maxCap, NOT_AVAILABLE); + const absUsedCapacityResources = this.splitCapacity(absUsedCapacity, NOT_AVAILABLE); queue.capacity = this.formatCapacity(configCapResources); queue.maxCapacity = this.formatCapacity(maxCapResources); queue.usedCapacity = this.formatCapacity(usedCapResources); - queue.absoluteUsedCapacity = absUsedCapacity ? absUsedCapacity : '0'; + queue.absoluteUsedCapacity = this.formatAbsCapacity(absUsedCapacityResources); } - private splitCapacity(capacity: string = ''): ResourceInfo { + private splitCapacity(capacity: string = '', defaultValue: string): ResourceInfo { const splitted = capacity .replace('map', '') .replace(/[\[\]]/g, '') .split(' '); const resources: ResourceInfo = { - memory: '0', - vcore: '0' + memory: defaultValue, + vcore: defaultValue }; for (const resource of splitted) { if (resource) { const values = resource.split(':'); - if (values[0] === 'memory') { + if (values[0] === 'memory' && values[1] !== '') { resources.memory = values[1]; } - if (values[0] === 'vcore') { + if (values[0] === 'vcore' && values[1] !== '') { resources.vcore = values[1]; } } @@ -217,8 +278,19 @@ export class SchedulerService { private formatCapacity(resourceInfo: ResourceInfo) { const formatted = []; - formatted.push(`[memory: ${CommonUtil.formatMemory(+resourceInfo.memory)}`); + if (resourceInfo.memory !== NOT_AVAILABLE) { + formatted.push(`[memory: ${CommonUtil.formatMemory(+resourceInfo.memory)}`); + } else { + formatted.push(`[memory: ${resourceInfo.memory}`); + } formatted.push(`vcore: ${resourceInfo.vcore}]`); return formatted.join(', '); } + + private formatAbsCapacity(resourceInfo: ResourceInfo) { + const formatted = []; + formatted.push(`[memory: ${resourceInfo.memory}%`); + formatted.push(`vcore: ${resourceInfo.vcore}%]`); + return formatted.join(', '); + } } diff --git a/src/app/testing/mocks.ts b/src/app/testing/mocks.ts index 9c3b3470..cf3510c8 100644 --- a/src/app/testing/mocks.ts +++ b/src/app/testing/mocks.ts @@ -26,7 +26,8 @@ export const MockSchedulerService = { fetchSchedulerQueues: () => of({}), fetchAppList: () => of([]), fetchAppHistory: () => of([]), - fetchContainerHistory: () => of([]) + fetchContainerHistory: () => of([]), + fetchNodeList: () => of([]) }; export const MockNgxSpinnerService = { diff --git a/src/app/utils/constants.ts b/src/app/utils/constants.ts index bc2a7d5c..4a416b72 100644 --- a/src/app/utils/constants.ts +++ b/src/app/utils/constants.ts @@ -18,3 +18,4 @@ export const DEFAULT_PARTITION_VALUE = ''; export const DEFAULT_PROTOCOL = 'http:'; +export const NOT_AVAILABLE = 'n/a'; diff --git a/src/styles.scss b/src/styles.scss index a856c8c8..639bfeb9 100644 --- a/src/styles.scss +++ b/src/styles.scss @@ -233,3 +233,8 @@ p { padding: 10px; } } + +.white-mat-form-field .mat-form-field-flex { + background-color: #fff; + padding: 0 5px 6px 5px; +}