Merge pull request #1312 from stevenc987/main

NR Duplicates Report#14287

.github/workflows/nr-duplicates-report-cd.yml

@@ -0,0 +1,103 @@
+name: Namex Duplicates Report Job CD
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "jobs/nr-duplicates-report/**"
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: "Environment (dev/test/prod)"
+        required: true
+        default: "dev"
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./jobs/nr-duplicates-report
+
+env:
+  APP_NAME: "nr-duplicates-report"
+  TAG_NAME: "dev"
+
+jobs:
+  nr-duplicates-report-cd-by-push:
+    runs-on: ubuntu-20.04
+
+    if: github.event_name == 'push' && github.repository == 'bcgov/namex'
+    environment:
+      name: "dev"
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Login Openshift
+        shell: bash
+        run: |
+          oc login --server=${{secrets.OPENSHIFT4_LOGIN_REGISTRY}} --token=${{secrets.OPENSHIFT4_SA_TOKEN}}
+
+      - name: CD Flow
+        shell: bash
+        env:
+          OPS_REPOSITORY: ${{ secrets.OPS_REPOSITORY }}
+          OPENSHIFT_DOCKER_REGISTRY: ${{ secrets.OPENSHIFT4_DOCKER_REGISTRY }}
+          OPENSHIFT_SA_NAME: ${{ secrets.OPENSHIFT4_SA_NAME }}
+          OPENSHIFT_SA_TOKEN: ${{ secrets.OPENSHIFT4_SA_TOKEN }}
+          OPENSHIFT_REPOSITORY: ${{ secrets.OPENSHIFT4_REPOSITORY }}
+          TAG_NAME: ${{ env.TAG_NAME }}
+        run: |
+          make cd
+
+      - name: Rocket.Chat Notification
+        uses: RocketChat/Rocket.Chat.GitHub.Action.Notification@master
+        if: failure()
+        with:
+          type: ${{ job.status }}
+          job_name: "*NR Duplicates Report Job Built and Deployed to ${{env.TAG_NAME}}*"
+          channel: "#registries-bot"
+          url: ${{ secrets.ROCKETCHAT_WEBHOOK }}
+          commit: true
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+  nr-duplicates-report-cd-by-dispatch:
+    runs-on: ubuntu-20.04
+
+    if: github.event_name == 'workflow_dispatch' && github.repository == 'bcgov/namex'
+    environment:
+      name: "${{ github.event.inputs.environment }}"
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set env by input
+        run: |
+          echo "TAG_NAME=${{ github.event.inputs.environment }}" >> $GITHUB_ENV
+
+      - name: Login Openshift
+        shell: bash
+        run: |
+          oc login --server=${{secrets.OPENSHIFT4_LOGIN_REGISTRY}} --token=${{secrets.OPENSHIFT4_SA_TOKEN}}
+
+      - name: CD Flow
+        shell: bash
+        env:
+          OPS_REPOSITORY: ${{ secrets.OPS_REPOSITORY }}
+          OPENSHIFT_DOCKER_REGISTRY: ${{ secrets.OPENSHIFT4_DOCKER_REGISTRY }}
+          OPENSHIFT_SA_NAME: ${{ secrets.OPENSHIFT4_SA_NAME }}
+          OPENSHIFT_SA_TOKEN: ${{ secrets.OPENSHIFT4_SA_TOKEN }}
+          OPENSHIFT_REPOSITORY: ${{ secrets.OPENSHIFT4_REPOSITORY }}
+          TAG_NAME: ${{ env.TAG_NAME }}
+        run: |
+          make cd
+
+      - name: Rocket.Chat Notification
+        uses: RocketChat/Rocket.Chat.GitHub.Action.Notification@master
+        if: failure()
+        with:
+          type: ${{ job.status }}
+          job_name: "*NR Duplicates Report Job Built and Deployed to ${{env.TAG_NAME}}*"
+          channel: "#registries-bot"
+          url: ${{ secrets.ROCKETCHAT_WEBHOOK }}
+          commit: true
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/nr-duplicates-report-ci.yml

@@ -0,0 +1,90 @@
+name: NR Duplicates Report Job CI
+
+on:
+  pull_request:
+    types: [assigned, synchronize]
+    paths:
+      - "jobs/nr-duplicates-report/**"
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./jobs/nr-duplicates-report
+
+jobs:
+  setup-job:
+    runs-on: ubuntu-20.04
+
+    if: github.repository == 'bcgov/namex'
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: "true"
+
+  linting:
+    needs: setup-job
+    runs-on: ubuntu-20.04
+
+    strategy:
+      matrix:
+        python-version: [3.8]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          make setup
+      - name: Lint with pylint
+        id: pylint
+        run: |
+          make pylint
+      - name: Lint with flake8
+        id: flake8
+        run: |
+          make flake8
+
+  testing:
+    needs: setup-job
+    env:
+      PG_USER: postgres
+      PG_PASSWORD: postgres
+      PG_DB_NAME: postgres
+      PG_HOST: localhost
+      PG_PORT: 5432      
+
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          make setup
+      - name: Test with pytest
+        id: test
+        run: |
+      #    make test
+      #- name: Upload coverage to Codecov
+      #  uses: codecov/codecov-action@v1
+      #  with:
+      #    file: ./queue_services/entity-pay/coverage.xml
+      #    flags: entitypay
+      #    name: codecov-entity-pay
+      #    fail_ci_if_error: true
+
+  build-check:
+    needs: setup-job
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: build to check strictness
+        id: build
+        run: |
+          make build-nc

jobs/nr-duplicates-report/Dockerfile

@@ -0,0 +1,35 @@
+FROM python:3.8.5-buster
+
+ARG VCS_REF="missing"
+ARG BUILD_DATE="missing"
+
+ENV VCS_REF=${VCS_REF}
+ENV BUILD_DATE=${BUILD_DATE}
+
+LABEL org.label-schema.vcs-ref=${VCS_REF} \
+    org.label-schema.build-date=${BUILD_DATE}
+
+USER root
+
+# Create working directory
+RUN mkdir /opt/app-root && chmod 755 /opt/app-root
+RUN mkdir /opt/app-root/data && chmod 777 /opt/app-root/data
+WORKDIR /opt/app-root
+
+# Install the requirements
+COPY ./requirements.txt .
+
+#RUN pip install --upgrade pip
+RUN pip install pip==20.3.3
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+USER 1001
+
+# Set Python path
+ENV PYTHONPATH=/opt/app-root/src
+
+EXPOSE 8080
+
+CMD [ "python", "/opt/app-root/notebookreport.py" ]

jobs/nr-duplicates-report/Makefile

@@ -0,0 +1,150 @@
+.PHONY: license
+.PHONY: setup
+.PHONY: ci cd
+.PHONY: db run
+
+MKFILE_PATH:=$(abspath $(lastword $(MAKEFILE_LIST)))
+CURRENT_ABS_DIR:=$(patsubst %/,%,$(dir $(MKFILE_PATH)))
+
+PROJECT_NAME:=nr-duplicates-report
+DOCKER_NAME:=nr-duplicates-report
+
+#################################################################################
+# COMMANDS -- license                                                           #
+#################################################################################
+license: ## Verify source code license headers.
+	./scripts/verify_license_headers.sh $(CURRENT_ABS_DIR)/src $(CURRENT_ABS_DIR)/tests
+
+#################################################################################
+# COMMANDS -- Setup                                                             #
+#################################################################################
+setup: install install-dev ## Setup the project
+
+clean: clean-build clean-pyc clean-test ## Clean the project
+	rm -rf venv/
+
+clean-build: ## Clean build files
+	rm -fr build/
+	rm -fr dist/
+	rm -fr .eggs/
+	find . -name '*.egg-info' -exec rm -fr {} +
+	find . -name '*.egg' -exec rm -fr {} +
+
+clean-pyc: ## Clean cache files
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	find . -name '__pycache__' -exec rm -fr {} +
+
+clean-test: ## clean test files
+	find . -name '.pytest_cache' -exec rm -fr {} +
+	rm -fr .tox/
+	rm -f .coverage
+	rm -fr htmlcov/
+
+build-req: clean ## Upgrade requirements
+	test -f venv/bin/activate || python3.8 -m venv  $(CURRENT_ABS_DIR)/venv ;\
+	. venv/bin/activate ;\
+	pip install pip==21.1.2 ;\
+	pip install -Ur requirements/prod.txt ;\
+	pip freeze | sort > requirements.txt
+
+install: clean ## Install python virtrual environment
+	test -f venv/bin/activate || python3.8 -m venv  $(CURRENT_ABS_DIR)/venv ;\
+	. venv/bin/activate ;\
+	pip install pip==21.1.2 ;\
+	pip install -Ur requirements.txt
+
+install-dev: ## Install local application
+	. venv/bin/activate ; \
+	pip install -Ur requirements/dev.txt; \
+	pip install -e .
+
+#################################################################################
+# COMMANDS - CI                                                                 #
+#################################################################################
+ci: pylint flake8 test ## CI flow
+
+pylint: ## Linting with pylint
+	. venv/bin/activate && pylint --rcfile=setup.cfg  notebookreport.py
+
+flake8: ## Linting with flake8
+	. venv/bin/activate && flake8 notebookreport.py
+
+lint: pylint flake8 ## run all lint type scripts
+
+test: ## Unit testing
+	. venv/bin/activate && pytest
+
+mac-cov: local-test ## Run the coverage report and display in a browser window (mac)
+	open -a "Google Chrome" htmlcov/index.html
+
+#################################################################################
+# COMMANDS - CD
+# expects the terminal to be docker login
+# expects export OPENSHIFT_DOCKER_REGISTRY=""
+# expects export OPENSHIFT_SA_NAME="$(oc whoami)"
+# expects export OPENSHIFT_SA_TOKEN="$(oc whoami -t)"
+# expects export OPENSHIFT_REPOSITORY=""
+# expects export TAG_NAME="dev/test/prod"
+# expects export OPS_REPOSITORY=""                                                        #
+#################################################################################
+cd: ## CD flow
+ifeq ($(TAG_NAME), test)
+cd: update-env
+	oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):dev $(DOCKER_NAME):$(TAG_NAME)
+else ifeq ($(TAG_NAME), prod)
+cd: update-env
+	oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):$(TAG_NAME) $(DOCKER_NAME):$(TAG_NAME)-$(shell date +%F)
+	oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):test $(DOCKER_NAME):$(TAG_NAME)
+else
+TAG_NAME=dev
+cd: build update-env tag
+endif
+
+build: ## Build the docker container
+	docker build . -t $(DOCKER_NAME) \
+		--build-arg VCS_REF=$(shell git rev-parse --short HEAD) \
+		--build-arg BUILD_DATE=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") \
+
+build-nc: ## Build the docker container without caching
+	docker build --no-cache -t $(DOCKER_NAME) .
+
+REGISTRY_IMAGE=$(OPENSHIFT_DOCKER_REGISTRY)/$(OPENSHIFT_REPOSITORY)-tools/$(DOCKER_NAME)
+push: #build ## Push the docker container to the registry & tag latest
+	@echo "$(OPENSHIFT_SA_TOKEN)" | docker login $(OPENSHIFT_DOCKER_REGISTRY) -u $(OPENSHIFT_SA_NAME) --password-stdin ;\
+    docker tag $(DOCKER_NAME) $(REGISTRY_IMAGE):latest ;\
+    docker push $(REGISTRY_IMAGE):latest
+
+VAULTS=`cat devops/vaults.json`
+update-env: ## Update env from 1pass
+	oc -n "$(OPS_REPOSITORY)-$(TAG_NAME)" exec "dc/vault-service-$(TAG_NAME)" -- ./scripts/1pass.sh \
+		-m "secret" \
+		-e "$(TAG_NAME)" \
+		-a "$(DOCKER_NAME)-$(TAG_NAME)" \
+		-n "$(OPENSHIFT_REPOSITORY)-$(TAG_NAME)" \
+		-v "$(VAULTS)" \
+		-r "false" \
+		-f "false"
+
+tag: push ## tag image
+	oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):latest $(DOCKER_NAME):$(TAG_NAME)
+
+#################################################################################
+# COMMANDS - Local                                                              #
+#################################################################################
+run: db ## Run the project in local
+	. venv/bin/activate && python -m flask run -p 5000
+
+db: ## Update the local database
+	. venv/bin/activate && python -m manage.py db upgrade
+
+#################################################################################
+# Self Documenting Commands                                                     #
+#################################################################################
+.PHONY: help
+
+.DEFAULT_GOAL := help
+
+help:
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'