From a3dcea8cb04d5a45fbab36d176742f55e05ce13d Mon Sep 17 00:00:00 2001 From: Dennis Periquet Date: Mon, 18 Nov 2024 14:28:12 -0500 Subject: [PATCH] ART-10508: Add endpoint to automate PR create for new images (#150) * Add endpoint to automate PR and Jira creation for new image * Add Makefile and ssh to Dockerfile to enable vscode debug * move git user to the .env files --- .gitignore | 1 + Dockerfile.update | 12 +++ Makefile | 122 ++++++++++++++++++++++ README.md | 94 ++++++++++++++++- api/urls.py | 1 + api/views.py | 253 ++++++++++++++++++++++++++++++++++++++++++++++ conf/dev.env | 3 + conf/prod.env | 3 + 8 files changed, 486 insertions(+), 3 deletions(-) create mode 100644 Makefile diff --git a/.gitignore b/.gitignore index 853807c..783ee85 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ __pycache__/ venv/ .idea/ .aws/ +launch.json diff --git a/Dockerfile.update b/Dockerfile.update index 0c0d24f..a4c626e 100644 --- a/Dockerfile.update +++ b/Dockerfile.update @@ -15,10 +15,22 @@ USER 0 COPY container/doozer-settings.yaml /home/"$USERNAME"/.config/doozer/settings.yaml COPY container/elliott-settings.yaml /home/"$USERNAME"/.config/elliott/settings.yaml +# If you want to run with an ssh server (for debugging in vscode), uncomment these four lines +# RUN dnf install -y openssh-server && \ +# ssh-keygen -A && \ +# sed -i 's/#Port 22/Port 22/' /etc/ssh/sshd_config && \ +# sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/' /etc/ssh/sshd_config + # Switching back to default user USER "$USER_UID" WORKDIR /workspaces/art-dash + +# If you want to run with an ssh server (for debugging in vscode), uncomment these three lines after adding a the launch.json +# RUN mkdir .vscode +# RUN sudo chmod a+rw .vscode +# COPY launch.json .vscode + # Upadate pip RUN python3 -m pip install --upgrade pip COPY requirements.txt ./ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..107a9ba --- /dev/null +++ b/Makefile @@ -0,0 +1,122 @@ +OPENSHIFT_DEV_DIR=$(HOME)/tmp/art-ui +GIT_TOKEN_FILE=$(OPENSHIFT_DEV_DIR)/git_token +NETWORK_NAME=art-dashboard-network +MARIADB_CONTAINER_NAME=mariadb +MARIADB_IMAGE=docker.io/library/mariadb:10.6.14 +MARIADB_ROOT_PASSWORD=secret +MARIADB_DATABASE=doozer_build + +ART_DASHBOARD_SERVER_DIR=$(OPENSHIFT_DEV_DIR)/art-dashboard-server +ART_TOOLS_DIR=$(OPENSHIFT_DEV_DIR)/art-tools +TEST_URL=http://localhost:8080/api/v1/test + +# Build the development environment base image +.PHONY: build-dev-base +build-dev-base: + podman build -f Dockerfile.base -t art-dash-server:base --build-arg USERNAME=$(USER) --build-arg USER_UID=1000 . + +# Build the development environment update image +.PHONY: build-dev +build-dev: + podman build -f Dockerfile.update -t art-dash-server:latest --build-arg USERNAME=$(USER) --build-arg USER_UID=1000 . + +# If the user wants to use their own .ssh directory, they need to copy it +.PHONY: setup-dev-env +setup-dev-env: check-network check-mariadb clone-repos + +# Check if the Podman network exists, create if it doesn't +.PHONY: check-network +check-network: + @if ! podman network exists $(NETWORK_NAME); then \ + echo "Creating Podman network $(NETWORK_NAME)"; \ + podman network create $(NETWORK_NAME); \ + else \ + echo "Podman network $(NETWORK_NAME) already exists"; \ + fi + +# Check if the MariaDB container is running, start if it's not; if already +# there but stopped, start it. +.PHONY: check-mariadb +check-mariadb: + @if ! podman ps --format "{{.Names}}" | grep -w $(MARIADB_CONTAINER_NAME) > /dev/null; then \ + if podman ps -a --format "{{.Names}}" | grep -w $(MARIADB_CONTAINER_NAME) > /dev/null; then \ + echo "MariaDB container exists but is stopped. Starting it..."; \ + podman start $(MARIADB_CONTAINER_NAME); \ + else \ + echo "Starting a new MariaDB container"; \ + podman run --net $(NETWORK_NAME) --name $(MARIADB_CONTAINER_NAME) \ + -e MARIADB_ROOT_PASSWORD=$(MARIADB_ROOT_PASSWORD) \ + -e MARIADB_DATABASE=$(MARIADB_DATABASE) \ + -d $(MARIADB_IMAGE); \ + fi \ + else \ + echo "MariaDB container $(MARIADB_CONTAINER_NAME) already running"; \ + fi + +# Create the art-dash database in the MariaDB container. +create-db: check-mariadb + sleep 4 + podman exec $(MARIADB_CONTAINER_NAME) mysql -uroot -psecret -e "CREATE DATABASE IF NOT EXISTS art_dash;" + +# Make some local dirs to share with the ART-ui server container +# Clone repositories if they don't exist +.PHONY: clone-repos +clone-repos: + mkdir -p $(OPENSHIFT_DEV_DIR)/.git + mkdir -p $(OPENSHIFT_DEV_DIR)/.docker + mkdir -p $(OPENSHIFT_DEV_DIR)/.ssh + cd $(OPENSHIFT_DEV_DIR) + touch $(OPENSHIFT_DEV_DIR)/.git/.gitconfig + touch $(OPENSHIFT_DEV_DIR)/.docker/config.json + @if [ ! -d $(ART_DASHBOARD_SERVER_DIR) ]; then \ + echo "Cloning art-dashboard-server"; \ + git clone https://github.com/openshift-eng/art-dashboard-server.git $(ART_DASHBOARD_SERVER_DIR); \ + fi + @if [ ! -d $(ART_TOOLS_DIR) ]; then \ + echo "Cloning art-tools"; \ + git clone https://github.com/openshift-eng/art-tools.git $(ART_TOOLS_DIR); \ + fi + +EXTRA_ARGS = + +ifeq ($(DEBUG_MODE),1) + EXTRA_ARGS = bash -c "sudo /usr/sbin/sshd -D -e" +endif + +# Run the development environment +# Run like this if you want to debug with vscode across ssh: 'make run-dev DEBUG_MODE=1' + +.PHONY: run-dev +run-dev: + cd $(OPENSHIFT_DEV_DIR) + @if [ ! -f $(GIT_TOKEN_FILE) ]; then \ + echo "Error: GitHub token file not found."; \ + exit 1; \ + fi + podman run --privileged -it --name dj1 --rm -p 8080:8080 -p 5678:5678 -p 3022:22 --net $(NETWORK_NAME) \ + -v "$(ART_TOOLS_DIR)/doozer/":/workspaces/doozer/:cached,z \ + -v "$(ART_TOOLS_DIR)/elliott/":/workspaces/elliott/:cached,z \ + -v $(OPENSHIFT_DEV_DIR)/.ssh:/home/$(USER)/.ssh:ro,cached,z \ + -v $(OPENSHIFT_DEV_DIR)/.docker/config.json:/home/$(USER)/.docker/config.json:ro,cached,z \ + -v $(OPENSHIFT_DEV_DIR)/.git/.gitconfig:/home/$(USER)/.gitconfig:ro,cached,z \ + -e RUN_ENV=development \ + -e GITHUB_PERSONAL_ACCESS_TOKEN=$$(cat $(GIT_TOKEN_FILE)) \ + art-dash-server:latest $(EXTRA_ARGS) + +# Test if the server is running by checking the response of curl to the API +.PHONY: dev-test +dev-test: + @curl -s -o /dev/null -w "%{http_code}" $(TEST_URL) | grep -q 200 && \ + echo "dev environment is working" || echo "dev environment is not working" + +# Clean up development environment by stopping and removing containers and network +.PHONY: clean-dev +clean-dev: + @if podman ps -a --format "{{.Names}}" | grep -w $(MARIADB_CONTAINER_NAME) > /dev/null; then \ + echo "Stopping and removing MariaDB container"; \ + podman stop $(MARIADB_CONTAINER_NAME) && podman rm $(MARIADB_CONTAINER_NAME); \ + fi + @if podman network exists $(NETWORK_NAME); then \ + echo "Removing Podman network $(NETWORK_NAME)"; \ + podman network rm $(NETWORK_NAME); \ + fi diff --git a/README.md b/README.md index d0d769a..9b445f9 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,21 @@ you can build it from scratch ``` # only required once unless RPM reqs change +# requires access to certs.corp.redhat.com (which is only accessible on the Redhat network) podman build -f Dockerfile.base -t art-dash-server:base --build-arg USERNAME=$USER --build-arg USER_UID=$(id -u) . # repeat this to update the app as it changes podman build -f Dockerfile.update -t art-dash-server:latest --build-arg USERNAME=$USER --build-arg USER_UID=$(id -u) . ``` + +If you get an error like this when running the `podman build`: + +``` +useradd warning: 's uid 4205753 outside of the UID_MIN 1000 and UID_MAX 60000 range. +``` + +Change `$(id -u)` above to something within that range (e.g., 1000). + or you can get the build from [cluster](https://console-openshift-console.apps.artc2023.pc3z.p1.openshiftapps.com/k8s/ns/art-dashboard-server/imagestreams/art-dash-server) ``` # after you log in to the cluster on CLI @@ -30,12 +40,15 @@ podman network create art-dashboard-network ## 3. Setup local database +During development, if your modifications perform or depend on any DB operations, you'll need to +populate the database; if not, skip to the section that just creates the database. + Start the local DB server using a specific version of MariaDB (10.6.14), as the latest version doesn't include MySQL. ``` podman run --net art-dashboard-network --name mariadb -e MARIADB_ROOT_PASSWORD=secret -e MARIADB_DATABASE=doozer_build -d docker.io/library/mariadb:10.6.14 ``` -Download the test database as `test.sql`. +Download the test database as `test.sql`. ``` # Log in to OpenShift CLI and switch to the art-db project oc login @@ -62,10 +75,12 @@ Import db into the mariadb container podman cp test.sql mariadb:/test.sql podman exec -ti mariadb /bin/bash -# Inside the container +# Inside the container, create the database mysql -uroot -psecret CREATE DATABASE art_dash; exit + +# Do this if you need to import data. mysql -uroot -psecret art_dash < test.sql ``` Password is `secret` as defined in the podman run command. @@ -74,7 +89,11 @@ Password is `secret` as defined in the podman run command. ## 4. Run container ``` -OPENSHIFT=$HOME/ART-dash # create a workspace, clone art-tools and art-dash to this location. +# create a workspace, git clone art-tools and art-dashoard-server repos in this location. +OPENSHIFT=$HOME/ART-dash +cd $OPENSHIFT +git clone https://github.com/openshift-eng/art-dashboard-server.git +git clone https://github.com/openshift-eng/art-tools.git podman run -it --rm -p 8080:8080 --net art-dashboard-network \ -v "$OPENSHIFT/art-dashboard-server":/workspaces/art-dash:cached,z \ @@ -104,6 +123,9 @@ Referrer-Policy: same-origin Cross-Origin-Opener-Policy: same-origin ``` + +NOTE: If you want to run the server on a specific host, add it to the `ALLOWED_HOSTS` list in [settings.py](https://github.com/openshift-eng/art-dashboard-server/blob/00e65d2dfd13207ead5fa856a66aff164febf077/build_interface/settings.py#L101). For example, to run on `192.168.1.100`, add `192.168.1.100`; to run on `myhost.com`, add `myhost.com`. If you forget to do that, a curl like this: `curl -i http://myhost.com:8080...` will result in a `400 Bad Request`. + To stop `art-dash-server:latest`, use `Ctrl-C` To stop mariadb server, run `podman stop mariadb` @@ -116,3 +138,69 @@ To stop mariadb server, run `podman stop mariadb` - Environment variables that is common to both development and production should be defined in `conf/common.env`. The variables in that file is loaded first and then the ones on `prod.env` or `dev.env` depending on the environment. - It's recommended to set up a kerberos config file similar to the one in this project so that you can easily mount your keytab as shown above. Otherwise, you'll have to `kinit` inside the container everytime. Please make modifications to the volume mount command to reflect the keytab format in your local. - If an error like `failed to export image: failed to create image: failed to get layer` shows up during container build, re-run the command again. + +## Building using the Makefile + +You can use the [Makefile](./Makefile) to help you build and debug. Here are the commands supported in the Makefile: + +* Create the base image: `make build-dev-base` +* Create the update image: `make build-dev` +* Setup the develoment environment: `make setup-dev-env` + * Once this is done, create a file called `git_token` in `$(HOME)/tmp/art-ui` +* Create the art-dash database: `make create-db` +* Run the container: `make run-dev` + * For debug mode, use: `make run-dev DEBUG_MODE=1` + +### Debugging with vscode and ssh + +To run the python debugger with vscode for the container, you'll need these: + +* ability to login to the container via ssh without a password + * for example, modify your ~/.ssh/config to do this + ``` + Host art1 + Hostname 127.0.0.1 + StrictHostKeyChecking no + Port 3022 + IdentityFile ~/tmp/art_ui/.ssh/id_rsa + User dperique + ``` + + * Add your ssh private key and `authorized_keys` file (containing the corresponding ssh public key) to the .ssh subdir in your development environment (~/tmp/art-ui/.ssh): + + ```bash + cd ~/tmp/arti-ui + cp ~/.ssh/id_rsa ./.ssh + cat ~/.ssh/id_rsa.pub >> ./.ssh/authorized_keys + ``` + +* vscode with python debugging plugins installed + * install the "Python Debugger extension using debugpy" from Microsoft (if you try to debug, vscode will ask you if you want to install this) +* this .vscode/launch.json (modify it to use your own email address, github token and jira token) + +```json +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: Django", + "type": "debugpy", + "request": "launch", + "args": [ + "runserver", + "0.0.0.0:8080", + "--noreload" + ], + "env": { + "RUN_ENV": "development", + "GITHUB_PERSONAL_ACCESS_TOKEN": "", + "JIRA_EMAIL": "", + "JIRA_TOKEN": "" + }, + "django": true, + "autoStartBrowser": false, + "program": "${workspaceFolder}/manage.py" + } + ] +} +``` diff --git a/api/urls.py b/api/urls.py index 72bd3aa..8022111 100644 --- a/api/urls.py +++ b/api/urls.py @@ -11,6 +11,7 @@ re_path('ga-version', views.ga_version), re_path('branch/', views.branch_data, name='branch_data_view'), re_path('test', views.test), + re_path('git_jira_api', views.git_jira_api), re_path('rpms_images_fetcher', views.rpms_images_fetcher_view), re_path('login', views.login_view, name='login'), re_path('check_auth', views.check_auth, name='check_auth') diff --git a/api/views.py b/api/views.py index 943d266..3f630ed 100644 --- a/api/views.py +++ b/api/views.py @@ -11,10 +11,15 @@ from . import request_dispatcher from .serializer import BuildSerializer import django_filters +from github import Github, GithubException +from jira import JIRA import json import re import os import jwt +import time +import uuid +import requests from datetime import datetime, timedelta from build_interface.settings import SECRET_KEY, SESSION_COOKIE_DOMAIN, JWTAuthentication @@ -177,6 +182,254 @@ def test(request): }, status=200) +@api_view(["GET"]) +def git_jira_api(request): + + TEST_ART_JIRA = "TEST-ART-999" + + file_content = request.query_params.get('file_content', None) + image_name = request.query_params.get('image_name', None) + release_for_image = request.query_params.get('release_for_image', None) + jira_summary = request.query_params.get('jira_summary', None) + jira_description = request.query_params.get('jira_description', None) + jira_project_id = request.query_params.get('jira_project_id', None) + jira_story_type_id = request.query_params.get('jira_story_type_id', None) + jira_component = request.query_params.get('jira_component', None) + jira_priority = request.query_params.get('jira_priority', None) + + git_test_mode_value = request.query_params.get('git_test_mode', None) + jira_test_mode_value = request.query_params.get('jira_test_mode', None) + + # extract the host from the request. + host = request.get_host() + + if not all([file_content, release_for_image, image_name, jira_summary, jira_description, jira_project_id, jira_story_type_id, jira_component, jira_priority]): + # These are all required. If any are missing, return an error and + # list what the user passed in. + return Response({ + "status": "failure", + "error": "Missing required parameters", + "parameters": { + "file_content": file_content, + "image_name": image_name, + "release_for_image": release_for_image, + "jira_summary": jira_summary, + "jira_description": jira_description, + "jira_project_id": jira_project_id, + "jira_story_type_id": jira_story_type_id, + "jira_component": jira_component, + "jira_priority": jira_priority + } + }, status=400) + + # Test mode is the default (e.g., when not specified) to force being intentional + # about actually creating the PR and Jira. + git_test_mode = False + jira_test_mode = False + if not git_test_mode_value or 'true' in git_test_mode_value.lower(): + git_test_mode = True + if not jira_test_mode_value or 'true' in jira_test_mode_value.lower(): + jira_test_mode = True + + git_user = os.getenv("GIT_USER") + if not git_user: + return Response({ + "status": "failure", + "error": "git user not in GIT_USER environment variable" + }, status=500) + + if git_test_mode: + # Just create a success status and fake PR without using the git API + pr_status = { + "status": "success", + "payload": f"{host}: Fake PR created successfully", + "pr_url": f"https://github.com/{git_user}/ocp-build-data/pull/10" + } + else: + try: + # Load the git token from an environment variable, later we can update the deployment + # to get the token from a Kubernetes environment variable sourced from a secret. + github_token = os.getenv("GITHUB_PERSONAL_ACCESS_TOKEN") + if not github_token: + return Response({ + "status": "failure", + "error": "git token not in GITHUB_PERSONAL_ACCESS_TOKEN environment variable" + }, status=500) + + git_object = Github(github_token) + + def make_github_request(func, *args, **kwargs): + """ + This function applies retry logic (with exponential backoff) to git api calls. + It will raise exceptions for maximum number of retries exceeded, server error, + and unexpected errors. GithubException 404 is propagated to the caller. + """ + + max_retries = 3 + retry_delay = 5 + last_message = "" + func_error_str = f"Error on git API request to '{func.__name__}'" + + for attempt in range(max_retries): + try: + return func(*args, **kwargs) + except GithubException as e: + if e.status == 403 and "rate limit" in e.data.get("message", "").lower(): + last_message = "Rate limit exceeded" + elif e.status == 404: + raise + elif 500 <= e.status < 600: + last_message = f"Server error {e.status}" + else: + last_message = f"Unknown error {e.status}, {e.data.get('message', '')}" + print(last_message + ", retrying in {retry_delay} seconds...") + + except Exception as e: + print(f"Unexpected error: {func_error_str} {str(e)}") + raise Exception(f"{func_error_str}: {str(e)}") + + time.sleep(retry_delay) + retry_delay *= 2 + + raise Exception(f"Max retries exceeded, {func_error_str}; message: '{last_message}'") + + # Get the repository + repo = make_github_request(git_object.get_repo, f"{git_user}/ocp-build-data") + + # Get the base branch where we will make the PR against. + base_branch = make_github_request(repo.get_branch, release_for_image) + + # Generate a unique branch name based on current time so you can easily tell how + # old the branch is in case we need to clean up. + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S-%f") + new_branch_name = f"art-dashboard-new-image-{timestamp}" + + # Create a new branch off the base branch + make_github_request(repo.create_git_ref, ref=f"refs/heads/{new_branch_name}", sha=base_branch.commit.sha) + + # Create the file (images/pf-status-relay.yml) on the new branch + file_path = f"images/{image_name}.yml" + make_github_request( + repo.create_file, + path=file_path, + message=f"{image_name} image add", + content=file_content, + branch=new_branch_name + ) + + # Create a pull request from the new branch to the base branch. + # Since we don't yet have a Jira ID, we'll use Jira-TBD for now. + pr = make_github_request( + repo.create_pull, + title=f"[JIRA-TBD] {image_name} image add", + body=f"Ticket: JIRA-TBD\n\nThis PR adds the {image_name} image file", + head=new_branch_name, + base=release_for_image + ) + + print(f"Pull request created: {pr.html_url} on branch {new_branch_name}") + pr_status = { + "status": "success", + "payload": "PR created successfully", + "pr_url": pr.html_url + } + + except GithubException as e: + print(f"GithubException: {str(e)}") + return Response({ + "status": "failure", + "error": f"GithubException: {str(e)}" + }, status=e.status) + + except Exception as e: + print(f"Other Exception: {str(e)}") + return Response({ + "status": "failure", + "error": f"Other Exception: {str(e)}" + }, status=500) + + # Extract the PR url from the pr_status + pr_url = pr_status['pr_url'] + + if jira_test_mode: + jira_status = { + "status": "success", + "jira_url": f"https://issues.redhat.com/browse/{TEST_ART_JIRA}", + "pr_url": pr_url + } + return Response(jira_status, status=200) + else: + # Login to Jira + jira_email = os.environ.get('JIRA_EMAIL') + jira_api_token = os.environ.get('JIRA_TOKEN') + + if not jira_email or not jira_api_token: + return Response({ + "status": "failure", + "error": "Missing Jira credentials: JIRA_EMAIL or JIRA_TOKEN values are missing" + }, status=400) + + try: + # Attempt to connect to Jira + headers = JIRA.DEFAULT_OPTIONS["headers"].copy() + headers["Authorization"] = f"Bearer {jira_api_token}" + + jira = JIRA(server='https://issues.redhat.com/', options={"headers": headers}) + + # Test the connection by retrieving something basic (the user's profile). + user = jira.current_user() + if not user: + return Response({ + "status": "failure", + "error": "Failed to properly authenticate to Jira." + }, status=400) + + except Exception as e: + # Handle failed authentication or connection issues + print(f"Authentication error: {str(e)}") + return Response({ + "status": "failure", + "error": f"Authentication error: {str(e)}" + }, status=400) + + jira_data = { + "project": {"key": jira_project_id}, + "summary": jira_summary, + "description": jira_description, + "issuetype": {"name": jira_story_type_id}, + "components": [{"name": jira_component}], + "priority": {"name": jira_priority}, + } + + try: + # Attempt to create the Jira + new_jira = jira.create_issue(fields=jira_data) + except Exception as e: + return Response({ + "status": "failure", + "error": f"An error occurred while creating the jira: {str(e)}; jira_data: {jira_data}" + }, status=400) + + jiraID = new_jira.key + + try: + # Now that we have a Jira, attempt to patch the PR title with the JiraID + pr.edit(title=f"[{jiraID}] {image_name} image add") + pr.edit(body=f"Ticket: {jiraID}\n\nThis PR adds the {image_name} image file") + except Exception as e: + return Response({ + "status": "failure", + "error": f"An error occurred while patching the PR: {str(e)}" + }, status=400) + + jira_status = { + "status": "success", + "jira_url": f"https://issues.redhat.com/browse/{jiraID}", + "pr_url": pr_url + } + return Response(jira_status, status=200) + + @api_view(["GET"]) def rpms_images_fetcher_view(request): release = request.query_params.get("release", None) diff --git a/conf/dev.env b/conf/dev.env index d332e20..aea8b74 100644 --- a/conf/dev.env +++ b/conf/dev.env @@ -5,3 +5,6 @@ MYSQL_USER=root MYSQL_PASSWORD=secret MYSQL_CONNECTION_PORT=3306 +# git user for ocp-build-data +# User any user that has a valid ocp-build-data repo and where you can get a git token +GIT_USER="DennisPeriquet" diff --git a/conf/prod.env b/conf/prod.env index 78f312e..ec55dae 100644 --- a/conf/prod.env +++ b/conf/prod.env @@ -5,3 +5,6 @@ MYSQL_CONNECTION_PORT=3306 # Kerberos KERBEROS_KEYTAB=/tmp/keytab/keytab KERBEROS_PRINCIPAL=exd-ocp-buildvm-bot-prod@IPA.REDHAT.COM + +# git user for ocp-build-data +GIT_USER="openshift-eng"