Test deployment locally with tmt

Using a Vagrant libvirt based virtual machine
and OpenShift Local.

.ansible-lint

@@ -14,6 +14,7 @@ mock_modules:
   # Ansible 2.9.27 in F35 still contains the k8s module so we can ignore the error until F36,
   # where we can switch to kubernetes.core.k8s as ansible-5.x in F36 contains it.
   - k8s
+  - kubernetes.core.k8s
   # Ignore until F36, where these are in community.crypto collection (part of ansible-5.x rpm).
   - openssh_keypair
   - openssl_certificate

.fmf/version

@@ -0,0 +1 @@
+1

Makefile

@@ -6,12 +6,22 @@ AP := ansible-playbook -vv -c local -i localhost, -e ansible_python_interpreter=
 # "By default, Ansible runs as if --tags all had been specified."
 # https://docs.ansible.com/ansible/latest/user_guide/playbooks_tags.html#special-tags
 TAGS ?= all
+VAGRANT_SSH_PORT = "$(shell cd containers && vagrant ssh-config | awk '/Port/{print $$2}')"
+VAGRANT_SSH_USER = "$(shell cd containers && vagrant ssh-config | awk '/User/{print $$2}')"
+VAGRANT_SSH_GUEST = "$(shell cd containers && vagrant ssh-config | awk '/HostName/{print $$2}')"
+VAGRANT_SSH_IDENTITY_FILE = "$(shell cd containers && vagrant ssh-config | awk '/IdentityFile/{print $$2}')"
+VAGRANT_SSH_CONFIG = $(shell cd containers && vagrant ssh-config | awk 'NR>1 {print " -o "$$1"="$$2}')
+VAGRANT_SHARED_DIR = "/vagrant"
+
+CENTOS_VAGRANT_BOX = CentOS-Stream-Vagrant-8-latest.x86_64.vagrant-libvirt.box
+CENTOS_VAGRANT_URL = https://cloud.centos.org/centos/8-stream/x86_64/images/$(CENTOS_VAGRANT_BOX)
 
 ifneq "$(shell whoami)" "root"
 	ASK_PASS ?= --ask-become-pass
 endif
 
 # Only for Packit team members with access to Bitwarden vault
+# if not working prepend OPENSSL_CONF=/dev/null to script invocation
 download-secrets:
 	./scripts/download_secrets.sh
 
@@ -50,3 +60,31 @@ check:
 move-stable:
 	[[ -d move_stable_repositories ]] || scripts/move_stable.py init
 	scripts/move_stable.py move-all
+
+oc-cluster-create:
+# vagrant pointer is broken...
+	[[ -f $(CENTOS_VAGRANT_BOX) ]] || wget $(CENTOS_VAGRANT_URL)
+	cd containers && vagrant up
+
+oc-cluster-destroy:
+	cd containers && vagrant destroy
+
+oc-cluster-up:
+	cd containers && vagrant up
+	cd containers && vagrant ssh -c "cd $(VAGRANT_SHARED_DIR) && $(AP) playbooks/oc-cluster-run.yml"
+
+oc-cluster-down:
+	cd containers && vagrant halt
+
+oc-cluster-ssh: oc-cluster-up
+	ssh $(VAGRANT_SSH_CONFIG) localhost
+
+test-deploy:
+# to be run inside VM where the oc cluster is running! Call make tmt-tests instead.
+	DEPLOYMENT=dev $(AP) playbooks/generate-local-secrets.yml
+	DEPLOYMENT=dev $(AP) -e '{"src_dir": $(VAGRANT_SHARED_DIR)}' playbooks/test_deploy_setup.yml
+	DEPLOYMENT=dev $(AP) -e '{"container_engine": "podman", "registry": "default-route-openshift-image-registry.apps-crc.testing", "registry_user": "kubeadmin", "src_dir": $(VAGRANT_SHARED_DIR)}' playbooks/deploy.yml
+	DEPLOYMENT=dev $(AP) -e '{"container_engine": "podman", "registry": "default-route-openshift-image-registry.apps-crc.testing", "registry_user": "kubeadmin", "src_dir": $(VAGRANT_SHARED_DIR)}' playbooks/check.yml
+
+tmt-tests:
+	tmt run --all provision --how connect --user vagrant --guest $(VAGRANT_SSH_GUEST) --port $(VAGRANT_SSH_PORT) --key $(VAGRANT_SSH_IDENTITY_FILE)

containers/Vagrantfile

@@ -0,0 +1,94 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+VAGRANTFILE_API_VERSION = "2"
+# export VAGRANT_EXPERIMENTAL="1"
+# export VAGRANT_EXPERIMENTAL="disks"
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+ config.vm.box = "centos/stream8"
+ #config.vm.box_url = "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-Vagrant-8-latest.x86_64.vagrant-libvirt.box" gives 404
+ config.vm.box_url = "file:///$VagrantProjectHome/../CentOS-Stream-Vagrant-8-latest.x86_64.vagrant-libvirt.box"
+
+
+ # Forward traffic on the host to the development server on the guest
+ config.vm.network "forwarded_port", guest: 5000, host: 5000
+ # Forward traffic on the host to Redis on the guest
+ config.vm.network "forwarded_port", guest: 6379, host: 6379
+ # Forward traffic on the host to the SSE server on the guest
+ config.vm.network "forwarded_port", guest: 8080, host: 8080
+
+
+ if Vagrant.has_plugin?("vagrant-hostmanager")
+     config.hostmanager.enabled = true
+     config.hostmanager.manage_host = true
+ end
+
+ # Vagrant can share the source directory using rsync, NFS, or SSHFS (with the vagrant-sshfs
+ # plugin). By default it rsyncs the current working directory to /vagrant.
+ #
+ # If you would prefer to use NFS to share the directory uncomment this and configure NFS
+ # config.vm.synced_folder ".", "/vagrant", type: "nfs", nfs_version: 4, nfs_udp: false
+ config.vm.synced_folder "..", "/vagrant"
+ # config.vm.synced_folder ".", "/vagrant", disabled: true
+ # config.vm.synced_folder ".", "/srv/pagure",
+ #     ssh_opts_append: "-o IdentitiesOnly=yes",
+ #     type: "sshfs"
+
+ # To cache update packages (which is helpful if frequently doing `vagrant destroy && vagrant up`)
+ # you can create a local directory and share it to the guest's DNF cache. The directory needs to
+ # exist, so create it before you uncomment the line below.
+ #Dir.mkdir('.dnf-cache') unless File.exists?('.dnf-cache')
+ #config.vm.synced_folder ".dnf-cache", "/var/cache/dnf",
+ #   type: "sshfs",
+ #   sshfs_opts_append: "-o nonempty"
+
+ # Comment this line if you would like to disable the automatic update during provisioning
+ config.vm.provision "shell", inline: "sudo dnf -y --disablerepo '*' --enablerepo=extras swap centos-linux-repos centos-stream-repos"
+
+ # !!!!!!! resize disk image !!!!!!!!!
+ config.vm.provision "shell", inline: "sudo dnf install -y cloud-utils-growpart"
+ config.vm.provision "shell", inline: "sudo growpart /dev/vda 1"
+ config.vm.provision "shell", inline: "sudo xfs_growfs /dev/vda1"
+
+ # bootstrap and run with ansible
+ config.vm.provision "ansible" do |ansible|
+     # ansible.verbose = "-vvv"
+     ansible.verbose = true
+     ansible.playbook = "../playbooks/oc-cluster-setup.yml"
+ end
+ config.vm.provision "ansible" do |ansible|
+     # ansible.verbose = "-vvv"
+     ansible.verbose = true
+     ansible.playbook = "../playbooks/oc-cluster-run.yml"
+     ansible.extra_vars = "../secrets/openshift-local-pull-secret.yml"
+ end
+ config.vm.provision "ansible" do |ansible|
+     # ansible.verbose = "-vvv"
+     ansible.verbose = true
+     ansible.playbook = "../playbooks/oc-cluster-tests-setup.yml"
+     ansible.extra_vars = {"src_dir": "/vagrant"}
+ end
+
+ # Create the box
+ config.vm.define "packit-oc-cluster" do |oc|
+    oc.vm.host_name = "packit-oc-cluster.example.com"
+
+    oc.vm.provider :libvirt do |domain|
+        # Season to taste
+        domain.cpus = 6
+        domain.graphics_type = "spice"
+        domain.memory = 14336
+        domain.video_type = "qxl"
+        domain.machine_virtual_size = 100
+
+        # Uncomment the following line if you would like to enable libvirt's unsafe cache
+        # mode. It is called unsafe for a reason, as it causes the virtual host to ignore all
+        # fsync() calls from the guest. Only do this if you are comfortable with the possibility of
+        # your development guest becoming corrupted (in which case you should only need to do a
+        # vagrant destroy and vagrant up to get a new one).
+        #
+        # domain.volume_cache = "unsafe"
+    end
+ end
+end

docs/deployment/testing-changes.md

@@ -68,3 +68,38 @@ This repository provides helpful playbook to do this with one command:
 
 Zuul provides a public key for every project. The ansible playbook downloads Zuul repository and pass the project tenant and name as parameters to encryption script. This script then encrypts files with public key of the project.
 For more information please refer to [official docs](https://ansible.softwarefactory-project.io/docs/user/zuul_user.html#create-a-secret-to-be-used-in-jobs).
+
+### Test Deployment locally with OpenShift Local
+
+For using OpenShift Local you need a _pull secret_, download it here: https://console.redhat.com/openshift/create/local. Save it in a file called `secrets\openshift-local-pull-secret.yml` following this format:
+
+```
+---
+pull_secret: <<< DOWNLOADED PULL SECRET CONTENT >>>
+```
+
+Populate the `secrets` dir with all the other secrets.
+You _should use_ your own secrets but if you have access to `stg` secrets
+you can also do:
+
+```
+DEPLOYMENT=stg make download-secrets
+```
+
+Now you can create and start the OpenShift Local cluster (it take as long as an hour) in a Vagrant Virtual Machine with:
+
+```
+make oc-cluster-create
+```
+
+And once it is up and running you can test the `packit-service` deployment with the command:
+
+```
+make tmt-tests
+```
+
+This command will sshed the virtual machine and run the tests there (`make test-deploy`),
+you can run the tests as many time you want as long as the virtual machine is up and running and the `crc cluster` is started (`make oc-cluster-up` after every `make oc-cluster-down`).
+You can skip the `tmt` environment and run the test directly inside the VM using `make oc-cluster-ssh` and `cd /vagrant && make test-deploy`.
+
+You can destroy the `libvirt` machine with `make oc-cluster-destroy` and re-create it again with `make oc-cluster-create`.

openshift/redis.yml.j2

@@ -20,7 +20,7 @@ spec:
     spec:
       containers:
         - name: redis
-          image: quay.io/sclorg/redis-7-c9s
+          image: quay.io/sclorg/redis-7-c9s:c9s
           ports:
             - containerPort: 6379
           volumeMounts:

plans/main.fmf

@@ -0,0 +1,13 @@
+summary: Packit deployment checks
+
+discover:
+  how: fmf
+
+execute:
+  how: tmt
+
+/deployment:
+  summary: Run packit-service deployment test
+  discover+:
+    test:
+      - deployment

playbooks/deploy.yml

@@ -65,6 +65,8 @@
           memory: "128Mi"
           cpu: "50m"
     appcode: PCKT-002
+    registry: 172.30.1.1:5000
+    registry_user: developer
   tasks:
     - name: Include tasks/project-dir.yml
       ansible.builtin.include_tasks: tasks/project-dir.yml
@@ -124,25 +126,25 @@
             tls_verify_false: "{{ '--tls-verify=false' if 'podman' in container_engine else '' }}"
           changed_when: false
         - name: Login to local cluster
-          ansible.builtin.shell: "{{ container_engine }} login -u developer -p $(oc whoami -t) 172.30.1.1:5000 {{ tls_verify_false }}"
+          ansible.builtin.shell: "{{ container_engine }} login -u {{ registry_user }} -p $(oc whoami -t) {{ registry }} {{ tls_verify_false }}"
           changed_when: false
         - name: Inspect service image
           ansible.builtin.command: "{{ container_engine }} inspect {{ image }}"
           changed_when: false
         - name: Tag the image with :dev
-          ansible.builtin.command: "{{ container_engine }} tag {{ image }} 172.30.1.1:5000/myproject/packit-service:dev"
+          ansible.builtin.command: "{{ container_engine }} tag {{ image }} {{ registry }}/myproject/packit-service:dev"
           changed_when: true
         - name: Push the image
-          ansible.builtin.command: "{{ container_engine }} push 172.30.1.1:5000/myproject/packit-service:dev {{ tls_verify_false }}"
+          ansible.builtin.command: "{{ container_engine }} push {{ registry }}/myproject/packit-service:dev {{ tls_verify_false }}"
           changed_when: true
         - name: Inspect worker image
           ansible.builtin.command: "{{ container_engine }} inspect {{ image_worker }}"
           changed_when: false
         - name: Tag the image with :dev
-          ansible.builtin.command: "{{ container_engine }} tag {{ image_worker }} 172.30.1.1:5000/myproject/packit-worker:dev"
+          ansible.builtin.command: "{{ container_engine }} tag {{ image_worker }} {{ registry }}/myproject/packit-worker:dev"
           changed_when: true
         - name: Push the image
-          ansible.builtin.command: "{{ container_engine }} push 172.30.1.1:5000/myproject/packit-worker:dev {{ tls_verify_false }}"
+          ansible.builtin.command: "{{ container_engine }} push {{ registry }}/myproject/packit-worker:dev {{ tls_verify_false }}"
           changed_when: true
 
     - name: Deploy secrets
@@ -439,7 +441,7 @@
 
     - name: Wait for deploymentconfig rollouts to complete
       # timeout 10min to not wait indefinitely in case of a problem
-      ansible.builtin.command: timeout 10m oc rollout status -w deploy/{{ item }}
+      ansible.builtin.command: timeout 15m oc rollout status -w deploy/{{ item }}
       register: oc_rollout_status
       changed_when: false
       failed_when: '"successfully rolled out" not in oc_rollout_status.stdout'

playbooks/oc-cluster-run.yml

@@ -0,0 +1,19 @@
+---
+- name: Start Openshift Local cluster
+  hosts: all
+  gather_facts: False
+  vars:
+    pull_secret: "!!!TAKEN FROM VAGRANT FILE!!!"
+    dest_pull_secret_file: /tmp/openshift-local-pull-secret.txt
+  tasks:
+    - name: Show pull secret
+      ansible.builtin.debug:
+        msg: "{{ pull_secret }}"
+    - name: Create secret file from pull request var
+      ansible.builtin.copy:
+        content: "{{ pull_secret }}"
+        dest: "{{ dest_pull_secret_file }}"
+        mode: "0640"
+    - name: Start cluster
+      ansible.builtin.command: crc start -p {{ dest_pull_secret_file }}
+      changed_when: False

playbooks/oc-cluster-setup.yml

@@ -0,0 +1,52 @@
+---
+- name: Install an Openshift Local cluster
+  hosts: all
+  vars:
+    crc_download_link: https://developers.redhat.com/content-gateway/rest/mirror/pub/openshift-v4/clients/crc/latest/crc-linux-amd64.tar.xz
+  tasks:
+    - name: Add vagrant user to sudoers
+      become: True
+      ansible.builtin.lineinfile:
+        path: /etc/sudoers
+        state: present
+        line: "vagrant ALL=(ALL) NOPASSWD: ALL"
+        validate: /usr/sbin/visudo -cf %s
+    - name: Download CRC
+      ansible.builtin.get_url:
+        url: "{{ crc_download_link }}"
+        dest: /tmp
+        mode: "0664"
+      register: download
+    - name: Unarchive tarball
+      ansible.builtin.unarchive:
+        src: "{{ download.dest }}"
+        dest: /tmp
+        list_files: True
+        remote_src: True
+      register: unarchive
+    - name: Show crc executable path
+      ansible.builtin.debug:
+        msg: "/tmp/{{ unarchive.files[0] }}crc"
+    - name: Create ~/.local/bin if it does not exist
+      ansible.builtin.file:
+        path: ~/.local/bin
+        state: directory
+        mode: "0755"
+    - name: Create symbolic link to crc
+      ansible.builtin.file:
+        src: "/tmp/{{ unarchive.files[0] }}/crc"
+        dest: ~/.local/bin/crc
+        state: link
+        force: True
+    - name: Setup consent-telemetry no
+      ansible.builtin.command: crc config set consent-telemetry no
+      changed_when: False
+    - name: Setup minimum packit-service memory requirements
+      ansible.builtin.command: crc config set memory 13312 MiB
+      changed_when: False
+    - name: Setup minimum packit-service disk space requirements
+      ansible.builtin.command: crc config set disk-size 60 Gi
+      changed_when: False
+    - name: Setup cluster
+      ansible.builtin.command: crc setup
+      changed_when: False

playbooks/oc-cluster-tests-setup.yml

@@ -0,0 +1,45 @@
+# Copyright Contributors to the Packit project.
+# SPDX-License-Identifier: MIT
+
+---
+- name: Create vars/packit/dev.yml and deploy
+  hosts: all
+  vars:
+    oc_download_link: https://mirror.openshift.com/pub/openshift-v4/amd64/clients/ocp/stable/openshift-client-linux.tar.gz
+    test_project_name: myproject
+    crc_url: https://api.crc.testing:6443
+    image: quay.io/packit/packit-service:stg
+    image_worker: quay.io/packit/packit-worker:stg
+    image_fedmsg: quay.io/packit/packit-service-fedmsg:stg
+    image_dashboard: quay.io/packit/dashboard:stg
+    image_tokman: quay.io/packit/tokman:stg
+  tasks:
+    - name: Enable EPEL repositories
+      ansible.builtin.package:
+        name:
+          - epel-release
+      become: true
+    - name: Install packages for deployment
+      ansible.builtin.package:
+        name:
+          - ansible
+          - python3-openshift
+          - python3-pip
+          - python3-passlib # for using htpasswd ansible module
+          - make
+          - podman
+      become: true
+
+    - name: Download OC
+      ansible.builtin.get_url:
+        url: "{{ oc_download_link }}"
+        dest: /tmp
+        mode: "0644"
+      register: download
+    - name: Unarchive tarball in /usr/bin
+      become: True
+      ansible.builtin.unarchive:
+        src: "{{ download.dest }}"
+        dest: /usr/bin
+        list_files: True
+        remote_src: True