Replace shell get_download_url_from_github_api function (#689)

* Replace shell get_download_url_from_github_api function With the fix for #676 we can now replace shell function with it's python equivalent. Signed-off-by: Michal Pryc <[email protected]> * Source venv before running cli_def_tools_script Make it depend on the set up venv also Signed-off-by: Kevin M Granger <[email protected]> * Add make target for a single CLI dev tool, make chart-lint depend on ct Signed-off-by: Kevin M Granger <[email protected]> Signed-off-by: Michal Pryc <[email protected]> Signed-off-by: Kevin M Granger <[email protected]> Co-authored-by: Kevin M Granger <[email protected]>
dora-metrics · Oct 17, 2022 · a31f814 · a31f814
1 parent adbca1f
commit a31f814
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 87 deletions.
diff --git a/Makefile b/Makefile
@@ -77,8 +77,15 @@ git-blame:
 
 ## cli_dev_tools: install all necessary CLI dev tools
 .PHONY: cli_dev_tools
-cli_dev_tools:
-	./scripts/install_dev_tools -v $(PELORUS_VENV)
+cli_dev_tools: $(PELORUS_VENV)
+	. ${PELORUS_VENV}/bin/activate && \
+		./scripts/install_dev_tools -v $(PELORUS_VENV)
+
+
+# for installing a single CLI dev tool
+$(PELORUS_VENV)/bin/%: $(PELORUS_VENV)
+	. ${PELORUS_VENV}/bin/activate && \
+		./scripts/install_dev_tools -v $(PELORUS_VENV) -c $(notdir $@)
 
 ## dev-env: set up everything needed for development (install tools, set up virtual environment, git configuration)
 dev-env: $(PELORUS_VENV) cli_dev_tools exporters git-blame \
@@ -216,8 +223,7 @@ chart-check-bump: $(PELORUS_VENV)
 	. ${PELORUS_VENV}/bin/activate && \
 	./scripts/chart-check-and-bump
 
-chart-lint: $(PELORUS_VENV)
-	./scripts/install_dev_tools -v $(PELORUS_VENV) -c ct && \
+chart-lint: $(PELORUS_VENV) $(PELORUS_VENV)/bin/ct $(PELORUS_VENV)/bin/helm
 	. ${PELORUS_VENV}/bin/activate && \
 	ct lint --config ct.yaml
 

diff --git a/scripts/install_dev_tools b/scripts/install_dev_tools
@@ -26,16 +26,6 @@ DEFAULT_VENV="${SCRIPT_DIR}/../.venv"
 
 TMP_DIR_PREFIX="pelorus_tmp_"
 
-TKN_CLIENT_API_URL="https://api.github.com/repos/tektoncd/cli/releases/latest"
-CT_CLIENT_API_URL="https://api.github.com/repos/helm/chart-testing/releases/latest"
-CONFTEST_CLIENT_API_URL="https://api.github.com/repos/open-policy-agent/conftest/releases/latest"
-
-# Required to get noobaa CLI
-NOOBAA_OPERATOR_API_URL="https://api.github.com/repos/noobaa/noobaa-operator/releases/latest"
-
-# Required to get the promtool
-PROMETHEUS_API_URL="https://api.github.com/repos/prometheus/prometheus/releases/latest"
-
 HELM_INSTALL_SCRIPT="https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3"
 # Used to download HELM version if git API failed with providing correct one
 HELM_FALLBACK_VERSION="v3.9.2"
@@ -81,69 +71,6 @@ function extract_file_to_dir() {
     tar xvzf "${file_path}" -C "${dest_dir}" ${file_names}
 }
 
-function get_download_url_from_github_api() {
-    local api_url=$1
-    if [ -z "${api_url}" ]; then
-        echo "get_download_url_from_github(): API URL not provided" >&2
-        return 2
-    fi
-
-    local arch
-    local kernel_name
-    arch="$(uname -m)"
-    kernel_name="$(uname -s)" # e.g. Linux / Darwin
-
-    # Noobaa CLI provides direct link to the binary,
-    # which uses either "linux" or "mac" as part of the url
-    noobaa_arch="linux"
-
-    # Little hack for grep as some projects are shipping release tarballs
-    # as x86_64 and some as amd64.
-    if [[ ${kernel_name} == "Darwin" ]]; then
-        noobaa_arch="mac"
-        # Also, some projects are shipping universal binaries to support both
-        # x86 and arm macs. They are exposed as "all".
-        # This is a special case so we don't get `-e -e ${arch}`
-        if [[ ${arch} == "x86_64" ]]; then
-            arch="-e ${arch} -e amd64 -e all"
-        elif [[ ${arch} == "amd64" ]]; then
-            arch="-e ${arch} -e x86_64 -e all"
-        elif [[ ${arch} == "arm64" ]]; then
-            arch="-e ${arch} -e all"
-        fi
-    else
-        if [[ ${arch} == "x86_64" ]]; then
-            arch="-e ${arch} -e amd64"
-        elif [[ ${arch} == "amd64" ]]; then
-            arch="-e ${arch} -e x86_64"
-        fi
-    fi
-
-    # Call the API to find the latest binary matching kernel and architecture
-    local url_cmd
-    local download_url
-
-    # Noobaa uses different binary url schema
-    if [ "${api_url}" == "${NOOBAA_OPERATOR_API_URL}" ]; then
-        url_cmd="curl -s ${api_url} | grep 'download_url' | \
-              grep -o -E 'https://(.*)-${noobaa_arch}-(.*)[0-9]'  | head -n 1" 
-    else
-        url_cmd="curl -s ${api_url} | \
-              grep -o -E 'https://(.*)(.tar.gz)' | \
-              grep -i '${kernel_name}' | \
-              grep ${arch} | head -n 1"
-    fi
-    download_url=$(eval "${url_cmd}")
-    if [ -z "${download_url}" ]; then
-        echo "get_download_url_from_github(): download_url not available." >&2
-        echo "command used to get URL: $ ${url_cmd}" >&2
-        return 2
-    fi
-
-    # Return URL
-    echo "${download_url}"
-}
-
 # Installs helm into .venv path
 function install_helm() {
     local helm_install_script=$1
@@ -261,7 +188,9 @@ fi
 if should_cli_be_installed "tkn" "${cli_tools_arr[@]}" && \
     ! [ -x "$(command -v "${VENV}/bin/tkn")" ]; then
       echo "Installing tkn CLI to: ${VENV}/bin/tkn"
-      TKN_CLIENT_URL=$(get_download_url_from_github_api "${TKN_CLIENT_API_URL}")
+      TKN_CLIENT_URL=$("${SCRIPT_DIR}/get_github_dl_url.py" tkn) \
+                    || cleanup_and_exit 2
+      echo "$TKN_CLIENT_URL"
       download_file_from_url "${TKN_CLIENT_URL}" "${DWN_DIR}"
       TKN_CLIENT_PATH="${DWN_DIR}"/$(basename "${TKN_CLIENT_URL}")
       extract_file_to_dir "${TKN_CLIENT_PATH}" "${VENV}/bin/" "tkn"
@@ -280,8 +209,9 @@ fi
 # CT install
 if should_cli_be_installed "ct" "${cli_tools_arr[@]}" && \
     ! [ -x "$(command -v "${VENV}/bin/ct")" ]; then
-      CT_CLIENT_URL=$(get_download_url_from_github_api "${CT_CLIENT_API_URL}") \
-                    || (echo "$CT_CLIENT_URL"; cleanup_and_exit 2)
+      CT_CLIENT_URL=$("${SCRIPT_DIR}/get_github_dl_url.py" ct) \
+                    || cleanup_and_exit 2
+      echo "$CT_CLIENT_URL"
       download_file_from_url "${CT_CLIENT_URL}" "${DWN_DIR}"
       CT_CLIENT_PATH="${DWN_DIR}"/$(basename "${CT_CLIENT_URL}")
       extract_file_to_dir "${CT_CLIENT_PATH}" "${VENV}/bin/" "ct"
@@ -290,8 +220,9 @@ fi
 # Promtool install
 if should_cli_be_installed "promtool" "${cli_tools_arr[@]}" && \
     ! [ -x "$(command -v "${VENV}/bin/promtool")" ]; then
-      PROMTOOL_CLIENT_URL=$(get_download_url_from_github_api "${PROMETHEUS_API_URL}") \
-                    || (echo "$PROMTOOL_CLIENT_URL"; cleanup_and_exit 2)
+      PROMTOOL_CLIENT_URL=$("${SCRIPT_DIR}/get_github_dl_url.py" promtool) \
+                    || cleanup_and_exit 2
+      echo "$PROMTOOL_CLIENT_URL"
       download_file_from_url "${PROMTOOL_CLIENT_URL}" "${DWN_DIR}"
       CT_CLIENT_PATH="${DWN_DIR}"/$(basename "${PROMTOOL_CLIENT_URL}")
       # promtool is in the dir with same name as tar, we need to
@@ -306,8 +237,9 @@ fi
 # Conftest CLI
 if should_cli_be_installed "conftest" "${cli_tools_arr[@]}" && \
     ! [ -x "$(command -v "${VENV}/bin/conftest")" ]; then
-      CONFTEST_CLIENT_URL=$(get_download_url_from_github_api "${CONFTEST_CLIENT_API_URL}") \
-                    || (echo "$CONFTEST_CLIENT_URL"; cleanup_and_exit 2)
+      CONFTEST_CLIENT_URL=$("${SCRIPT_DIR}/get_github_dl_url.py" conftest) \
+                    || cleanup_and_exit 2
+      echo "$CONFTEST_CLIENT_URL"
       download_file_from_url "${CONFTEST_CLIENT_URL}" "${DWN_DIR}"
       CONFTEST_CLIENT_PATH="${DWN_DIR}"/$(basename "${CONFTEST_CLIENT_URL}")
       extract_file_to_dir "${CONFTEST_CLIENT_PATH}" "${VENV}/bin/" "conftest"
@@ -336,8 +268,8 @@ fi
 # nooba CLI
 if should_cli_be_installed "noobaa" "${cli_tools_arr[@]}" && \
     ! [ -x "$(command -v "${VENV}/bin/noobaa")" ]; then
-      NOOBAA_CLIENT_URL=$(get_download_url_from_github_api "${NOOBAA_OPERATOR_API_URL}") \
-                    || (echo "$NOOBAA_CLIENT_URL"; cleanup_and_exit 2)
+      NOOBAA_CLIENT_URL=$("${SCRIPT_DIR}/get_github_dl_url.py" noobaa) \
+                    || cleanup_and_exit 2
       echo "$NOOBAA_CLIENT_URL"
       download_file_from_url "${NOOBAA_CLIENT_URL}" "${DWN_DIR}"
       NOOBAA_CLIENT_PATH="${DWN_DIR}"/$(basename "${NOOBAA_CLIENT_URL}")
@@ -353,4 +285,4 @@ if should_cli_be_installed "operator-sdk" "${cli_tools_arr[@]}" && \
       download_file_from_url "${OPERATOR_SDK_URL}" "${DWN_DIR}"
       mv "${DWN_DIR}"/operator-sdk_* "${VENV}/bin/operator-sdk"
       chmod +x "${VENV}/bin/operator-sdk"
-fi
+fi