Adding svn validations basic flow

.github/workflows/ci.yml

@@ -0,0 +1,21 @@
+name: Test Gh publish
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+      - name: "Run tests"
+        run: |
+            python3 -m pip install pytest
+            pytest ./tests

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+*./__pycache__/.*

README.md

@@ -1 +1,71 @@
-# gh-svn-pypi-publisher
+# GH SVN PyPI Publisher
+
+## Description
+`Gh Publish` is a GitHub Action designed to validate artifacts and publish to PyPI. It includes steps for setting up Python, parsing configuration files, checking out SVN repositories, performing SVN and checksum checks, and publishing to PyPI.
+
+## Inputs
+- `publish-config` (required): Path to the publish config file. Default is `publish-config.yml`.
+- `temp-dir` (optional): Temporary directory to checkout SVN repo. Default is `temp-svn-repo`.
+- `mode` (optional): Mode to run the action. Default is `verify`.
+
+## Usage
+To use this action, include it in your workflow YAML file:
+
+```yaml
+name: Publish to PyPI
+
+on:
+  workflow_dispatch:
+
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Gh Publish
+        uses: ./gopidesupavan/gh-pub@main
+        with:
+          publish-config: 'path/to/your/publish-config.yml'
+          temp-dir: 'temp-svn-repo'
+          mode: 'publish'
+```
+# About publish-config.yml
+
+The `publish-config.yml` file has composed of multiple rules to validate the artifacts and publish them to PyPI. The configuration file is structured as follows:
+
+```yaml
+project:
+    Name: example-pub
+    Description: Example project for publishing to PyPI
+
+publishers:
+    name: providers
+    url: https://dist.apache.org/repos/dist/dev/airflow/
+    path: "airflow/providers/pypi"
+    version_pattern: '^(.*?)-(rc\d+-)?\d+'
+    extensions:
+      - .tar.gz
+      - .tar.gz.asc
+      - .tar.gz.sha512
+      - -py3-none-any.whl
+      - -py3-none-any.whl.asc
+      - -py3-none-any.whl.sha512
+    rules:
+      svn-check:
+          name: "SVN Check"
+          type: "svn"
+          enabled: "false"
+      checksum-check:
+          name: "SHA512 Check"
+          type: "512"
+          enabled: "true"
+          script: "/home/runner/work/example-pub/example-pub/scripts/checksum_check.sh"
+```
+svn-check: This rule is used to validate the package extension. It checks each package have the required extension or not. eg: .tar.gz, .tar.gz.asc, .tar.gz.sha512, -py3-none-any.whl, -py3-none-any.whl.asc, -py3-none-any.whl.sha512, total 6 extensions are required. 
+
+checksum-check: This rule is used to validate the checksum of the package. It checks the checksum of the package with the provided checksum type. eg: SHA512 checksum is required for each package.
+
+script: Script use to validate the rules. if there is no script provided in the publish-config.yml file, the default script will be used to validate the rules. default scripts are under the src/scripts directory.

action.yml

@@ -1,6 +1,18 @@
-
 name: 'GH SVN PyPI Publisher'
 description: 'Publishes artifacts to pypi'
+inputs:
+  publish-config:
+    description: 'Path to the publish config file'
+    required: true
+    default: 'publish-config.yml'
+  temp-dir:
+    description: 'Temporary directory to checkout svn repo'
+    required: false
+    default: 'temp-svn-repo'
+  mode:
+    description: 'Mode to operate publish or verify'
+    required: false
+    default: 'verify'
 
 runs:
   using: "composite"
@@ -9,3 +21,73 @@ runs:
       uses: actions/setup-python@v4
       with:
         python-version: '3.9'
+
+    - name: "Config parser"
+      shell: bash
+      id: config-parser
+      env:
+        PUBLISH_CONFIG: ${{ inputs.publish-config }}
+        GITHUB_ACTION_PATH: ${{ github.action_path }}
+      run: |
+        mkdir -p ${{ inputs.temp-dir }}
+        python3 -m pip install pyyaml
+        python3 $GITHUB_ACTION_PATH/src/scripts/config_parser.py "${PUBLISH_CONFIG}"
+
+    - name: Checkout SVN
+      shell: bash
+      env:
+        repo_url: ${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.url }}
+      run: |
+        echo "Checking out SVN repo at $repo_url"
+        svn co $repo_url
+        echo "SVN repo checked out"
+      working-directory: "./${{ inputs.temp-dir }}"
+
+    - name: SVN check
+      shell: bash
+      if: ${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.rules.svn-check.enabled == 'true' }}
+      env:
+        FILE_EXTENSIONS: ${{ toJson(fromJSON(steps.config-parser.outputs.pub_config).publishers.extensions) }}
+        PATH: ${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.path }}
+        NAME: ${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.rules.svn-check.name }}
+        VERSION_FORMAT: ${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.version_pattern }}
+        SCRIPT: ${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.rules.svn-check.script }}
+      working-directory: ./${{ inputs.temp-dir }}/${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.path }}
+      run: |
+        echo "Verifying $NAME"
+        python3 $SCRIPT
+
+    - name: Checksum check
+      shell: bash
+      if: ${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.rules.checksum-check.enabled ==  'true' }}
+      env:
+        NAME: ${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.rules.checksum-check.name }}
+        type: ${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.rules.checksum-check.type }}
+        SCRIPT: ${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.rules.checksum-check.script }}
+      working-directory: ./${{ inputs.temp-dir }}/${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.path }}
+      run: |
+        echo "Verifying $NAME"
+        chmod +x $SCRIPT
+        $SCRIPT "$type"
+
+    - name: Signature check
+      shell: bash
+      if: ${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.rules.signature-check.enabled == 'true' }}
+      env:
+        NAME: ${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.rules.signature-check.name }}
+        SCRIPT: ${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.rules.signature-check.script }}
+        KEYS: ${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.rules.signature-check.keys }}
+        KEYS_FILE_LOCATION: ${{ github.workspace }}/${{ inputs.temp-dir }}/KEYS
+      working-directory: ./${{ inputs.temp-dir }}/${{ fromJSON(steps.config-parser.outputs.pub_config).publishers.path }}
+      run: |
+        curl -o $KEYS_FILE_LOCATION $KEYS 
+        gpg --import $KEYS_FILE_LOCATION
+        echo "Verifying $NAME"
+        chmod +x $SCRIPT
+        $SCRIPT
+
+    - name: Publish to PyPI
+      shell: bash
+      if: ${{ inputs.mode == 'publish' }}
+      run: |
+        echo "TBD Publishing to PyPI"

publish-config.yml

@@ -0,0 +1,30 @@
+project:
+    Name: gh-svn-pypi-publisher
+    Description: Example project for publishing to PyPI
+
+publishers:
+    name: providers
+    url: https://gh-svn-pypi-publisher/
+    path: "airflow/providers/"
+    version_pattern: '^(.*?)-(rc\d+-)?\d+'
+    extensions:
+      - .tar.gz
+      - .tar.gz.asc
+      - .tar.gz.sha512
+      - -py3-none-any.whl
+      - -py3-none-any.whl.asc
+      - -py3-none-any.whl.sha512
+    rules:
+      svn-check:
+          name: "SVN Check"
+          type: "svn"
+          enabled: "false"
+      checksum-check:
+            name: "SHA512 Check"
+            type: "512"
+            enabled: "false"
+      signature-check:
+            name: "Signature Check"
+            enabled: "false"
+            script: "/home/runner/work/gh-svn-pypi-publisher/gh-svn-pypi-publisher/scripts/signature_check.sh"
+            keys: "https://dist.apache.org/repos/dist/release/airflow/KEYS"

src/scripts/checksum_check.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+EXIT=0
+
+for i in *.asc
+do
+   echo -e "Checking $i\n"; gpg --verify $i
+done
+
+if [ $EXIT -eq 1 ]; then
+    echo "One or more checksums did not match."
+    exit 1
+else
+    echo "All checksums match."
+fi

src/scripts/config_parser.py

@@ -0,0 +1,44 @@
+import os
+import sys
+import json
+import yaml
+
+DEFAULT_SVN_CHECKER_SCRIPT = "{github_action_path}/src/scripts/svn_checker.py"
+DEFAULT_CHECK_SUM_SCRIPT = "{github_action_path}/src/scripts/checksum_check.sh"
+DEFAULT_SIGNATURE_CHECK_SCRIPT = "{github_action_path}/src/scripts/signature_check.sh"
+
+
+def set_default_config(config_data: dict):
+    svn_checker_script = config_data.get("publishers", {}).get("rules", {}).get("svn-check", {}).get("script")
+    if not svn_checker_script:
+        config_data["publishers"]["rules"]["svn-check"]["script"] = DEFAULT_SVN_CHECKER_SCRIPT.format(github_action_path=os.environ.get("GITHUB_ACTION_PATH"))
+
+    check_sum_script = config_data.get("publishers", {}).get("rules", {}).get("checksum-check", {}).get("script")
+    if not check_sum_script:
+        config_data["publishers"]["rules"]["checksum-check"]["script"] = DEFAULT_CHECK_SUM_SCRIPT.format(github_action_path=os.environ.get("GITHUB_ACTION_PATH"))
+
+    signature_check_script = config_data.get("publishers", {}).get("rules", {}).get("signature-check", {}).get("script")
+    if not signature_check_script:
+        config_data["publishers"]["rules"]["signature-check"]["script"] = DEFAULT_SIGNATURE_CHECK_SCRIPT.format(github_action_path=os.environ.get("GITHUB_ACTION_PATH"))
+
+    return config_data
+
+
+def parse_config(path: str):
+    with open(path, 'r') as file:
+        config_data = yaml.safe_load(file)
+
+    updated_config_data = set_default_config(config_data)
+
+    def set_multiline_output(name, updated_data):
+        with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
+            value = json.dumps(updated_data)
+            f.write(f'{name}={value}')
+
+    set_multiline_output("pub_config", updated_config_data)
+
+
+
+if __name__ == '__main__':
+    config_path = sys.argv[1]
+    parse_config(config_path)

src/scripts/signature_check.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+EXIT=0
+
+for i in *.asc
+do
+   echo -e "Checking $i\n"
+   if ! gpg --verify $i 2>&1 | grep -q "Good signature"; then
+       echo "Signature check failed for $i"
+       EXIT=1
+   fi
+done
+
+if [ $EXIT -eq 1 ]; then
+    echo "One or more signature checks did not match."
+    exit 1
+else
+    echo "All signature checks match."
+fi

src/scripts/svn_checker.py

@@ -0,0 +1,73 @@
+import ast
+import os
+import re
+import sys
+from collections import Counter
+
+unknown_files = []
+unknown_file_extensions = []
+valid_files = []
+failed_count_check = []
+
+def check_extension(file, extensions):
+    for extension in extensions:
+        if file.endswith(extension):
+            print(f"File extension {extension} matched with {file}")
+            return True
+    return False
+
+def extract_name(file_name_pattern, file):
+    match = re.match(file_name_pattern, file)
+    if match:
+        name_before_version = match.group(1)
+        return name_before_version
+    return None
+
+def validate_package_name_count(file_with_count, extensions):
+    for package_name, count in file_with_count.items():
+        if not (count == len(extensions)):
+            failed_count_check.append(f"package name: {package_name}, count: {count}, expected count: {len(extensions)}")
+
+def check_files(version_pattern: str, extensions: list[str]):
+    exit_code = 0
+    files = os.listdir()
+    print(f"Found total files in {os.getcwd()}: ", len(files))
+    for file in files:
+        if not check_extension(file, extensions):
+            unknown_file_extensions.append(file)
+            continue
+
+        package_name = extract_name(version_pattern, file)
+        if not package_name:
+            unknown_files.append(file)
+            continue
+
+        # Just to make sure that we are counting to same package name, ex: apache-airflow and apache_airflow in the dist folder
+        valid_files.append(package_name.replace("-", "_"))
+
+    file_with_count = Counter(valid_files)
+    validate_package_name_count(file_with_count, extensions)
+
+    if failed_count_check:
+        print(f"Following packages are not matching the count: {failed_count_check}")
+        exit_code = 1
+
+    if unknown_files:
+        exit_code = 1
+        print(f"Following files are not matching the pattern: {unknown_files}")
+
+    if unknown_file_extensions:
+        exit_code = 1
+        print(f"Following files are not matching the extensions: {unknown_file_extensions}")
+
+    if exit_code == 0:
+        print("SVN check passed successfully.")
+
+    sys.exit(exit_code)
+
+
+
+if __name__ == "__main__":
+    file_extensions = ast.literal_eval(os.environ.get("FILE_EXTENSIONS"))
+    version_format = os.environ.get("VERSION_FORMAT")
+    check_files(version_format, file_extensions)