From 3f0aa1bd532324bea85835138bdcdfe8e2b291d9 Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Mon, 8 Jan 2024 13:21:23 +0100 Subject: [PATCH 1/8] CM-30406 - Attach onedir CLI to GitHub releases with checksums --- .github/workflows/build_executable.yml | 48 ++------ process_executable_file.py | 156 +++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 39 deletions(-) create mode 100755 process_executable_file.py diff --git a/.github/workflows/build_executable.yml b/.github/workflows/build_executable.yml index 0de165b6..6acb9ac7 100644 --- a/.github/workflows/build_executable.yml +++ b/.github/workflows/build_executable.yml @@ -5,6 +5,7 @@ on: push: branches: - main + - CM-30406-attach-onedir-cli-to-git-hub-releases-with-checksums permissions: contents: write @@ -188,47 +189,16 @@ jobs: :: verify signature signtool.exe verify /v /pa ".\dist\cycode-cli.exe" + - name: Prepare files + env: + PROCESS_ONEDIR: ${{ matrix.mode == 'onedir' }} + run: echo "ARTIFACT_NAME=$(./process_executable_file.py dist/cycode-cli)" >> $GITHUB_ENV + - name: Prepare files on Windows if: runner.os == 'Windows' - run: | - echo "ARTIFACT_NAME=cycode-win" >> $GITHUB_ENV - mv dist/cycode-cli.exe dist/cycode-win.exe - powershell -Command "(Get-FileHash -Algorithm SHA256 dist/cycode-win.exe).Hash" > sha256 - head -c 64 sha256 > dist/cycode-win.exe.sha256 - - - name: Prepare files on Intel macOS (onefile) - if: runner.os == 'macOS' && runner.arch == 'X64' && matrix.mode == 'onefile' - run: | - echo "ARTIFACT_NAME=cycode-mac" >> $GITHUB_ENV - mv dist/cycode-cli dist/cycode-mac - shasum -a 256 dist/cycode-mac > sha256 - head -c 64 sha256 > dist/cycode-mac.sha256 - - - name: Prepare files on Apple Silicon macOS (onefile) - if: runner.os == 'macOS' && runner.arch == 'ARM64' && matrix.mode == 'onefile' - run: | - echo "ARTIFACT_NAME=cycode-mac-arm" >> $GITHUB_ENV - mv dist/cycode-cli dist/cycode-mac-arm - shasum -a 256 dist/cycode-mac-arm > sha256 - head -c 64 sha256 > dist/cycode-mac-arm.sha256 - - - name: Prepare files on Intel macOS (onedir) - if: runner.os == 'macOS' && runner.arch == 'X64' && matrix.mode == 'onedir' - run: | - echo "ARTIFACT_NAME=cycode-mac-onedir" >> $GITHUB_ENV - - - name: Prepare files on Apple Silicon macOS (onedir) - if: runner.os == 'macOS' && runner.arch == 'ARM64' && matrix.mode == 'onedir' - run: | - echo "ARTIFACT_NAME=cycode-mac-arm-onedir" >> $GITHUB_ENV - - - name: Prepare files on Linux - if: runner.os == 'Linux' - run: | - echo "ARTIFACT_NAME=cycode-linux" >> $GITHUB_ENV - mv dist/cycode-cli dist/cycode-linux - sha256sum dist/cycode-linux > sha256 - head -c 64 sha256 > dist/cycode-linux.sha256 + env: + PROCESS_ONEDIR: ${{ matrix.mode == 'onedir' }} + run: echo "ARTIFACT_NAME=$(./process_executable_file.py dist/cycode-cli.exe)" >> $GITHUB_ENV - name: Upload files as artifact uses: actions/upload-artifact@v3 diff --git a/process_executable_file.py b/process_executable_file.py new file mode 100755 index 00000000..827cca82 --- /dev/null +++ b/process_executable_file.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 + +""" +Used in the GitHub Actions workflow (build_executable.yml) to process the executable file. +This script calculates hash and renames executable file depending on the OS, arch, and build mode. +It also creates a file with the hash of the executable file. +It uses SHA256 algorithm to calculate the hash. +It returns the name of the executable file which is used as artifact name. +""" + +import argparse +import hashlib +import os +import platform +from pathlib import Path +from string import Template +from typing import List, Tuple, Union + + +def get_hash_of_file(file_path: Union[str, Path]) -> str: + with open(file_path, 'rb') as f: + return hashlib.sha256(f.read()).hexdigest() + + +_HASH_FILE_EXT = '.sha256' + +DirHashes = List[Tuple[str, str]] + + +def calculate_hash_of_every_file_in_the_directory(dir_path: Path) -> DirHashes: + hashes = [] + + for root, _, files in os.walk(dir_path): + for file in files: + file_path = os.path.join(root, file) + file_hash = get_hash_of_file(file_path) + hashes.append((file_hash, file_path)) + + # sort by file path + hashes.sort(key=lambda x: x[1]) + + return hashes + + +_OS_TO_CLI_DIST_TEMPLATE = { + 'darwin': Template('cycode-mac$suffix$ext'), + 'linux': Template('cycode-linux$suffix$ext'), + 'windows': Template('cycode-win$suffix.exe$ext'), +} + + +def is_arm() -> bool: + return platform.machine().lower() in ('arm', 'arm64', 'aarch64') + + +def get_os_name() -> str: + return platform.system().lower() + + +def get_cli_file_name(suffix: str = '', ext: str = '') -> str: + os_name = get_os_name() + if os_name not in _OS_TO_CLI_DIST_TEMPLATE: + raise Exception(f'Unsupported OS: {os_name}') + + template = _OS_TO_CLI_DIST_TEMPLATE[os_name] + return template.substitute(suffix=suffix, ext=ext) + + +def get_cli_file_suffix(arm: bool, onedir: bool) -> str: + suffixes = [] + + if arm: + suffixes.append('-arm') + if onedir: + suffixes.append('-onedir') + + return ''.join(suffixes) + + +def write_hash_to_file(file_hash: str, output_path: str) -> None: + with open(output_path, 'w') as f: + f.write(file_hash) + + +def write_hashes_db_to_file(hashes: DirHashes, output_path: str) -> None: + content = '' + for file_hash, file_path in hashes: + content += f'{file_hash} {file_path}\n' + + with open(output_path, 'w') as f: + f.write(content) + + +def get_cli_filename(arm: bool, onedir: bool) -> str: + return get_cli_file_name(get_cli_file_suffix(arm, onedir)) + + +def get_cli_path(output_path: Path, arm: bool, onedir: bool) -> str: + return os.path.join(output_path, get_cli_filename(arm, onedir)) + + +def get_cli_hash_filename(arm: bool, onedir: bool) -> str: + return get_cli_file_name(suffix=get_cli_file_suffix(arm, onedir), ext=_HASH_FILE_EXT) + + +def get_cli_hash_path(output_path: Path, arm: bool, onedir: bool) -> str: + return os.path.join(output_path, get_cli_hash_filename(arm, onedir)) + + +def process_executable_file(input_path: Path, arm: bool, onedir: bool) -> str: + output_path = input_path.parent + hash_file_path = get_cli_hash_path(output_path, arm, onedir) + + if onedir: + hashes = calculate_hash_of_every_file_in_the_directory(input_path) + write_hashes_db_to_file(hashes, hash_file_path) + else: + file_hash = get_hash_of_file(input_path) + write_hash_to_file(file_hash, hash_file_path) + + if not onedir: + os.rename(input_path, get_cli_path(output_path, arm, onedir)) + + return get_cli_filename(arm, onedir) + + +def parse_bool(value: Union[str, bool]) -> bool: + if isinstance(value, bool): + return value + + if value.lower() in ('true', '1'): + return True + if value.lower() in ('false', '0'): + return False + + raise ValueError(f'Invalid value: {value}') + + +def main() -> None: + parser = argparse.ArgumentParser() + + parser.add_argument('--onedir', '-o', action='store_true', help='One directory mode') + parser.add_argument('--arm', '-a', action='store_true', help='Is it ARM arch') + parser.add_argument('input', help='Path to executable or directory') + + args = parser.parse_args() + onedir = args.onedir or parse_bool(os.environ.get('PROCESS_ONEDIR', False)) + arm = args.arm or parse_bool(os.environ.get('PROCESS_ARM', False)) or is_arm() + + artifact_name = process_executable_file(Path(args.input), arm, onedir) + + print(artifact_name) # noqa: T201 + + +if __name__ == '__main__': + main() From 09bc1103060e06118edff73fb404575c9865e9eb Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Mon, 8 Jan 2024 13:54:49 +0100 Subject: [PATCH 2/8] simplify workflow --- .github/workflows/build_executable.yml | 6 ------ process_executable_file.py | 10 ++++++++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_executable.yml b/.github/workflows/build_executable.yml index 6acb9ac7..04e31ece 100644 --- a/.github/workflows/build_executable.yml +++ b/.github/workflows/build_executable.yml @@ -194,12 +194,6 @@ jobs: PROCESS_ONEDIR: ${{ matrix.mode == 'onedir' }} run: echo "ARTIFACT_NAME=$(./process_executable_file.py dist/cycode-cli)" >> $GITHUB_ENV - - name: Prepare files on Windows - if: runner.os == 'Windows' - env: - PROCESS_ONEDIR: ${{ matrix.mode == 'onedir' }} - run: echo "ARTIFACT_NAME=$(./process_executable_file.py dist/cycode-cli.exe)" >> $GITHUB_ENV - - name: Upload files as artifact uses: actions/upload-artifact@v3 with: diff --git a/process_executable_file.py b/process_executable_file.py index 827cca82..6013e601 100755 --- a/process_executable_file.py +++ b/process_executable_file.py @@ -118,7 +118,7 @@ def process_executable_file(input_path: Path, arm: bool, onedir: bool) -> str: file_hash = get_hash_of_file(input_path) write_hash_to_file(file_hash, hash_file_path) - if not onedir: + # for example rename cycode-cli to cycode-mac or cycode-mac-arm-onedir os.rename(input_path, get_cli_path(output_path, arm, onedir)) return get_cli_filename(arm, onedir) @@ -144,10 +144,16 @@ def main() -> None: parser.add_argument('input', help='Path to executable or directory') args = parser.parse_args() + onedir = args.onedir or parse_bool(os.environ.get('PROCESS_ONEDIR', False)) arm = args.arm or parse_bool(os.environ.get('PROCESS_ARM', False)) or is_arm() - artifact_name = process_executable_file(Path(args.input), arm, onedir) + input_path = Path(args.input) + if get_os_name() == 'windows' and not onedir and input_path.suffix != '.exe': + # add .exe on windows if was missed (to simplify GHA workflow) + input_path = input_path.with_suffix('.exe') + + artifact_name = process_executable_file(input_path, arm, onedir) print(artifact_name) # noqa: T201 From bf7c627aa0b2c7caf1b5361e9bcbd379e158e8a6 Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Mon, 8 Jan 2024 14:06:34 +0100 Subject: [PATCH 3/8] remove test branch from triggers --- .github/workflows/build_executable.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_executable.yml b/.github/workflows/build_executable.yml index 04e31ece..82735a80 100644 --- a/.github/workflows/build_executable.yml +++ b/.github/workflows/build_executable.yml @@ -5,7 +5,6 @@ on: push: branches: - main - - CM-30406-attach-onedir-cli-to-git-hub-releases-with-checksums permissions: contents: write @@ -189,7 +188,7 @@ jobs: :: verify signature signtool.exe verify /v /pa ".\dist\cycode-cli.exe" - - name: Prepare files + - name: Prepare files for artifact and release (rename and calculate sha256) env: PROCESS_ONEDIR: ${{ matrix.mode == 'onedir' }} run: echo "ARTIFACT_NAME=$(./process_executable_file.py dist/cycode-cli)" >> $GITHUB_ENV From 89871edf5e06a08ac4d1c38b9690ef2d5499338f Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Mon, 8 Jan 2024 14:35:50 +0100 Subject: [PATCH 4/8] remove env vars --- .github/workflows/build_executable.yml | 3 +- process_executable_file.py | 57 +++++++++----------------- 2 files changed, 21 insertions(+), 39 deletions(-) diff --git a/.github/workflows/build_executable.yml b/.github/workflows/build_executable.yml index 82735a80..308db310 100644 --- a/.github/workflows/build_executable.yml +++ b/.github/workflows/build_executable.yml @@ -5,6 +5,7 @@ on: push: branches: - main + - CM-30406-attach-onedir-cli-to-git-hub-releases-with-checksums permissions: contents: write @@ -189,8 +190,6 @@ jobs: signtool.exe verify /v /pa ".\dist\cycode-cli.exe" - name: Prepare files for artifact and release (rename and calculate sha256) - env: - PROCESS_ONEDIR: ${{ matrix.mode == 'onedir' }} run: echo "ARTIFACT_NAME=$(./process_executable_file.py dist/cycode-cli)" >> $GITHUB_ENV - name: Upload files as artifact diff --git a/process_executable_file.py b/process_executable_file.py index 6013e601..9455bd93 100755 --- a/process_executable_file.py +++ b/process_executable_file.py @@ -66,12 +66,12 @@ def get_cli_file_name(suffix: str = '', ext: str = '') -> str: return template.substitute(suffix=suffix, ext=ext) -def get_cli_file_suffix(arm: bool, onedir: bool) -> str: +def get_cli_file_suffix(is_onedir: bool) -> str: suffixes = [] - if arm: + if is_arm(): suffixes.append('-arm') - if onedir: + if is_onedir: suffixes.append('-onedir') return ''.join(suffixes) @@ -91,27 +91,27 @@ def write_hashes_db_to_file(hashes: DirHashes, output_path: str) -> None: f.write(content) -def get_cli_filename(arm: bool, onedir: bool) -> str: - return get_cli_file_name(get_cli_file_suffix(arm, onedir)) +def get_cli_filename(is_onedir: bool) -> str: + return get_cli_file_name(get_cli_file_suffix(is_onedir)) -def get_cli_path(output_path: Path, arm: bool, onedir: bool) -> str: - return os.path.join(output_path, get_cli_filename(arm, onedir)) +def get_cli_path(output_path: Path, is_onedir: bool) -> str: + return os.path.join(output_path, get_cli_filename(is_onedir)) -def get_cli_hash_filename(arm: bool, onedir: bool) -> str: - return get_cli_file_name(suffix=get_cli_file_suffix(arm, onedir), ext=_HASH_FILE_EXT) +def get_cli_hash_filename(is_onedir: bool) -> str: + return get_cli_file_name(suffix=get_cli_file_suffix(is_onedir), ext=_HASH_FILE_EXT) -def get_cli_hash_path(output_path: Path, arm: bool, onedir: bool) -> str: - return os.path.join(output_path, get_cli_hash_filename(arm, onedir)) +def get_cli_hash_path(output_path: Path, is_onedir: bool) -> str: + return os.path.join(output_path, get_cli_hash_filename(is_onedir)) -def process_executable_file(input_path: Path, arm: bool, onedir: bool) -> str: +def process_executable_file(input_path: Path, is_onedir: bool) -> str: output_path = input_path.parent - hash_file_path = get_cli_hash_path(output_path, arm, onedir) + hash_file_path = get_cli_hash_path(output_path, is_onedir) - if onedir: + if is_onedir: hashes = calculate_hash_of_every_file_in_the_directory(input_path) write_hashes_db_to_file(hashes, hash_file_path) else: @@ -119,41 +119,24 @@ def process_executable_file(input_path: Path, arm: bool, onedir: bool) -> str: write_hash_to_file(file_hash, hash_file_path) # for example rename cycode-cli to cycode-mac or cycode-mac-arm-onedir - os.rename(input_path, get_cli_path(output_path, arm, onedir)) + os.rename(input_path, get_cli_path(output_path, is_onedir)) - return get_cli_filename(arm, onedir) - - -def parse_bool(value: Union[str, bool]) -> bool: - if isinstance(value, bool): - return value - - if value.lower() in ('true', '1'): - return True - if value.lower() in ('false', '0'): - return False - - raise ValueError(f'Invalid value: {value}') + return get_cli_filename(is_onedir) def main() -> None: parser = argparse.ArgumentParser() - - parser.add_argument('--onedir', '-o', action='store_true', help='One directory mode') - parser.add_argument('--arm', '-a', action='store_true', help='Is it ARM arch') parser.add_argument('input', help='Path to executable or directory') args = parser.parse_args() - - onedir = args.onedir or parse_bool(os.environ.get('PROCESS_ONEDIR', False)) - arm = args.arm or parse_bool(os.environ.get('PROCESS_ARM', False)) or is_arm() - input_path = Path(args.input) - if get_os_name() == 'windows' and not onedir and input_path.suffix != '.exe': + is_onedir = input_path.is_dir() + + if get_os_name() == 'windows' and not is_onedir and input_path.suffix != '.exe': # add .exe on windows if was missed (to simplify GHA workflow) input_path = input_path.with_suffix('.exe') - artifact_name = process_executable_file(input_path, arm, onedir) + artifact_name = process_executable_file(input_path, is_onedir) print(artifact_name) # noqa: T201 From 99dc01018d36f9cacac1606c66cd3f031ab1c99a Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Mon, 8 Jan 2024 14:39:59 +0100 Subject: [PATCH 5/8] remove test trigger --- .github/workflows/build_executable.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_executable.yml b/.github/workflows/build_executable.yml index 308db310..dcf2a42b 100644 --- a/.github/workflows/build_executable.yml +++ b/.github/workflows/build_executable.yml @@ -5,7 +5,6 @@ on: push: branches: - main - - CM-30406-attach-onedir-cli-to-git-hub-releases-with-checksums permissions: contents: write From cff2b19297cca66a9c514e71417901597b55ce0a Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Mon, 8 Jan 2024 14:49:54 +0100 Subject: [PATCH 6/8] make filepath relative in hashes DB --- process_executable_file.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/process_executable_file.py b/process_executable_file.py index 9455bd93..aaffd026 100755 --- a/process_executable_file.py +++ b/process_executable_file.py @@ -34,7 +34,9 @@ def calculate_hash_of_every_file_in_the_directory(dir_path: Path) -> DirHashes: for file in files: file_path = os.path.join(root, file) file_hash = get_hash_of_file(file_path) - hashes.append((file_hash, file_path)) + + relative_file_path = file_path[file_path.find(dir_path.name):] + hashes.append((file_hash, relative_file_path)) # sort by file path hashes.sort(key=lambda x: x[1]) From 7499cde74b338c49e7d5eb36c8fa60202fb3ceaf Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Tue, 9 Jan 2024 11:39:19 +0100 Subject: [PATCH 7/8] move constants to the top --- process_executable_file.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/process_executable_file.py b/process_executable_file.py index aaffd026..37886362 100755 --- a/process_executable_file.py +++ b/process_executable_file.py @@ -16,17 +16,21 @@ from string import Template from typing import List, Tuple, Union +_HASH_FILE_EXT = '.sha256' +_OS_TO_CLI_DIST_TEMPLATE = { + 'darwin': Template('cycode-mac$suffix$ext'), + 'linux': Template('cycode-linux$suffix$ext'), + 'windows': Template('cycode-win$suffix.exe$ext'), +} + +DirHashes = List[Tuple[str, str]] + def get_hash_of_file(file_path: Union[str, Path]) -> str: with open(file_path, 'rb') as f: return hashlib.sha256(f.read()).hexdigest() -_HASH_FILE_EXT = '.sha256' - -DirHashes = List[Tuple[str, str]] - - def calculate_hash_of_every_file_in_the_directory(dir_path: Path) -> DirHashes: hashes = [] @@ -44,13 +48,6 @@ def calculate_hash_of_every_file_in_the_directory(dir_path: Path) -> DirHashes: return hashes -_OS_TO_CLI_DIST_TEMPLATE = { - 'darwin': Template('cycode-mac$suffix$ext'), - 'linux': Template('cycode-linux$suffix$ext'), - 'windows': Template('cycode-win$suffix.exe$ext'), -} - - def is_arm() -> bool: return platform.machine().lower() in ('arm', 'arm64', 'aarch64') From e7c5ad4322c8daac05deb23fd7342cd67edad8e1 Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Tue, 9 Jan 2024 11:56:23 +0100 Subject: [PATCH 8/8] refactoring --- process_executable_file.py | 44 +++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/process_executable_file.py b/process_executable_file.py index 37886362..eafeada3 100755 --- a/process_executable_file.py +++ b/process_executable_file.py @@ -22,6 +22,8 @@ 'linux': Template('cycode-linux$suffix$ext'), 'windows': Template('cycode-win$suffix.exe$ext'), } +_WINDOWS = 'windows' +_WINDOWS_EXECUTABLE_SUFFIX = '.exe' DirHashes = List[Tuple[str, str]] @@ -31,21 +33,38 @@ def get_hash_of_file(file_path: Union[str, Path]) -> str: return hashlib.sha256(f.read()).hexdigest() -def calculate_hash_of_every_file_in_the_directory(dir_path: Path) -> DirHashes: +def get_hashes_of_many_files(root: str, file_paths: List[str]) -> DirHashes: + hashes = [] + + for file_path in file_paths: + file_path = os.path.join(root, file_path) + file_hash = get_hash_of_file(file_path) + + hashes.append((file_hash, file_path)) + + return hashes + + +def get_hashes_of_every_file_in_the_directory(dir_path: Path) -> DirHashes: hashes = [] for root, _, files in os.walk(dir_path): - for file in files: - file_path = os.path.join(root, file) - file_hash = get_hash_of_file(file_path) + hashes.extend(get_hashes_of_many_files(root, files,)) - relative_file_path = file_path[file_path.find(dir_path.name):] - hashes.append((file_hash, relative_file_path)) + return hashes + + +def normalize_hashes_db(hashes: DirHashes, dir_path: Path) -> DirHashes: + normalized_hashes = [] + + for file_hash, file_path in hashes: + relative_file_path = file_path[file_path.find(dir_path.name):] + normalized_hashes.append((file_hash, relative_file_path)) # sort by file path - hashes.sort(key=lambda x: x[1]) + normalized_hashes.sort(key=lambda hash_item: hash_item[1]) - return hashes + return normalized_hashes def is_arm() -> bool: @@ -111,8 +130,9 @@ def process_executable_file(input_path: Path, is_onedir: bool) -> str: hash_file_path = get_cli_hash_path(output_path, is_onedir) if is_onedir: - hashes = calculate_hash_of_every_file_in_the_directory(input_path) - write_hashes_db_to_file(hashes, hash_file_path) + hashes = get_hashes_of_every_file_in_the_directory(input_path) + normalized_hashes = normalize_hashes_db(hashes, input_path) + write_hashes_db_to_file(normalized_hashes, hash_file_path) else: file_hash = get_hash_of_file(input_path) write_hash_to_file(file_hash, hash_file_path) @@ -131,9 +151,9 @@ def main() -> None: input_path = Path(args.input) is_onedir = input_path.is_dir() - if get_os_name() == 'windows' and not is_onedir and input_path.suffix != '.exe': + if get_os_name() == _WINDOWS and not is_onedir and input_path.suffix != _WINDOWS_EXECUTABLE_SUFFIX: # add .exe on windows if was missed (to simplify GHA workflow) - input_path = input_path.with_suffix('.exe') + input_path = input_path.with_suffix(_WINDOWS_EXECUTABLE_SUFFIX) artifact_name = process_executable_file(input_path, is_onedir)