From bb4202b0ad9f52c76997b3ae39bca58d2ccb60b3 Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Fri, 16 Aug 2024 12:17:20 +0800 Subject: [PATCH] fix transformers Signed-off-by: Sun, Xuehao --- .github/workflows/scripts/freeze_requirements.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scripts/freeze_requirements.sh b/.github/workflows/scripts/freeze_requirements.sh index 2b87622487..3e60e4464d 100644 --- a/.github/workflows/scripts/freeze_requirements.sh +++ b/.github/workflows/scripts/freeze_requirements.sh @@ -15,13 +15,17 @@ function freeze() { --output-file "$folder/freeze.txt" \ "$file" echo "::endgroup::" + if [[ -e "$folder/freeze.txt" ]]; then if [[ "$keep_origin_packages" == "true" ]]; then + # fix corner cases sed -i '/^\s*#/d; s/#.*//; /^\s*$/d; s/ //g' "$file" sed -i '/^\s*#/d; s/#.*//; /^\s*$/d; s/ //g; s/huggingface-hub\[inference\]/huggingface-hub/g; s/uvicorn\[standard\]/uvicorn/g' "$folder/freeze.txt" - + if grep -q '^transformers$' $file && ! grep -q '^transformers\[sentencepiece\]$' $file; then + sed -i "s/transformers\[sentencepiece\]/transformers/" "$folder/freeze.txt" + fi packages1=$(tr '><' '=' <"$file" | cut -d'=' -f1 | tr '[:upper:]' '[:lower:]' | sed 's/[-_]/-/g') - packages2=$(cut -d'=' -f1 "$folder/freeze.txt" | tr '[:upper:]' '[:lower:]' | sed 's/[-_]/-/g' ) + packages2=$(cut -d'=' -f1 "$folder/freeze.txt" | tr '[:upper:]' '[:lower:]' | sed 's/[-_]/-/g') common_packages=$(comm -12 <(echo "$packages2" | sort) <(echo "$packages1" | sort)) grep '^git\+' "$file" >temp_file || touch temp_file rm -rf "$file" && mv temp_file "$file" @@ -51,6 +55,7 @@ function check_branch_name() { function main() { check_branch_name + upgrade_transformers echo "::group::pip install pip-tools" && pip install pip-tools --upgrade && echo "::endgroup::" export -f freeze find . -name "requirements.txt" | xargs -n 1 -I {} bash -c 'freeze "$@"' _ {}