fix

zhusun890 · Nov 8, 2020 · 9fbeae4 · 9fbeae4
1 parent 944f1cf
commit 9fbeae4
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 11 deletions.
diff --git a/.vs/slnx.sqlite b/.vs/slnx.sqlite
diff --git a/LanguageNetwork/GPT2/scripts/formatter.py b/LanguageNetwork/GPT2/scripts/formatter.py
@@ -84,17 +84,17 @@ def coarse_formatter(text):
 	if(lens >= 10):
 		while count < lens - 5:
 			#print("para: ", para ," | final: ", lens - 8)
-			paras.append(text_list[para:para+5])
+			paras.append(text_list[count:count+5])
 			count += 5
 		# print("现在添加段尾：", text_list[para:-1])
 		if count == lens - 1:
 			pass
 		else:
 			paras.append(text_list[count:-1])
 	else:
-		paras.append(3)
-		paras.append(lens - 5)
-		paras.append(lens - 3)
+		paras.append(text_list[:3])
+		paras.append(text_list[3:lens - 5])
+		paras.append(text_list[lens - 5:lens])
 	# print("最终段落为：", paras)
 	for para in paras:
 		# print("paras: ", para)

diff --git a/LanguageNetwork/GPT2/scripts/gdown.pl b/LanguageNetwork/GPT2/scripts/gdown.pl
@@ -1,11 +1,12 @@
+
 #!/usr/bin/env perl
 #
 # Google Drive direct download of big files
 # ./gdown.pl 'gdrive file url' ['desired file name']
 #
 # v1.0 by circulosmeos 04-2014.
 # v1.1 by circulosmeos 01-2017.
-# v1.2, v1.3, v1.4 by circulosmeos 01-2019, 02-2019.
+# v1.2, 2.0 by circulosmeos 01-2019.
 # //circulosmeos.wordpress.com/2014/04/12/google-drive-direct-download-of-big-files
 # Distributed under GPL 3 (//www.gnu.org/licenses/gpl-3.0.html)
 #
@@ -22,7 +23,7 @@
 die "\n./gdown.pl 'gdrive file url' [desired file name]\n\n" if $URL eq '';
 
 my $FILENAME=shift;
-$FILENAME='gdown.'.strftime("%Y%m%d%H%M%S", localtime).'.'.substr(rand,2) if $FILENAME eq '';
+my $TEMP_FILENAME='gdown.'.strftime("%Y%m%d%H%M%S", localtime).'.'.substr(rand,2);
 
 if ($URL=~m#^https?://drive.google.com/file/d/([^/]+)#) {
     $URL="https://docs.google.com/uc?id=$1&export=download";
@@ -33,8 +34,8 @@
 
 execute_command();
 
-while (-s $FILENAME < 100000) { # only if the file isn't the download yet
-    open fFILENAME, '<', $FILENAME;
+while (-s $TEMP_FILENAME < 100000) { # only if the file isn't the download yet
+    open fFILENAME, '<', $TEMP_FILENAME;
     $check=0;
     foreach (<fFILENAME>) {
         if (/href="(\/uc\?export=download[^"]+)/) {
@@ -63,13 +64,40 @@
     $URL=~s/confirm=([^;&]+)/confirm=$confirm/ if $confirm ne '';
 
     execute_command();
+
 }
 
 unlink $TEMP;
 
 sub execute_command() {
-    $COMMAND="wget -q --show-progress --no-check-certificate --load-cookie $TEMP --save-cookie $TEMP \"$URL\"";
+    my $OUTPUT_FILENAME = $TEMP_FILENAME;
+    my $CONTINUE = '';
+
+    # check contents before download & if a $FILENAME has been indicated resume on content download
+    # please, note that for this to work, wget must correctly provide --spider with --server-response (-S)
+    if ( length($FILENAME) > 0 ) {
+        $COMMAND="wget -q -S --no-check-certificate --spider --load-cookie $TEMP --save-cookie $TEMP \"$URL\" 2>&1";
+        my @HEADERS=`$COMMAND`;
+        foreach my $header (@HEADERS) {
+            if ( $header =~ /Content-Type: (.+)/ ) {
+                if ( $1 !~ 'text/html' ) {
+                    $OUTPUT_FILENAME = $FILENAME;
+                    $CONTINUE = '-c';
+                }
+            }
+        }
+    }
+
+    $COMMAND="wget $CONTINUE --progress=dot:giga --no-check-certificate --load-cookie $TEMP --save-cookie $TEMP \"$URL\"";
     $COMMAND.=" -O \"$FILENAME\"" if $FILENAME ne '';
-    system ( $COMMAND );
+
+    my $OUTPUT = system( $COMMAND );
+    if ( $OUTPUT == 2 ) { # do a clean exit with Ctrl+C
+        unlink $TEMP;
+        die "\nDownloading interrupted by user\n\n";
+    } elsif ( $OUTPUT == 0 && length($CONTINUE)>0 ) { # do a clean exit with $FILENAME provided
+        unlink $TEMP;
+        die "\nDownloading complete\n\n";
+    }
     return 1;
 }
diff --git a/colab_online.ipynb b/colab_online.ipynb
@@ -52,7 +52,7 @@
         "!mkdir -p /home/EssayKiller_V2/LanguageNetwork/GPT2/finetune/trained_models\n",
         "\n",
         "%cd /home/EssayKiller_V2/LanguageNetwork/GPT2/finetune/\n",
-        "!perl /home/EssayKiller_V2/LanguageNetwork/GPT2/scripts/gdown.pl https://drive.google.com/open?id=1ujWYTOvRLGJX0raH-f-lPZa3-RN58ZQx trained_models/model.ckpt-280000.data-00000-of-00001\n",
+        "!perl /home/EssayKiller_V2/LanguageNetwork/GPT2/scripts/gdown.pl https://drive.google.com/file/d/1ujWYTOvRLGJX0raH-f-lPZa3-RN58ZQx trained_models/model.ckpt-280000.data-00000-of-00001\n",
         "!wget -q --show-progress https://github.com/EssayKillerBrain/EssayKiller/releases/download/v1.0/model.ckpt-280000.index -P /home/EssayKiller_V2/LanguageNetwork/GPT2/finetune/trained_models\n",
         "!wget -q --show-progress https://github.com/EssayKillerBrain/EssayKiller/releases/download/v1.0/model.ckpt-280000.meta -P /home/EssayKiller_V2/LanguageNetwork/GPT2/finetune/trained_models\n",
         "\n",