chore(Worker): Add isort formatting

worker/Makefile

@@ -15,10 +15,12 @@ venv: requirements.txt requirements-dev.txt package.json
 # Format code
 format: venv
 	$(VE)/black .
+	$(VE)/isort .
 
 # Lint code
 lint: venv
 	$(VE)/black --check .
+	$(VE)/isort --check .
 	$(VE)/flake8
 	$(VE)/mypy --config-file pyproject.toml .
 	$(VE)/pydocstyle --match-dir='^(?!venv)'

worker/conftest.py

@@ -1,7 +1,7 @@
 """Configuration for pytest."""
 
-from testfixtures import TempDirectory
 import pytest
+from testfixtures import TempDirectory
 
 
 @pytest.fixture()

worker/jobs/archive.py

@@ -1,6 +1,6 @@
-from typing import Optional, List, Union, Dict
 import os
 import shutil
+from typing import Dict, List, Optional, Union
 
 from config import get_snapshot_dir
 from jobs.base.job import Job

worker/jobs/base/job.py

@@ -1,14 +1,14 @@
-from datetime import datetime
-from typing import Any, Optional
 import os
 import traceback
+from datetime import datetime
 from pathlib import Path
+from typing import Any, Optional
 
 import celery
+import sentry_sdk
 from celery import states
 from celery.exceptions import Ignore, SoftTimeLimitExceeded
 from celery.utils.log import get_task_logger
-import sentry_sdk
 from sentry_sdk.integrations.celery import CeleryIntegration
 from sentry_sdk.integrations.redis import RedisIntegration
 

worker/jobs/base/job_test.py

@@ -3,7 +3,7 @@
 import pytest
 from celery.exceptions import Ignore
 
-from .job import Job, DEBUG, INFO, WARN, ERROR
+from .job import DEBUG, ERROR, INFO, WARN, Job
 
 
 def test_logging():

worker/jobs/base/subprocess_job.py

@@ -4,7 +4,7 @@
 import threading
 from typing import List, Optional, Union
 
-from .job import Job, DEBUG, INFO, WARN, ERROR
+from .job import DEBUG, ERROR, INFO, WARN, Job
 
 
 class SubprocessJob(Job):

worker/jobs/base/subprocess_job_test.py

@@ -4,8 +4,8 @@
 import pytest
 from celery.exceptions import Ignore
 
+from .job import DEBUG, ERROR, INFO, WARN, Job
 from .subprocess_job import SubprocessJob
-from .job import Job, DEBUG, INFO, WARN, ERROR
 
 
 def test_success():

worker/jobs/convert.py

@@ -1,5 +1,5 @@
-from typing import cast, List, Union, Dict
 import os
+from typing import Dict, List, Union, cast
 
 from config import get_node_modules_bin
 from jobs.base.subprocess_job import SubprocessJob

worker/jobs/decode.py

@@ -1,5 +1,5 @@
 import json
-from typing import cast, List, Union, Dict
+from typing import Dict, List, Union, cast
 
 from .convert import Convert
 

worker/jobs/encode.py

@@ -1,5 +1,5 @@
 import json
-from typing import Any, Union, Dict
+from typing import Any, Dict, Union
 
 from .convert import Convert
 

worker/jobs/pin.py

@@ -1,12 +1,11 @@
 import os
 import re
-from typing import cast, Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple, cast
 
 from dxf import DXF
 
 from jobs.base.job import Job
 
-
 # Regex for parsing a container image identifier
 # Based on answers at https://stackoverflow.com/questions/39671641/regex-to-parse-docker-tag
 # Playground at https://regex101.com/r/hP8bK1/42

worker/jobs/pull/__init__.py

@@ -1,5 +1,5 @@
-from typing import Callable, Dict, List
 import os
+from typing import Callable, Dict, List
 
 import config
 from jobs.base.job import Job
@@ -8,10 +8,10 @@
 from .gdoc import pull_gdoc
 from .gdrive import pull_gdrive
 from .github import pull_github
+from .helpers import Files
 from .http import pull_http
 from .plos import pull_plos
 from .upload import pull_upload
-from .helpers import Files
 
 # Functions for pulling individual source types
 PULL_FUNCS: Dict[str, Callable[..., Files]] = {

worker/jobs/pull/elife.py

@@ -1,12 +1,12 @@
 import io
 import os
 import re
-import requests
 from typing import List
 
+import requests
 from lxml import etree
 
-from .helpers import HttpSession, Files, begin_pull, end_pull
+from .helpers import Files, HttpSession, begin_pull, end_pull
 
 
 def pull_elife(source: dict, working_dir: str, path: str, **kwargs) -> Files:

worker/jobs/pull/gdoc.py

@@ -5,7 +5,7 @@
 from googleapiclient.discovery import build
 from oauth2client.client import GoogleCredentials
 
-from util.files import file_info, Files
+from util.files import Files, file_info
 
 
 def pull_gdoc(source: dict, path: str, secrets: Dict = {}, **kwargs) -> Files:

worker/jobs/pull/gdoc_test.py

@@ -1,7 +1,9 @@
 import os
+
 import pytest
 
 from util.working_directory import working_directory
+
 from .gdoc import pull_gdoc
 
 # The following access token is expired, but was valid when this test

worker/jobs/pull/gdrive.py

@@ -2,22 +2,17 @@
 import os
 import shutil
 import typing
+from typing import List
 
 from googleapiclient.discovery import build
 from googleapiclient.http import MediaIoBaseDownload
 from oauth2client.client import GoogleCredentials
-from typing import List
 
-from .helpers import begin_pull, end_pull, Files
+from util.path_operations import (utf8_isdir, utf8_makedirs, utf8_normpath,
+                                  utf8_path_exists, utf8_path_join,
+                                  utf8_unlink)
 
-from util.path_operations import (
-    utf8_path_join,
-    utf8_normpath,
-    utf8_makedirs,
-    utf8_path_exists,
-    utf8_isdir,
-    utf8_unlink,
-)
+from .helpers import Files, begin_pull, end_pull
 
 
 def pull_gdrive(source: dict, working_dir: str, path: str, **kwargs) -> Files:

worker/jobs/pull/gdrive_test.py

@@ -1,7 +1,9 @@
 import os
+
 import pytest
 
 from util.path_operations import utf8_path_exists, utf8_path_join
+
 from .gdrive import pull_gdrive
 
 # The following access token is expired, but was valid when this test

worker/jobs/pull/github.py

@@ -1,22 +1,17 @@
 import json
 import os
 import shutil
+from io import BytesIO
+from typing import List
 
 from github import Github
 from github.ContentFile import ContentFile
-from io import BytesIO
-from typing import List
 
-from util.path_operations import (
-    utf8_path_join,
-    utf8_normpath,
-    utf8_makedirs,
-    utf8_path_exists,
-    utf8_isdir,
-    utf8_unlink,
-)
+from util.path_operations import (utf8_isdir, utf8_makedirs, utf8_normpath,
+                                  utf8_path_exists, utf8_path_join,
+                                  utf8_unlink)
 
-from .helpers import begin_pull, end_pull, Files
+from .helpers import Files, begin_pull, end_pull
 
 
 def pull_github(source: dict, working_dir: str, path: str, **kwargs) -> Files:

worker/jobs/pull/github_test.py

@@ -1,4 +1,5 @@
 import os
+
 import pytest
 
 from .github import pull_github

worker/jobs/pull/helpers.py

@@ -1,15 +1,16 @@
-from typing import Dict, Any
-from pathlib import Path
-import tempfile
 import ipaddress
 import mimetypes
 import os
 import re
-import requests
 import shutil
+import tempfile
+from pathlib import Path
 from socket import gethostbyname
+from typing import Any, Dict
 from urllib.parse import urlparse
 
+import requests
+
 from util.files import Files, list_files
 
 

worker/jobs/pull/helpers_test.py

@@ -1,5 +1,6 @@
 import os
 from pathlib import Path
+
 import pytest
 
 from .helpers import begin_pull, end_pull

worker/jobs/pull/http.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 from typing import List
 
-from .helpers import HttpSession, begin_pull, end_pull, Files
+from .helpers import Files, HttpSession, begin_pull, end_pull
 
 
 def pull_http(source: dict, working_dir: str, path: str, **kwargs) -> Files:

worker/jobs/pull/plos.py

@@ -5,7 +5,7 @@
 
 from lxml import etree
 
-from .helpers import HttpSession, begin_pull, end_pull, Files
+from .helpers import Files, HttpSession, begin_pull, end_pull
 
 
 def pull_plos(source: dict, working_dir: str, path: str, **kwargs) -> Files:

worker/jobs/push/__init__.py

@@ -1,5 +1,5 @@
-from typing import Callable, Dict, List
 import os
+from typing import Callable, Dict, List
 
 import config
 from jobs.base.job import Job

worker/jobs/push/gdoc.py

@@ -1,11 +1,11 @@
 import json
 import os
 import tempfile
+from typing import List
 
 from googleapiclient.discovery import build
 from googleapiclient.http import MediaFileUpload
 from oauth2client.client import GoogleCredentials
-from typing import List
 
 from jobs.convert import Convert
 

worker/jobs/push/gdoc_test.py

@@ -1,7 +1,7 @@
 import json
 import os
-import pytest
 
+import pytest
 from googleapiclient.discovery import build
 from oauth2client.client import GoogleCredentials
 

worker/jobs/session/kubernetes_session.py

@@ -6,8 +6,8 @@
 import sys
 import time
 
-from celery.exceptions import Ignore, SoftTimeLimitExceeded
 import kubernetes
+from celery.exceptions import Ignore, SoftTimeLimitExceeded
 
 from config import get_snapshot_dir, get_working_dir
 from jobs.base.job import Job

worker/jobs/session/kubernetes_session_test.py

@@ -1,7 +1,7 @@
 import pytest
 from celery.exceptions import SoftTimeLimitExceeded
 
-from .kubernetes_session import api_instance, KubernetesSession
+from .kubernetes_session import KubernetesSession, api_instance
 
 
 @pytest.mark.skipif(api_instance is None, reason="can only run if K8s is available")

worker/jobs/session/session.py

@@ -1,6 +1,6 @@
 from typing import Type, Union
 
-from .kubernetes_session import api_instance, KubernetesSession
+from .kubernetes_session import KubernetesSession, api_instance
 from .subprocess_session import SubprocessSession
 
 # If on a K8s is available then use that

worker/jobs/sleep.py

@@ -1,5 +1,5 @@
-from datetime import datetime
 import time
+from datetime import datetime
 
 from jobs.base.job import Job
 

worker/requirements-dev.txt

@@ -1,5 +1,6 @@
 black==19.10b0
 flake8==3.8.4
+isort==5.6.1
 mypy==0.790
 pydocstyle==5.1.1
 pytest==6.1.1

worker/util/files.py

@@ -1,10 +1,10 @@
-from pathlib import Path
-from typing import Dict, Any, Optional
 import hashlib
-import os
 import mimetypes
+import os
 import shutil
 import tempfile
+from pathlib import Path
+from typing import Any, Dict, Optional
 
 FileInfo = Dict[str, Any]
 Files = Dict[str, FileInfo]

worker/worker.py

@@ -10,9 +10,8 @@
 from celery import Celery
 from kombu import Queue
 
-from jobs.base.job import Job
-
 from jobs.archive import Archive
+from jobs.base.job import Job
 from jobs.clean import Clean
 from jobs.convert import Convert
 from jobs.decode import Decode