Support GitHub Enterprise

pep8speaks/constants.py

@@ -3,4 +3,12 @@
 # HEADERS is deprecated, use AUTH only
 HEADERS = {"Authorization": "token " + os.environ.setdefault("GITHUB_TOKEN", "")}
 AUTH = (os.environ.setdefault("BOT_USERNAME", ""), os.environ.setdefault("GITHUB_TOKEN", ""))
-BASE_URL = 'https://api.github.com'
+
+# on github enterprise, set appropriate FQDN by environment variable.
+# export GH_BASE='github.example.com'
+# export GH_API='github.example.com/api/v3'
+# export GH_RAW='raw.github.example.com'
+GH_BASE = os.environ.setdefault('GH_BASE', 'github.com')
+GH_API = os.environ.setdefault('GH_API', 'api.github.com')
+GH_RAW = os.environ.setdefault('GH_RAW', 'raw.githubusercontent.com')
+GH_USER_ID = int(os.environ.setdefault('GH_USER_ID', '24736507'))

pep8speaks/helpers.py

@@ -14,6 +14,7 @@
 import unidiff
 import yaml
 from pep8speaks import utils
+from pep8speaks.constants import GH_BASE, GH_RAW, GH_USER_ID
 
 
 def update_users(repository):
@@ -98,12 +99,12 @@ def get_config(repo, base_branch, after_commit_hash):
 
     # Read setup.cfg for [pycodestyle] or [flake8] section
     setup_config_file = ""
-    query = f"https://raw.githubusercontent.com/{repo}/{base_branch}/setup.cfg"
+    query = f"https://{GH_RAW}/{repo}/{base_branch}/setup.cfg"
     r = utils.query_request(query)
     if r.status_code == 200:
         setup_config_file = r.text
     else:  # Try to look for a config in the head branch of the Pull Request
-        new_query = f"https://raw.githubusercontent.com/{repo}/{after_commit_hash}/setup.cfg"
+        new_query = f"https://{GH_RAW}/{repo}/{after_commit_hash}/setup.cfg"
         r_new = utils.query_request(new_query)
         if r_new.status_code == 200:
             setup_config_file = r_new.text
@@ -120,13 +121,13 @@ def get_config(repo, base_branch, after_commit_hash):
     new_config_text = ""
 
     # Configuration file
-    query = f"https://raw.githubusercontent.com/{repo}/{base_branch}/.pep8speaks.yml"
+    query = f"https://{GH_RAW}/{repo}/{base_branch}/.pep8speaks.yml"
     r = utils.query_request(query)
 
     if r.status_code == 200:
         new_config_text = r.text
     else:  # Try to look for a config in the head branch of the Pull Request
-        new_query = f"https://raw.githubusercontent.com/{repo}/{after_commit_hash}/.pep8speaks.yml"
+        new_query = f"https://{GH_RAW}/{repo}/{after_commit_hash}/.pep8speaks.yml"
         r_new = utils.query_request(new_query)
         if r_new.status_code == 200:
             new_config_text = r_new.text
@@ -221,7 +222,7 @@ def run_pycodestyle(ghrequest, config):
     ghrequest.links = {}  # UI Link of each updated file in the PR
     for py_file in py_files:
         filename = py_file[1:]
-        query = f"https://raw.githubusercontent.com/{repo}/{commit}/{py_file}"
+        query = f"https://{GH_RAW}/{repo}/{commit}/{py_file}"
         r = utils.query_request(query)
         with open("file_to_check.py", 'w+', encoding=r.encoding) as file_to_check:
             file_to_check.write(r.text)
@@ -256,7 +257,7 @@ def run_pycodestyle(ghrequest, config):
                     ghrequest.results[filename].remove(error)
 
         ## Store the link to the file
-        url = f"https://github.com/{repo}/blob/{commit}{py_file}"
+        url = f"https://{GH_BASE}/{repo}/blob/{commit}{py_file}"
         ghrequest.links[filename + "_link"] = url
         os.remove("file_to_check.py")
 
@@ -397,7 +398,7 @@ def create_or_update_comment(ghrequest, comment, ONLY_UPDATE_COMMENT_BUT_NOT_CRE
     # Get the last comment id by the bot
     last_comment_id = None
     for old_comment in comments:
-        if old_comment["user"]["id"] == 24736507:  # ID of @pep8speaks
+        if old_comment["user"]["id"] == GH_USER_ID:  # ID of @pep8speaks
             last_comment_id = old_comment["id"]
             break
 
@@ -418,7 +419,7 @@ def create_or_update_comment(ghrequest, comment, ONLY_UPDATE_COMMENT_BUT_NOT_CRE
 def autopep8(ghrequest, config):
     # Run pycodestyle
 
-    r = utils.query_request(ghrequest.diff_url)
+    r = ghrequest.fetch_diff()
     ## All the python files with additions
     patch = unidiff.PatchSet(r.content.splitlines(), encoding=r.encoding)
 
@@ -442,7 +443,7 @@ def autopep8(ghrequest, config):
 
     for py_file in py_files:
         filename = py_file[1:]
-        url = f"https://raw.githubusercontent.com/{ghrequest.repository}/{ghrequest.sha}/{py_file}"
+        url = f"https://{GH_RAW}/{ghrequest.repository}/{ghrequest.sha}/{py_file}"
         r = utils.query_request(url)
         with open("file_to_fix.py", 'w+', encoding=r.encoding) as file_to_fix:
             file_to_fix.write(r.text)
@@ -458,7 +459,7 @@ def autopep8(ghrequest, config):
 
         ## Store the link to the file
         ghrequest.links = {}
-        ghrequest.links[filename + "_link"] = f"https://github.com/{ghrequest.repository}/blob/{ghrequest.sha}{py_file}"
+        ghrequest.links[filename + "_link"] = f"https://{GH_BASE}/{ghrequest.repository}/blob/{ghrequest.sha}{py_file}"
         os.remove("file_to_fix.py")
 
 
@@ -485,7 +486,7 @@ def create_gist(ghrequest):
 def delete_if_forked(ghrequest):
     FORKED = False
     query = "/user/repos"
-    r = utils.query_request(query)
+    r = utils.query_request(query, params={'affiliation': 'owner'})
     for repo in r.json():
         if repo["description"]:
             if ghrequest.target_repo_fullname in repo["description"]:
@@ -553,7 +554,7 @@ def create_new_branch(ghrequest):
 
 def autopep8ify(ghrequest, config):
     # Run pycodestyle
-    r = utils.query_request(ghrequest.diff_url)
+    r = ghrequest.fetch_diff()
 
     ## All the python files with additions
     patch = unidiff.PatchSet(r.content.splitlines(), encoding=r.encoding)
@@ -578,7 +579,7 @@ def autopep8ify(ghrequest, config):
 
     for py_file in py_files:
         filename = py_file[1:]
-        query = f"https://raw.githubusercontent.com/{ghrequest.repository}/{ghrequest.sha}/{py_file}"
+        query = f"https://{GH_RAW}/{ghrequest.repository}/{ghrequest.sha}/{py_file}"
         r = utils.query_request(query)
         with open("file_to_fix.py", 'w+', encoding=r.encoding) as file_to_fix:
             file_to_fix.write(r.text)

pep8speaks/models.py

@@ -75,14 +75,15 @@ def _set_defaults(self, request):
         self.sha = self.pull_request['head']['sha']
         self.action = request.json['action']
         self.author = self.pull_request['user']['login']
-        self.pr_desc = self.pull_request['body']
+        self.pr_desc = self.pull_request['body'] if self.pull_request['body'] is not None else ''
         self.diff_url = self.pull_request['diff_url']
         self.pr_title = self.pull_request['title']
         self.pr_number = self.pull_request['number']
         self.repository = request.json['repository']['full_name']
         self.commits_url = self.pull_request['commits_url']
         self.base_branch = self.pull_request['base']['ref']
         self.after_commit_hash = self.pull_request['head']['sha']
+        self.private = self.pull_request['base']['repo']['private']
 
     def _set_conditionals(self, request, event):
         """
@@ -93,3 +94,15 @@ def _set_conditionals(self, request, event):
             self.review_url = request.json['comment']['html_url']
             self.comment = request.json['comment']['body']
             self.base_branch = request.json['repository']['default_branch']  # Overrides the default
+
+    def fetch_diff(self):
+        """
+        Fetch diff and return Response object.
+        """
+        if self.private:
+            # If the target repository is private, fetch diff using API.
+            # https://developer.github.com/v3/media/#commits-commit-comparison-and-pull-requests
+            return utils.query_request(f'/repos/{self.repository}/pulls/{self.pr_number}',
+                                       headers={'Accept': 'application/vnd.github.v3.diff'})
+        else:
+            return utils.query_request(self.diff_url)

pep8speaks/utils.py

@@ -7,7 +7,8 @@
 from flask import abort
 from flask import Response as FResponse
 import requests
-from pep8speaks.constants import AUTH, BASE_URL
+from requests.packages.urllib3.util.retry import Retry
+from pep8speaks.constants import AUTH, GH_API
 
 
 def query_request(query=None, method="GET", **kwargs):
@@ -19,13 +20,28 @@ def query_request(query=None, method="GET", **kwargs):
     """
 
     if query[0] == "/":
-        query = BASE_URL + query
+        query = 'https://' + GH_API + query
 
     request_kwargs = {
         "auth": AUTH,
+        "timeout": 8,
     }
     request_kwargs.update(**kwargs)
-    return requests.request(method, query, **request_kwargs)
+
+    s = requests.Session()
+    retries = Retry(
+        total=3,
+        connect=3,
+        read=3,
+        backoff_factor=1.0,
+        method_whitelist=frozenset(['GET', 'POST']),
+        status_forcelist=(500, 502, 503, 504, 409, 422),
+    )
+    s.mount("http://", requests.adapters.HTTPAdapter(max_retries=retries))
+    s.mount("https://", requests.adapters.HTTPAdapter(max_retries=retries))
+
+    return s.request(method, query, **request_kwargs)
+
 
 
 def Response(data=None, status=200, mimetype='application/json'):

tests/local/pep8speaks/test_utils.py

@@ -4,7 +4,7 @@
 import werkzeug
 import mock
 from pep8speaks.utils import update_dict, match_webhook_secret, query_request
-from pep8speaks.constants import BASE_URL
+from pep8speaks.constants import GH_API
 
 
 class TestUtils:
@@ -14,7 +14,7 @@ class TestUtils:
     ])
     def test_request(self, mocker, query, method, json, data, headers, params):
         mock_func = mock.MagicMock(return_value=True)
-        mocker.patch('requests.request', mock_func)
+        mocker.patch('requests.Session.request', mock_func)
         query_request(query, method, json=json, data=data,
                       headers=headers, params=params)
         assert mock_func.call_count == 1
@@ -24,7 +24,7 @@ def test_request(self, mocker, query, method, json, data, headers, params):
         assert mock_func.call_args[1]['params'] == params
         assert mock_func.call_args[1]['json'] == json
         if query[0] == "/":
-            assert mock_func.call_args[0][1] == BASE_URL + query
+            assert mock_func.call_args[0][1] == 'https://' + GH_API + query
         else:
             assert mock_func.call_args[0][1] == query