diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 468cd8d28..1c7a8cb61 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ - repo: git://github.com/pre-commit/pre-commit-hooks - sha: cf550fcab3f12015f8676b8278b30e1a5bc10e70 + sha: 29bf11d13689a0a9a895c41eb3591c7e942d377d hooks: - id: autopep8-wrapper args: @@ -18,4 +18,4 @@ - id: flake8 args: - --exclude=pytx/docs/*,pytx/build/* - - --ignore=W291 \ No newline at end of file + - --ignore=W291 diff --git a/pytx/README.rst b/pytx/README.rst index 14f53bc7f..fce354136 100644 --- a/pytx/README.rst +++ b/pytx/README.rst @@ -37,11 +37,11 @@ Quick Example .. code-block :: python - from pytx import init + from pytx import access_token from pytx import ThreatDescriptor from pytx.vocabulary import ThreatDescriptor as td - init('', '') + access_token('', '') results = ThreatDescriptor.objects(text='www.facebook.com') for result in results: print result.get(td.THREAT_TYPES) diff --git a/pytx/docs/conf.py b/pytx/docs/conf.py index 9a64c806b..e8cbd7cda 100644 --- a/pytx/docs/conf.py +++ b/pytx/docs/conf.py @@ -56,9 +56,9 @@ # built documents. # # The short X.Y version. -version = '0.2.0' +version = '0.3.0' # The full version, including alpha/beta/rc tags. -release = '0.2.0' +release = '0.3.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pytx/docs/index.rst b/pytx/docs/index.rst index 2ffb53dcd..a286c3434 100644 --- a/pytx/docs/index.rst +++ b/pytx/docs/index.rst @@ -15,6 +15,7 @@ Contents: quickstart pytx.access_token pytx.common + pytx.connection pytx.errors pytx.logger pytx.malware diff --git a/pytx/docs/pytx.connection.rst b/pytx/docs/pytx.connection.rst new file mode 100644 index 000000000..a29a67790 --- /dev/null +++ b/pytx/docs/pytx.connection.rst @@ -0,0 +1,10 @@ +pytx.connection package +======================= + +Module contents +--------------- + +.. automodule:: pytx.connection + :members: + :undoc-members: + :show-inheritance: diff --git a/pytx/docs/quickstart.rst b/pytx/docs/quickstart.rst index c5cdc171f..c7445127a 100644 --- a/pytx/docs/quickstart.rst +++ b/pytx/docs/quickstart.rst @@ -12,29 +12,29 @@ The app-id is public knowledge but your app-secret is sensitive. These values are provided to you once you've obtained access to ThreatExchange. pytx will try to find an access token to use or an access token can be passed to -pytx.access_token.init(). pytx needs an access token before it will function and -can properly make requests properly. Here are some examples of how to provide -your access token: +pytx.access_token.access_token(). pytx needs an access token before it will +function and can properly make requests properly. Here are some examples of +how to provide your access token: .. code-block :: python - from pytx import init + from pytx import access_token # Use environment variables to build the access token. # 1. Use the value of the 'TX_ACCESS_TOKEN' environment variable. # 2. Use the concatenation of the 'TX_APP_ID' and 'TX_APP_SECRET' environment variables. - # There is no need to call init() if the environment variables are set. + # There is no need to call access_token() if the environment variables are set. # 3. Use a .pytx file which contains your app-id and app-secret. # File should be: 'app-id|app-secret' on one line # pytx will use either '$PWD/.pytx' or ~/.pytx' if they are found. - # There is no need to call init() if the environment variables are set. + # There is no need to call access_token() if the environment variables are set. # 4. Use the concatenation of the app_id and app_secret parameters - init(app_id='', app_secret='') + access_token(app_id='', app_secret='') # 5. Use the first line of the file 'token_file' - init(token_file='/path/to/token/file') + access_token(token_file='/path/to/token/file') If you need to get the value of the access token pytx is using programmatically, @@ -60,6 +60,16 @@ information to that file. If the file cannot be written to expect some issues. If you do not provide an argument to setup_logger, no logging will occur. +If you need to setup a proxy or adjust the verify argument for requests, you can +use the connection() function to change them. More info can be found here: +http://docs.python-requests.org/en/latest/api/#requests.request + +.. code-block :: python + + from pytx import connection() + connection(proxies=, verify=) + + pytx uses classes as the primary method for developer interaction with the ThreatExchange API. There are several main classes: diff --git a/pytx/pytx/__init__.py b/pytx/pytx/__init__.py index 8877b2211..a9c32fc7b 100644 --- a/pytx/pytx/__init__.py +++ b/pytx/pytx/__init__.py @@ -1,4 +1,5 @@ -from access_token import init +from access_token import access_token +from connection import connection from logger import setup_logger from request import Broker from malware import Malware @@ -8,7 +9,8 @@ from threat_indicator import ThreatIndicator __all__ = [ - 'init', + 'access_token', + 'connection', 'setup_logger', 'Broker', 'Malware', diff --git a/pytx/pytx/access_token.py b/pytx/pytx/access_token.py index 7678e880b..6f6b81ecd 100644 --- a/pytx/pytx/access_token.py +++ b/pytx/pytx/access_token.py @@ -26,18 +26,18 @@ def _read_token_file(token_file): def get_access_token(): """ - Returns the existing access token if init() has been called. - Will attempt to init() in the case that there is no access token. + Returns the existing access token if access_token() has been called. + Will attempt to access_token() in the case that there is no access token. :raises: :class:`errors.pytxIniterror` if there is no access token. """ global __ACCESS_TOKEN if not __ACCESS_TOKEN: - init() + access_token() if not __ACCESS_TOKEN: - raise pytxInitError('Must init() before instantiating') + raise pytxInitError('Must access_token() before instantiating') return __ACCESS_TOKEN @@ -51,12 +51,12 @@ def _find_token_file(): return None -def init(app_id=None, app_secret=None, token_file=None): +def access_token(app_id=None, app_secret=None, token_file=None): """ Use the app_id and app_secret to store the access_token globally for all instantiated objects to leverage. - There are many ways to specify the app_id and app_secret. In order, init will try: + There are many ways to specify the app_id and app_secret. In order, we will try: 1. Use the value of the 'TX_ACCESS_TOKEN' environment variable. 2. Use the concatenation of the 'TX_APP_ID' and 'TX_APP_SECRET' environment variables. 3. Use the first line of the file '$PWD/.pytx' or ~/.pytx' diff --git a/pytx/pytx/common.py b/pytx/pytx/common.py index 6293d2a9f..b85368304 100644 --- a/pytx/pytx/common.py +++ b/pytx/pytx/common.py @@ -138,7 +138,8 @@ def to_dict(self): @classmethod def objects(cls, text=None, strict_text=False, type_=None, threat_type=None, fields=None, limit=None, since=None, until=None, __raw__=None, - full_response=False, dict_generator=False, retries=None): + full_response=False, dict_generator=False, retries=None, + proxies=None, verify=None): """ Get objects from ThreatExchange. @@ -169,6 +170,10 @@ def objects(cls, text=None, strict_text=False, type_=None, threat_type=None, :type dict_generator: bool :param retries: Number of retries to fetch a page before stopping. :type retries: int + :param proxies: proxy info for requests. + :type proxies: dict + :param verify: verify info for requests. + :type verify: bool, str :returns: Generator, dict (using json.loads()) """ @@ -189,18 +194,21 @@ def objects(cls, text=None, strict_text=False, type_=None, threat_type=None, until=until, ) if full_response: - return Broker.get(cls._URL, params=params, retries=retries) + return Broker.get(cls._URL, params=params, retries=retries, + proxies=proxies, verify=verify) else: return Broker.get_generator(cls, cls._URL, to_dict=dict_generator, params=params, - retries=retries) + retries=retries, + proxies=proxies, + verify=verify) @class_or_instance_method def details(cls_or_self, id=None, fields=None, connection=None, full_response=False, dict_generator=False, retries=None, - metadata=False): + proxies=None, verify=None, metadata=False): """ Get object details. Allows you to limit the fields returned in the object's details. Also allows you to provide a connection. If a @@ -237,6 +245,10 @@ def details(cls_or_self, id=None, fields=None, connection=None, :type dict_generator: bool :param retries: Number of retries to fetch a page before stopping. :type retries: int + :param proxies: proxy info for requests. + :type proxies: dict + :param verify: verify info for requests. + :type verify: bool, str :param metadata: Get extra metadata in the response. :type metadata: bool :returns: Generator, dict, class @@ -258,7 +270,8 @@ def details(cls_or_self, id=None, fields=None, connection=None, if metadata: params[t.METADATA] = 1 if full_response: - return Broker.get(url, params=params, retries=retries) + return Broker.get(url, params=params, retries=retries, + proxies=proxies, verify=verify) else: if connection: # Avoid circular imports @@ -281,17 +294,23 @@ def details(cls_or_self, id=None, fields=None, connection=None, url, to_dict=dict_generator, params=params, - retries=retries) + retries=retries, + proxies=proxies, + verify=verify) else: if isinstance(cls_or_self, type): return Broker.get_new(cls_or_self, Broker.get(url, params=params, - retries=retries)) + retries=retries, + proxies=proxies, + verify=verify)) else: cls_or_self.populate(Broker.get(url, params=params, - retries=retries)) + retries=retries, + proxies=proxies, + verify=verify)) cls_or_self._changed = [] def get_changed(self): @@ -307,7 +326,7 @@ def get_changed(self): ) @classmethod - def new(cls, params, retries=None): + def new(cls, params, retries=None, proxies=None, verify=None): """ Submit params to the graph to add an object. We will submit to the object URL used for creating new objects in the graph. When submitting @@ -318,6 +337,10 @@ def new(cls, params, retries=None): :type params: dict :param retries: Number of retries to submit before stopping. :type retries: int + :param proxies: proxy info for requests. + :type proxies: dict + :param verify: verify info for requests. + :type verify: bool, str :returns: dict (using json.loads()) """ @@ -328,9 +351,10 @@ def new(cls, params, retries=None): if (params[td.PRIVACY_TYPE] != pt.VISIBLE and len(params[td.PRIVACY_MEMBERS].split(',')) < 1): raise pytxValueError('Must provide %s' % td.PRIVACY_MEMBERS) - return Broker.post(cls._URL, params=params, retries=retries) + return Broker.post(cls._URL, params=params, retries=retries, + proxies=proxies, verify=verify) - def save(self, params=None, retries=None): + def save(self, params=None, retries=None, proxies=None, verify=None): """ Submit changes to the graph to update an object. We will determine the Details URL and submit there (used for updating an existing object). If @@ -341,15 +365,21 @@ def save(self, params=None, retries=None): :type params: dict :param retries: Number of retries to submit before stopping. :type retries: int + :param proxies: proxy info for requests. + :type proxies: dict + :param verify: verify info for requests. + :type verify: bool, str :returns: dict (using json.loads()) """ if params is None: params = self.get_changed() - return Broker.post(self._DETAILS, params=params, retries=retries) + return Broker.post(self._DETAILS, params=params, retries=retries, + proxies=proxies, verify=verify) @class_or_instance_method - def send(cls_or_self, id_=None, params=None, type_=None, retries=None): + def send(cls_or_self, id_=None, params=None, type_=None, retries=None, + proxies=None, verify=None): """ Send custom params to the object URL. If `id` is provided it will be appended to the URL. If this is an uninstantiated class we will use the @@ -366,7 +396,10 @@ def send(cls_or_self, id_=None, params=None, type_=None, retries=None): :type type_: str :param retries: Number of retries to submit before stopping. :type retries: int - + :param proxies: proxy info for requests. + :type proxies: dict + :param verify: verify info for requests. + :type verify: bool, str :returns: dict (using json.loads()) """ @@ -383,11 +416,13 @@ def send(cls_or_self, id_=None, params=None, type_=None, retries=None): if params is None: params = {} if type_ == 'GET': - return Broker.get(url, params=params, retries=retries) + return Broker.get(url, params=params, retries=retries, + proxies=proxies, verify=verify) else: - return Broker.post(url, params=params, retries=retries) + return Broker.post(url, params=params, retries=retries, + proxies=proxies, verify=verify) - def expire(self, timestamp, retries=None): + def expire(self, timestamp, retries=None, proxies=None, verify=None): """ Expire by setting the 'expired_on' timestamp. @@ -395,6 +430,10 @@ def expire(self, timestamp, retries=None): :type timestamp: str :param retries: Number of retries to submit before stopping. :type retries: int + :param proxies: proxy info for requests. + :type proxies: dict + :param verify: verify info for requests. + :type verify: bool, str :returns: dict (using json.loads()) """ @@ -402,9 +441,10 @@ def expire(self, timestamp, retries=None): params = { td.EXPIRED_ON: timestamp } - return Broker.post(self._DETAILS, params=params, retries=retries) + return Broker.post(self._DETAILS, params=params, retries=retries, + proxies=proxies, verify=verify) - def false_positive(self, object_id, retries=None): + def false_positive(self, object_id, retries=None, proxies=None, verify=None): """ Mark an object as a false positive by setting the status to UNKNOWN. @@ -413,15 +453,20 @@ def false_positive(self, object_id, retries=None): :type object_id: str :param retries: Number of retries to submit before stopping. :type retries: int + :param proxies: proxy info for requests. + :type proxies: dict + :param verify: verify info for requests. + :type verify: bool, str :returns: dict (using json.loads()) """ params = { c.STATUS: s.UNKNOWN } - return Broker.post(self._DETAILS, params=params, retries=retries) + return Broker.post(self._DETAILS, params=params, retries=retries, + proxies=proxies, verify=verify) - def add_connection(self, object_id, retries=None): + def add_connection(self, object_id, retries=None, proxies=None, verify=None): """ Use HTTP POST and add a connection between two objects. @@ -429,17 +474,23 @@ def add_connection(self, object_id, retries=None): :type object_id: str :param retries: Number of retries to submit before stopping. :type retries: int + :param proxies: proxy info for requests. + :type proxies: dict + :param verify: verify info for requests. + :type verify: bool, str :returns: dict (using json.loads()) """ params = { t.RELATED_ID: object_id } - return Broker.post(self._RELATED, params=params, retries=retries) + return Broker.post(self._RELATED, params=params, retries=retries, + proxies=proxies, verify=verify) # DELETE REQUESTS - def delete_connection(self, object_id, retries=None): + def delete_connection(self, object_id, retries=None, proxies=None, + verify=None): """ Use HTTP DELETE and remove the connection to another object. @@ -447,10 +498,15 @@ def delete_connection(self, object_id, retries=None): :type object_id: str :param retries: Number of retries to submit before stopping. :type retries: int + :param proxies: proxy info for requests. + :type proxies: dict + :param verify: verify info for requests. + :type verify: bool, str :returns: dict (using json.loads()) """ params = { t.RELATED_ID: object_id } - return Broker.delete(self._RELATED, params=params, retries=retries) + return Broker.delete(self._RELATED, params=params, retries=retries, + proxies=proxies, verify=verify) diff --git a/pytx/pytx/connection.py b/pytx/pytx/connection.py new file mode 100644 index 000000000..1f02a0531 --- /dev/null +++ b/pytx/pytx/connection.py @@ -0,0 +1,37 @@ +__PROXIES = None +__VERIFY = None + + +def get_proxies(): + """ + Returns the existing proxies setting. + """ + + global __PROXIES + return __PROXIES + + +def get_verify(): + """ + Returns the existing verify setting. + """ + + global __VERIFY + return __VERIFY + + +def connection(proxies=None, verify=None): + """ + Configure proxies and verify settings for requests. This is a global setting + that all requests calls will use unless overridden on a per-call basis. + + :param proxies: proxy info for requests. + :type proxies: dict + :param verify: verify info for requests. + :type verify: bool, str + """ + global __PROXIES + global __VERIFY + + __PROXIES = proxies + __VERIFY = verify diff --git a/pytx/pytx/request.py b/pytx/pytx/request.py index fbb593f70..14799d8f1 100644 --- a/pytx/pytx/request.py +++ b/pytx/pytx/request.py @@ -4,6 +4,7 @@ from requests.packages.urllib3.util import Retry from access_token import get_access_token +from connection import get_proxies, get_verify from logger import do_log, log_message from vocabulary import ThreatExchange as t @@ -147,8 +148,9 @@ def validate_get(cls, limit, since, until): cls.validate_limit(limit) @classmethod - def build_get_parameters(cls, text=None, strict_text=None, type_=None, threat_type=None, - fields=None, limit=None, since=None, until=None): + def build_get_parameters(cls, text=None, strict_text=None, type_=None, + threat_type=None, fields=None, limit=None, + since=None, until=None): """ Validate arguments and convert them into GET parameters. @@ -214,7 +216,7 @@ def build_session(cls, retries=None): return session @classmethod - def get(cls, url, params=None, retries=None): + def get(cls, url, params=None, retries=None, proxies=None, verify=None): """ Send a GET request. @@ -224,18 +226,27 @@ def get(cls, url, params=None, retries=None): :type params: dict :param retries: Number of retries before stopping. :type retries: int + :param proxies: proxy info for requests. + :type proxies: dict + :param verify: verify info for requests. + :type verify: bool, str :returns: dict (using json.loads()) """ + if not params: params = dict() + if proxies is None: + proxies = get_proxies() + if verify is None: + verify = get_verify() params[t.ACCESS_TOKEN] = get_access_token() session = cls.build_session(retries) - resp = session.get(url, params=params) + resp = session.get(url, params=params, proxies=proxies, verify=verify) return cls.handle_results(resp) @classmethod - def post(cls, url, params=None, retries=None): + def post(cls, url, params=None, retries=None, proxies=None, verify=None): """ Send a POST request. @@ -245,19 +256,27 @@ def post(cls, url, params=None, retries=None): :type params: dict :param retries: Number of retries before stopping. :type retries: int + :param proxies: proxy info for requests. + :type proxies: dict + :param verify: verify info for requests. + :type verify: bool, str :returns: dict (using json.loads()) """ if not params: params = dict() + if proxies is None: + proxies = get_proxies() + if verify is None: + verify = get_verify() params[t.ACCESS_TOKEN] = get_access_token() session = cls.build_session(retries) - resp = session.post(url, params=params) + resp = session.post(url, params=params, proxies=proxies, verify=verify) return cls.handle_results(resp) @classmethod - def delete(cls, url, params=None, retries=None): + def delete(cls, url, params=None, retries=None, proxies=None, verify=None): """ Send a DELETE request. @@ -267,20 +286,28 @@ def delete(cls, url, params=None, retries=None): :type params: dict :param retries: Number of retries before stopping. :type retries: int + :param proxies: proxy info for requests. + :type proxies: dict + :param verify: verify info for requests. + :type verify: bool, str :returns: dict (using json.loads()) """ if not params: params = dict() + if proxies is None: + proxies = get_proxies() + if verify is None: + verify = get_verify() params[t.ACCESS_TOKEN] = get_access_token() session = cls.build_session(retries) - resp = session.delete(url, params=params) + resp = session.delete(url, params=params, proxies=proxies, verify=verify) return cls.handle_results(resp) @classmethod def get_generator(cls, klass, url, to_dict=False, params=None, - retries=None): + retries=None, proxies=None, verify=None): """ Generator for managing GET requests. For each GET request it will yield the next object in the results until there are no more objects. If the @@ -299,6 +326,10 @@ def get_generator(cls, klass, url, to_dict=False, params=None, :type params: dict :param retries: Number of retries before stopping. :type retries: int + :param proxies: proxy info for requests. + :type proxies: dict + :param verify: verify info for requests. + :type verify: bool, str :returns: Generator """ @@ -306,9 +337,14 @@ def get_generator(cls, klass, url, to_dict=False, params=None, raise pytxValueError('Must provide a valid object to query.') if not params: params = dict() + if proxies is None: + proxies = get_proxies() + if verify is None: + verify = get_verify() next_ = True while next_: - results = cls.get(url, params, retries) + results = cls.get(url, params=params, retries=retries, + proxies=proxies, verify=verify) if do_log(): try: before = results[t.PAGING][p.CURSORS].get(pc.BEFORE, 'None') diff --git a/pytx/setup.py b/pytx/setup.py index 66404419a..e40d41c9b 100644 --- a/pytx/setup.py +++ b/pytx/setup.py @@ -13,7 +13,7 @@ setup( name='pytx', - version='0.2.0', + version='0.3.0', description='Python Library for Facebook ThreatExchange', long_description=long_description, author='Mike Goffin',