diff --git a/docs/index.rst b/docs/index.rst index 3b08f91..ffd95b1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -169,6 +169,55 @@ You can also specify filters tasks = api.tasks.list(project=1) +By default list returns all objects, eventually getting the +paginated results behind the scenes. + +Pagination +^^^^^^^^^^ + +Pagination is controlled by three parameters as explained below: + ++------------------+----------------------------+-------------+--------------------------------------------------------+ +|`pagination` | `page_size` (default: 100) | `page` | Output | ++==================+============================+=============+========================================================+ +| `True` (default) | `` | `None` | All results retrieved by using paginated results and | +| | | | loading them behind the scenes, using given page | +| | | | size (higher page size could yield better performances)| ++------------------+----------------------------+-------------+--------------------------------------------------------+ +| `True` (default) | `` | `` | Only results for the given page of the given size | +| | | | are retrieved | ++------------------+----------------------------+-------------+--------------------------------------------------------+ +| `False` | `unused` | `unused` | Current behavior: all results, ignoring pagination | ++------------------+----------------------------+-------------+--------------------------------------------------------+ + + +.. note:: non numerical or false `page_size` values is casted to the default value + +Examples +^^^^^^^^^ + +**No pagination** + +.. code:: python + + tasks = api.tasks.list(paginate=False) + +.. warning:: be aware that the unpaginated results may exceed + the data the parser can handle and may result in an error. + +**Retrieve a single page** + +.. code:: python + + tasks_page_1 = api.tasks.list(page=1) # Will only return page 1 + +**Specify the page size** + +.. code:: python + + tasks_page_1 = api.tasks.list(page=1, page_size=200) # Will 200 results from page 1 + + Attach a file ~~~~~~~~~~~~~ diff --git a/taiga/models/base.py b/taiga/models/base.py index 7e47a18..2edd0f5 100644 --- a/taiga/models/base.py +++ b/taiga/models/base.py @@ -34,11 +34,55 @@ def __init__(self, requester): class ListResource(Resource): - def list(self, **queryparams): + def list(self, pagination=True, page_size=None, page=None, **queryparams): + """ + Retrieves a list of objects. + + By default uses local cache and remote pagination + + If pagination is used and no page is requested (the default), all the + remote objects are retrieved and appended in a single list. + + If pagination is disabled, all the objects are fetched from the + endpoint and returned. This may trigger some parsing error if the + result set is very large. + + :param pagination: Use pagination (default: `True`) + :param page_size: Size of the pagination page (default: `100`). + Any non numeric value will be casted to the + default value + :param page: Page number to retrieve (default: `None`). Ignored if + `pagination` is `False` + :param queryparams: Additional filter parameters as accepted by the + remote API + :return: + """ + if page_size and pagination: + try: + page_size = int(page_size) + except (ValueError, TypeError): + page_size = 100 + queryparams['page_size'] = page_size result = self.requester.get( - self.instance.endpoint, query=queryparams + self.instance.endpoint, query=queryparams, paginate=pagination ) - objects = self.parse_list(result.json()) + objects = SearchableList() + objects.extend(self.parse_list(result.json())) + if result.headers.get('X-Pagination-Next', False) and not page: + next_page = 2 + else: + next_page = None + while next_page: + pageparams = queryparams.copy() + pageparams['page'] = next_page + result = self.requester.get( + self.instance.endpoint, query=pageparams, + ) + objects.extend(self.parse_list(result.json())) + if result.headers.get('X-Pagination-Next', False): + next_page += 1 + else: + next_page = None return objects def get(self, resource_id): diff --git a/taiga/requestmaker.py b/taiga/requestmaker.py index d6fc27f..0ff91bc 100644 --- a/taiga/requestmaker.py +++ b/taiga/requestmaker.py @@ -7,7 +7,7 @@ from requests.packages.urllib3.exceptions import InsecureRequestWarning -def _disable_pagination(): +def _requests_compatible_true(): if LooseVersion(requests.__version__) >= LooseVersion('2.11.0'): return 'True' else: @@ -61,12 +61,15 @@ def __init__(self, api_path, host, token, token_type='Bearer', - tls_verify=True): + tls_verify=True, + enable_pagination=True + ): self.api_path = api_path self.host = host self.token = token self.token_type = token_type self.tls_verify = tls_verify + self.enable_pagination = enable_pagination self._cache = RequestCache() if not self.tls_verify: requests.packages.urllib3.disable_warnings(InsecureRequestWarning) @@ -78,12 +81,15 @@ def cache(self): def is_bad_response(self, response): return 400 <= response.status_code <= 500 - def headers(self): + def headers(self, paginate=True): headers = { 'Content-type': 'application/json', 'Authorization': '{0} {1}'.format(self.token_type, self.token), - 'x-disable-pagination': _disable_pagination() } + if self.enable_pagination and paginate: + headers['x-lazy-pagination'] = _requests_compatible_true() + else: + headers['x-disable-pagination'] = _requests_compatible_true() return headers def urljoin(self, *parts): @@ -96,7 +102,7 @@ def get_full_url(self, uri, query={}, **parameters): ) return full_url - def get(self, uri, query={}, cache=False, **parameters): + def get(self, uri, query={}, cache=False, paginate=True, **parameters): try: full_url = self.urljoin( self.host, self.api_path, @@ -114,7 +120,7 @@ def get(self, uri, query={}, cache=False, **parameters): if not result: result = requests.get( full_url, - headers=self.headers(), + headers=self.headers(paginate), params=query, verify=self.tls_verify ) @@ -137,7 +143,7 @@ def post(self, uri, payload=None, query={}, files={}, **parameters): if files: headers = { 'Authorization': '{0} {1}'.format(self.token_type, self.token), - 'x-disable-pagination': _disable_pagination() + 'x-disable-pagination': _requests_compatible_true() } data = payload else: diff --git a/tests/resources/fakes_list_success.json b/tests/resources/fakes_list_success.json new file mode 100644 index 0000000..238e408 --- /dev/null +++ b/tests/resources/fakes_list_success.json @@ -0,0 +1,29 @@ +[ + { + "id": 1 + }, + { + "id": 2 + }, + { + "id": 3 + }, + { + "id": 4 + }, + { + "id": 5 + }, + { + "id": 6 + }, + { + "id": 7 + }, + { + "id": 8 + }, + { + "id": 9 + } +] \ No newline at end of file diff --git a/tests/test_issues.py b/tests/test_issues.py index bb4aa66..d4245cd 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -16,11 +16,14 @@ class TestIssues(unittest.TestCase): @patch('taiga.requestmaker.RequestMaker.get') def test_list_attachments(self, mock_requestmaker_get): + mock_requestmaker_get.return_value = MockResponse(200, + create_mock_json('tests/resources/issues_list_success.json')) rm = RequestMaker('/api/v1', 'fakehost', 'faketoken') Issue(rm, id=1).list_attachments() mock_requestmaker_get.assert_called_with( 'issues/attachments', query={"object_id": 1}, + paginate=True ) @patch('taiga.requestmaker.RequestMaker.post') diff --git a/tests/test_model_base.py b/tests/test_model_base.py index b008b05..b071d0e 100644 --- a/tests/test_model_base.py +++ b/tests/test_model_base.py @@ -1,11 +1,13 @@ # -*- coding: utf-8 -*- +import json +from taiga.models import Projects from taiga.requestmaker import RequestMaker from taiga.models.base import InstanceResource, ListResource, SearchableList import unittest from mock import patch import datetime -from .tools import MockResponse +from .tools import MockResponse, create_mock_json class Fake(InstanceResource): @@ -18,7 +20,7 @@ class Fake(InstanceResource): def my_method(self): response = self.requester.get('/users/{id}/starred', id=self.id) - return projects.Projects.parse(response.json(), self.requester) + return Projects.parse(response.json(), self.requester) class Fakes(ListResource): @@ -26,6 +28,20 @@ class Fakes(ListResource): instance = Fake +class FakeHeaders(dict): + sequence = [] + counter = -1 + + def __init__(self, sequence=[], *args, **kwargs): + self.sequence = sequence + self.counter = -1 + super(FakeHeaders, self).__init__(*args, **kwargs) + + def get(self, k, d=None): + self.counter += 1 + return self.sequence[self.counter] + + class TestModelBase(unittest.TestCase): def test_encoding(self): @@ -136,12 +152,112 @@ def test_call_model_base_delete_element_from_list(self, mock_requestmaker_delete @patch('taiga.requestmaker.RequestMaker.get') def test_call_model_base_list_elements(self, mock_requestmaker_get): + js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json')) + rm = RequestMaker('/api/v1', 'fakehost', 'faketoken') + fakes = Fakes(rm) + + data = json.dumps(js_list) + mock_requestmaker_get.return_value = MockResponse(200, data) + f_list = fakes.list() + mock_requestmaker_get.assert_called_with('fakes', query={}, paginate=True) + self.assertEqual(len(f_list), 9) + + data = json.dumps(js_list[0]) + mock_requestmaker_get.return_value = MockResponse(200, data) + f_list = fakes.list(id=1) + mock_requestmaker_get.assert_called_with('fakes', query={'id':1}, paginate=True) + self.assertEqual(len(f_list), 1) + + @patch('taiga.requestmaker.RequestMaker.get') + def test_call_model_base_list_page_size(self, mock_requestmaker_get): + js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json')) + rm = RequestMaker('/api/v1', 'fakehost', 'faketoken') + fakes = Fakes(rm) + + data = json.dumps(js_list) + mock_requestmaker_get.return_value = MockResponse(200, data, FakeHeaders( + [True, True, False], + **{'X-Pagination-Next': True} + )) + f_list = fakes.list(page_size=2) + mock_requestmaker_get.assert_called_with('fakes', query={'page': 3, 'page_size': 2}) + self.assertEqual(len(f_list), 27) + + data = json.dumps(js_list) + mock_requestmaker_get.return_value = MockResponse(200, data) + f_list = fakes.list(page_size='wrong') + mock_requestmaker_get.assert_called_with('fakes', query={'page_size': 100}, paginate=True) + self.assertEqual(len(f_list), 9) + + @patch('taiga.requestmaker.RequestMaker.get') + def test_call_model_base_list_elements_no_paginate(self, mock_requestmaker_get): + js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json')) + rm = RequestMaker('/api/v1', 'fakehost', 'faketoken') + fakes = Fakes(rm) + + data = json.dumps(js_list) + mock_requestmaker_get.return_value = MockResponse(200, data) + f_list = fakes.list(pagination=False) + mock_requestmaker_get.assert_called_with('fakes', query={}, paginate=False) + self.assertEqual(len(f_list), 9) + + data = json.dumps(js_list[0]) + mock_requestmaker_get.return_value = MockResponse(200, data) + f_list = fakes.list(id=1, pagination=False) + mock_requestmaker_get.assert_called_with('fakes', query={'id':1}, paginate=False) + self.assertEqual(len(f_list), 1) + + @patch('taiga.requestmaker.requests.get') + def test_call_model_base_list_elements_no_paginate_check_requests(self, mock_requestmaker_get): + js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json')) + rm = RequestMaker('/api/v1', 'fakehost', 'faketoken') + fakes = Fakes(rm) + + data = json.dumps(js_list) + mock_requestmaker_get.return_value = MockResponse(200, data) + f_list = fakes.list(pagination=False) + mock_requestmaker_get.assert_called_with( + 'fakehost/api/v1/fakes', verify=True, params={}, + headers={ + 'x-disable-pagination': 'True', 'Content-type': 'application/json', 'Authorization': 'Bearer faketoken' + } + ) + self.assertEqual(len(f_list), 9) + + @patch('taiga.requestmaker.requests.get') + def test_call_model_base_list_elements_paginate_check_requests(self, mock_requestmaker_get): + js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json')) rm = RequestMaker('/api/v1', 'fakehost', 'faketoken') fakes = Fakes(rm) - fakes.list() - mock_requestmaker_get.assert_called_with('fakes', query={}) - fakes.list(project_id=1) - mock_requestmaker_get.assert_called_with('fakes', query={'project_id':1}) + + data = json.dumps(js_list) + mock_requestmaker_get.return_value = MockResponse(200, data) + f_list = fakes.list() + mock_requestmaker_get.assert_called_with( + 'fakehost/api/v1/fakes', verify=True, params={}, + headers={ + 'x-lazy-pagination': 'True', 'Content-type': 'application/json', 'Authorization': 'Bearer faketoken' + } + ) + self.assertEqual(len(f_list), 9) + + @patch('taiga.requestmaker.RequestMaker.get') + def test_call_model_base_list_elements_single_page(self, mock_requestmaker_get): + js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json')) + rm = RequestMaker('/api/v1', 'fakehost', 'faketoken') + fakes = Fakes(rm) + + data = json.dumps(js_list[:5]) + mock_requestmaker_get.return_value = MockResponse(200, data) + f_list = fakes.list(page_size=5, page=1) + self.assertEqual(len(f_list), 5) + mock_requestmaker_get.assert_called_with('fakes', query={'page_size': 5}, paginate=True) + + data = json.dumps(js_list[5:]) + mock_requestmaker_get.return_value = MockResponse(200, data) + f_list = fakes.list(page_size=5, page=2) + self.assertEqual(len(f_list), 4) + mock_requestmaker_get.assert_called_with('fakes', query={'page_size': 5}, paginate=True) def test_to_dict_method(self): rm = RequestMaker('/api/v1', 'fakehost', 'faketoken') diff --git a/tests/test_requestmaker.py b/tests/test_requestmaker.py index f54d1ef..c732577 100644 --- a/tests/test_requestmaker.py +++ b/tests/test_requestmaker.py @@ -1,4 +1,4 @@ -from taiga.requestmaker import RequestMaker, _disable_pagination +from taiga.requestmaker import RequestMaker, _requests_compatible_true import taiga.exceptions import requests import unittest @@ -33,7 +33,7 @@ def test_call_requests_post_with_files(self, requests_post): data=None, params={}, headers={ 'Authorization': 'Bearer f4k3', - 'x-disable-pagination': _disable_pagination() + 'x-disable-pagination': _requests_compatible_true() } ) diff --git a/tests/test_tasks.py b/tests/test_tasks.py index fc777f8..0375e2a 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -5,6 +5,8 @@ from mock import patch import six +from tests.tools import MockResponse, create_mock_json + if six.PY2: import_open = '__builtin__.open' else: @@ -14,11 +16,14 @@ class TestTasks(unittest.TestCase): @patch('taiga.requestmaker.RequestMaker.get') def test_list_attachments(self, mock_requestmaker_get): + mock_requestmaker_get.return_value = MockResponse(200, + create_mock_json('tests/resources/tasks_list_success.json')) rm = RequestMaker('/api/v1', 'fakehost', 'faketoken') Task(rm, id=1).list_attachments() mock_requestmaker_get.assert_called_with( 'tasks/attachments', query={"object_id": 1}, + paginate=True ) @patch(import_open) diff --git a/tests/test_user_stories.py b/tests/test_user_stories.py index bce0476..f6f57d3 100644 --- a/tests/test_user_stories.py +++ b/tests/test_user_stories.py @@ -17,11 +17,14 @@ class TestUserStories(unittest.TestCase): @patch('taiga.requestmaker.RequestMaker.get') def test_list_attachments(self, mock_requestmaker_get): + mock_requestmaker_get.return_value = MockResponse(200, + create_mock_json('tests/resources/userstories_list_success.json')) rm = RequestMaker('/api/v1', 'fakehost', 'faketoken') UserStory(rm, id=1).list_attachments() mock_requestmaker_get.assert_called_with( 'userstories/attachments', query={"object_id": 1}, + paginate=True ) @patch('taiga.requestmaker.RequestMaker.get') diff --git a/tests/tools.py b/tests/tools.py index d571e90..32bd3ec 100644 --- a/tests/tools.py +++ b/tests/tools.py @@ -1,9 +1,10 @@ import json class MockResponse(): - def __init__(self, status_code, text): + def __init__(self, status_code, text, headers={}): self.status_code = status_code self.text = text + self.headers = headers def json(self): return json.loads(self.text)