Skip to content

Commit

Permalink
Merge pull request #61 from nephila/feature/pagination
Browse files Browse the repository at this point in the history 
Preliminary pagination support API
  • Loading branch information
@yakky
yakky authored Jan 10, 2018
2 parents 012fb19 + 499534f commit 618cb2a
Show file tree
Hide file tree
Showing 10 changed files with 275 additions and 19 deletions.
49 changes: 49 additions & 0 deletions docs/index.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,55 @@ You can also specify filters
tasks = api.tasks.list(project=1)
By default list returns all objects, eventually getting the
paginated results behind the scenes.

Pagination
^^^^^^^^^^

Pagination is controlled by three parameters as explained below:

+------------------+----------------------------+-------------+--------------------------------------------------------+
|`pagination` | `page_size` (default: 100) | `page` | Output |
+==================+============================+=============+========================================================+
| `True` (default) | `<integer>` | `None` | All results retrieved by using paginated results and |
| | | | loading them behind the scenes, using given page |
| | | | size (higher page size could yield better performances)|
+------------------+----------------------------+-------------+--------------------------------------------------------+
| `True` (default) | `<integer>` | `<integer>` | Only results for the given page of the given size |
| | | | are retrieved |
+------------------+----------------------------+-------------+--------------------------------------------------------+
| `False` | `unused` | `unused` | Current behavior: all results, ignoring pagination |
+------------------+----------------------------+-------------+--------------------------------------------------------+


.. note:: non numerical or false `page_size` values is casted to the default value

Examples
^^^^^^^^^

**No pagination**

.. code:: python
tasks = api.tasks.list(paginate=False)
.. warning:: be aware that the unpaginated results may exceed
the data the parser can handle and may result in an error.

**Retrieve a single page**

.. code:: python
tasks_page_1 = api.tasks.list(page=1) # Will only return page 1
**Specify the page size**

.. code:: python
tasks_page_1 = api.tasks.list(page=1, page_size=200) # Will 200 results from page 1
Attach a file
~~~~~~~~~~~~~

Expand Down
50 changes: 47 additions & 3 deletions taiga/models/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,55 @@ def __init__(self, requester):

class ListResource(Resource):

def list(self, **queryparams):
def list(self, pagination=True, page_size=None, page=None, **queryparams):
"""
Retrieves a list of objects.
By default uses local cache and remote pagination
If pagination is used and no page is requested (the default), all the
remote objects are retrieved and appended in a single list.
If pagination is disabled, all the objects are fetched from the
endpoint and returned. This may trigger some parsing error if the
result set is very large.
:param pagination: Use pagination (default: `True`)
:param page_size: Size of the pagination page (default: `100`).
Any non numeric value will be casted to the
default value
:param page: Page number to retrieve (default: `None`). Ignored if
`pagination` is `False`
:param queryparams: Additional filter parameters as accepted by the
remote API
:return: <SearchableList>
"""
if page_size and pagination:
try:
page_size = int(page_size)
except (ValueError, TypeError):
page_size = 100
queryparams['page_size'] = page_size
result = self.requester.get(
self.instance.endpoint, query=queryparams
self.instance.endpoint, query=queryparams, paginate=pagination
)
objects = self.parse_list(result.json())
objects = SearchableList()
objects.extend(self.parse_list(result.json()))
if result.headers.get('X-Pagination-Next', False) and not page:
next_page = 2
else:
next_page = None
while next_page:
pageparams = queryparams.copy()
pageparams['page'] = next_page
result = self.requester.get(
self.instance.endpoint, query=pageparams,
)
objects.extend(self.parse_list(result.json()))
if result.headers.get('X-Pagination-Next', False):
next_page += 1
else:
next_page = None
return objects

def get(self, resource_id):
Expand Down
20 changes: 13 additions & 7 deletions taiga/requestmaker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from requests.packages.urllib3.exceptions import InsecureRequestWarning


def _disable_pagination():
def _requests_compatible_true():
if LooseVersion(requests.__version__) >= LooseVersion('2.11.0'):
return 'True'
else:
Expand Down Expand Up @@ -61,12 +61,15 @@ def __init__(self,
api_path, host,
token,
token_type='Bearer',
tls_verify=True):
tls_verify=True,
enable_pagination=True
):
self.api_path = api_path
self.host = host
self.token = token
self.token_type = token_type
self.tls_verify = tls_verify
self.enable_pagination = enable_pagination
self._cache = RequestCache()
if not self.tls_verify:
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
Expand All @@ -78,12 +81,15 @@ def cache(self):
def is_bad_response(self, response):
return 400 <= response.status_code <= 500

def headers(self):
def headers(self, paginate=True):
headers = {
'Content-type': 'application/json',
'Authorization': '{0} {1}'.format(self.token_type, self.token),
'x-disable-pagination': _disable_pagination()
}
if self.enable_pagination and paginate:
headers['x-lazy-pagination'] = _requests_compatible_true()
else:
headers['x-disable-pagination'] = _requests_compatible_true()
return headers

def urljoin(self, *parts):
Expand All @@ -96,7 +102,7 @@ def get_full_url(self, uri, query={}, **parameters):
)
return full_url

def get(self, uri, query={}, cache=False, **parameters):
def get(self, uri, query={}, cache=False, paginate=True, **parameters):
try:
full_url = self.urljoin(
self.host, self.api_path,
Expand All @@ -114,7 +120,7 @@ def get(self, uri, query={}, cache=False, **parameters):
if not result:
result = requests.get(
full_url,
headers=self.headers(),
headers=self.headers(paginate),
params=query,
verify=self.tls_verify
)
Expand All @@ -137,7 +143,7 @@ def post(self, uri, payload=None, query={}, files={}, **parameters):
if files:
headers = {
'Authorization': '{0} {1}'.format(self.token_type, self.token),
'x-disable-pagination': _disable_pagination()
'x-disable-pagination': _requests_compatible_true()
}
data = payload
else:
Expand Down
29 changes: 29 additions & 0 deletions tests/resources/fakes_list_success.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
[
{
"id": 1
},
{
"id": 2
},
{
"id": 3
},
{
"id": 4
},
{
"id": 5
},
{
"id": 6
},
{
"id": 7
},
{
"id": 8
},
{
"id": 9
}
]
3 changes: 3 additions & 0 deletions tests/test_issues.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,14 @@ class TestIssues(unittest.TestCase):

@patch('taiga.requestmaker.RequestMaker.get')
def test_list_attachments(self, mock_requestmaker_get):
mock_requestmaker_get.return_value = MockResponse(200,
create_mock_json('tests/resources/issues_list_success.json'))
rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')
Issue(rm, id=1).list_attachments()
mock_requestmaker_get.assert_called_with(
'issues/attachments',
query={"object_id": 1},
paginate=True
)

@patch('taiga.requestmaker.RequestMaker.post')
Expand Down
128 changes: 122 additions & 6 deletions tests/test_model_base.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
# -*- coding: utf-8 -*-
import json

from taiga.models import Projects
from taiga.requestmaker import RequestMaker
from taiga.models.base import InstanceResource, ListResource, SearchableList
import unittest
from mock import patch
import datetime
from .tools import MockResponse
from .tools import MockResponse, create_mock_json


class Fake(InstanceResource):
Expand All @@ -18,14 +20,28 @@ class Fake(InstanceResource):

def my_method(self):
response = self.requester.get('/users/{id}/starred', id=self.id)
return projects.Projects.parse(response.json(), self.requester)
return Projects.parse(response.json(), self.requester)


class Fakes(ListResource):

instance = Fake


class FakeHeaders(dict):
sequence = []
counter = -1

def __init__(self, sequence=[], *args, **kwargs):
self.sequence = sequence
self.counter = -1
super(FakeHeaders, self).__init__(*args, **kwargs)

def get(self, k, d=None):
self.counter += 1
return self.sequence[self.counter]


class TestModelBase(unittest.TestCase):

def test_encoding(self):
Expand Down Expand Up @@ -136,12 +152,112 @@ def test_call_model_base_delete_element_from_list(self, mock_requestmaker_delete

@patch('taiga.requestmaker.RequestMaker.get')
def test_call_model_base_list_elements(self, mock_requestmaker_get):
js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json'))
rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')
fakes = Fakes(rm)

data = json.dumps(js_list)
mock_requestmaker_get.return_value = MockResponse(200, data)
f_list = fakes.list()
mock_requestmaker_get.assert_called_with('fakes', query={}, paginate=True)
self.assertEqual(len(f_list), 9)

data = json.dumps(js_list[0])
mock_requestmaker_get.return_value = MockResponse(200, data)
f_list = fakes.list(id=1)
mock_requestmaker_get.assert_called_with('fakes', query={'id':1}, paginate=True)
self.assertEqual(len(f_list), 1)

@patch('taiga.requestmaker.RequestMaker.get')
def test_call_model_base_list_page_size(self, mock_requestmaker_get):
js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json'))
rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')
fakes = Fakes(rm)

data = json.dumps(js_list)
mock_requestmaker_get.return_value = MockResponse(200, data, FakeHeaders(
[True, True, False],
**{'X-Pagination-Next': True}
))
f_list = fakes.list(page_size=2)
mock_requestmaker_get.assert_called_with('fakes', query={'page': 3, 'page_size': 2})
self.assertEqual(len(f_list), 27)

data = json.dumps(js_list)
mock_requestmaker_get.return_value = MockResponse(200, data)
f_list = fakes.list(page_size='wrong')
mock_requestmaker_get.assert_called_with('fakes', query={'page_size': 100}, paginate=True)
self.assertEqual(len(f_list), 9)

@patch('taiga.requestmaker.RequestMaker.get')
def test_call_model_base_list_elements_no_paginate(self, mock_requestmaker_get):
js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json'))
rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')
fakes = Fakes(rm)

data = json.dumps(js_list)
mock_requestmaker_get.return_value = MockResponse(200, data)
f_list = fakes.list(pagination=False)
mock_requestmaker_get.assert_called_with('fakes', query={}, paginate=False)
self.assertEqual(len(f_list), 9)

data = json.dumps(js_list[0])
mock_requestmaker_get.return_value = MockResponse(200, data)
f_list = fakes.list(id=1, pagination=False)
mock_requestmaker_get.assert_called_with('fakes', query={'id':1}, paginate=False)
self.assertEqual(len(f_list), 1)

@patch('taiga.requestmaker.requests.get')
def test_call_model_base_list_elements_no_paginate_check_requests(self, mock_requestmaker_get):
js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json'))
rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')
fakes = Fakes(rm)

data = json.dumps(js_list)
mock_requestmaker_get.return_value = MockResponse(200, data)
f_list = fakes.list(pagination=False)
mock_requestmaker_get.assert_called_with(
'fakehost/api/v1/fakes', verify=True, params={},
headers={
'x-disable-pagination': 'True', 'Content-type': 'application/json', 'Authorization': 'Bearer faketoken'
}
)
self.assertEqual(len(f_list), 9)

@patch('taiga.requestmaker.requests.get')
def test_call_model_base_list_elements_paginate_check_requests(self, mock_requestmaker_get):
js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json'))
rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')
fakes = Fakes(rm)
fakes.list()
mock_requestmaker_get.assert_called_with('fakes', query={})
fakes.list(project_id=1)
mock_requestmaker_get.assert_called_with('fakes', query={'project_id':1})

data = json.dumps(js_list)
mock_requestmaker_get.return_value = MockResponse(200, data)
f_list = fakes.list()
mock_requestmaker_get.assert_called_with(
'fakehost/api/v1/fakes', verify=True, params={},
headers={
'x-lazy-pagination': 'True', 'Content-type': 'application/json', 'Authorization': 'Bearer faketoken'
}
)
self.assertEqual(len(f_list), 9)

@patch('taiga.requestmaker.RequestMaker.get')
def test_call_model_base_list_elements_single_page(self, mock_requestmaker_get):
js_list = json.loads(create_mock_json('tests/resources/fakes_list_success.json'))
rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')
fakes = Fakes(rm)

data = json.dumps(js_list[:5])
mock_requestmaker_get.return_value = MockResponse(200, data)
f_list = fakes.list(page_size=5, page=1)
self.assertEqual(len(f_list), 5)
mock_requestmaker_get.assert_called_with('fakes', query={'page_size': 5}, paginate=True)

data = json.dumps(js_list[5:])
mock_requestmaker_get.return_value = MockResponse(200, data)
f_list = fakes.list(page_size=5, page=2)
self.assertEqual(len(f_list), 4)
mock_requestmaker_get.assert_called_with('fakes', query={'page_size': 5}, paginate=True)

def test_to_dict_method(self):
rm = RequestMaker('/api/v1', 'fakehost', 'faketoken')
Expand Down
Loading

0 comments on commit 618cb2a

Please sign in to comment.