Skip to content

Commit

Permalink
Merge pull request #174 from martincollignon/fix/wetlands-efficient-m…
Browse files Browse the repository at this point in the history
…erge

fix(wetlands): add missing _fetch_chunk method for batch processing
  • Loading branch information
martincollignon authored Dec 10, 2024
2 parents 27cbe3d + 553bfe5 commit 86056d7
Showing 1 changed file with 27 additions and 1 deletion.
28 changes: 27 additions & 1 deletion backend/src/sources/parsers/wetlands.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,4 +262,30 @@ async def write_to_storage(self, features, dataset):

async def fetch(self):
"""Not implemented - using sync() directly"""
raise NotImplementedError("This source uses sync() directly")
raise NotImplementedError("This source uses sync() directly")

async def _fetch_chunk(self, session, start_index):
"""Fetch a chunk of features starting at the given index"""
try:
async with self.request_semaphore:
params = self._get_params(start_index)
async with session.get(
self.config['url'],
params=params,
timeout=self.request_timeout
) as response:
text = await response.text()
root = ET.fromstring(text)

features = [
self._parse_feature(f)
for f in root.findall('.//natur:kulstof2022', self.namespaces)
]
return [f for f in features if f]

except asyncio.TimeoutError:
logger.error(f"Timeout fetching batch at {start_index}")
return None
except Exception as e:
logger.error(f"Error fetching batch at {start_index}: {str(e)}")
return None

0 comments on commit 86056d7

Please sign in to comment.