From 553bfe5f9836da0b1369e78db554a162a7530ffd Mon Sep 17 00:00:00 2001 From: Martin Collignon <2604526+martincollignon@users.noreply.github.com> Date: Tue, 10 Dec 2024 20:25:53 +0100 Subject: [PATCH] fix(wetlands): add missing _fetch_chunk method for batch processing --- backend/src/sources/parsers/wetlands.py | 28 ++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/backend/src/sources/parsers/wetlands.py b/backend/src/sources/parsers/wetlands.py index 74881150..f256a1a5 100644 --- a/backend/src/sources/parsers/wetlands.py +++ b/backend/src/sources/parsers/wetlands.py @@ -262,4 +262,30 @@ async def write_to_storage(self, features, dataset): async def fetch(self): """Not implemented - using sync() directly""" - raise NotImplementedError("This source uses sync() directly") \ No newline at end of file + raise NotImplementedError("This source uses sync() directly") + + async def _fetch_chunk(self, session, start_index): + """Fetch a chunk of features starting at the given index""" + try: + async with self.request_semaphore: + params = self._get_params(start_index) + async with session.get( + self.config['url'], + params=params, + timeout=self.request_timeout + ) as response: + text = await response.text() + root = ET.fromstring(text) + + features = [ + self._parse_feature(f) + for f in root.findall('.//natur:kulstof2022', self.namespaces) + ] + return [f for f in features if f] + + except asyncio.TimeoutError: + logger.error(f"Timeout fetching batch at {start_index}") + return None + except Exception as e: + logger.error(f"Error fetching batch at {start_index}: {str(e)}") + return None \ No newline at end of file