Skip to content

Commit

Permalink
fix(indeed): offset
Browse files Browse the repository at this point in the history
  • Loading branch information
cullenwatson committed Oct 23, 2024
1 parent 6372e41 commit f395597
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 10 deletions.
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "python-jobspy"
version = "1.1.71"
version = "1.1.72"
description = "Job scraper for LinkedIn, Indeed, Glassdoor & ZipRecruiter"
authors = ["Zachary Hampton <[email protected]>", "Cullen Watson <[email protected]>"]
homepage = "https://github.com/Bunsly/JobSpy"
Expand Down
16 changes: 7 additions & 9 deletions src/jobspy/scrapers/indeed/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,15 +69,8 @@ def scrape(self, scraper_input: ScraperInput) -> JobResponse:
page = 1

cursor = None
offset_pages = math.ceil(self.scraper_input.offset / 100)
for _ in range(offset_pages):
logger.info(f"skipping search page: {page}")
__, cursor = self._scrape_page(cursor)
if not __:
logger.info(f"found no jobs on page: {page}")
break

while len(self.seen_urls) < scraper_input.results_wanted:
while len(self.seen_urls) < scraper_input.results_wanted + scraper_input.offset:
logger.info(
f"search page: {page} / {math.ceil(scraper_input.results_wanted / 100)}"
)
Expand All @@ -87,7 +80,12 @@ def scrape(self, scraper_input: ScraperInput) -> JobResponse:
break
job_list += jobs
page += 1
return JobResponse(jobs=job_list[: scraper_input.results_wanted])
return JobResponse(
jobs=job_list[
scraper_input.offset : scraper_input.offset
+ scraper_input.results_wanted
]
)

def _scrape_page(self, cursor: str | None) -> Tuple[list[JobPost], str | None]:
"""
Expand Down

1 comment on commit f395597

@alisalamin
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for the quick turn around!

Please sign in to comment.