Skip to content

Commit

Permalink
feat(scraper): factor out loop sleep.
Browse files Browse the repository at this point in the history
  • Loading branch information
2e0byo committed Feb 1, 2022
1 parent 139137c commit bd6da75
Showing 1 changed file with 10 additions and 7 deletions.
17 changes: 10 additions & 7 deletions yadc/scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,15 @@ def error_callback(self):
"""Run something when we have an error."""
pass

def loop_sleep(self, period):
"""Sleep fn for main loop."""
period = max(period - 5, 0)
self._logger.debug(f"Sleeping for ~{period}s.")
if period:
randsleep(period, period + 45)
self._logger.debug(f"Sleeping for ~5s to introduce some randomness.")
randsleep(5)

def __call__(self):
while True:
self._logger.info("Starting search loop")
Expand Down Expand Up @@ -444,13 +453,7 @@ def __call__(self):
self.find_tests(browser, driver)
end = datetime.now()
period = self.time_to_next_event(elapsed=end - start)
period = max(period - 5, 0)
self._logger.debug(f"Sleeping for {period}s.")
spinner_sleep(period)
self._logger.debug(
f"Sleeping for ~5s to introduce some randomness."
)
randsleep(5)
self.loop_sleep(period)
searched += 1
except Exception as e:
errored += 1
Expand Down

0 comments on commit bd6da75

Please # to comment.