34 lines
1.1 KiB
Python
Raw Normal View History

from argparse import Namespace
import requests.exceptions
from requests import Session
from requests.adapters import HTTPAdapter
from urllib3.util import Retry
from src.logger import logger as l
def fetch(settings: list, args: Namespace) -> None:
logger = l.getChild(__name__)
for sets in settings:
logger.info(f"Working set: {sets}")
assert type(sets) == dict
assert 'src' in sets
assert 'rss' in sets
s = Session()
retries = Retry(
total=3,
backoff_factor=0.1,
status_forcelist=[502, 503, 504],
allowed_methods={'GET'},
)
s.mount(sets['src'], HTTPAdapter(max_retries=retries))
try:
r = s.get(sets['src'], timeout=3, stream=True)
if r.status_code == 200:
logger.info(f"Saving to file: {sets['rss']}")
with open("{0}/{1}".format(args.directory, sets['rss']), 'wb') as rss:
for chunk in r:
rss.write(chunk)
except requests.exceptions.ConnectionError as e:
logger.warning(f"Unable to fetch {sets['src']}", e)