47 lines
1.4 KiB
Python
47 lines
1.4 KiB
Python
|
import argparse
|
||
|
|
import json
|
||
|
|
|
||
|
|
import requests.exceptions
|
||
|
|
from requests import Session
|
||
|
|
from requests.adapters import HTTPAdapter
|
||
|
|
from urllib3.util import Retry
|
||
|
|
from src.logger import logger as l
|
||
|
|
|
||
|
|
|
||
|
|
logger = l.getChild(__name__)
|
||
|
|
logger.info("Starting")
|
||
|
|
|
||
|
|
parser = argparse.ArgumentParser(__name__)
|
||
|
|
parser.add_argument('-s', '--settings', required=True, help='Settings JSON')
|
||
|
|
parser.add_argument('-d', '--directory', required=True, help='Directory to store XMLs')
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
logger.info("Loading settings from settings JSON")
|
||
|
|
settings = []
|
||
|
|
with open(args.settings) as json_file:
|
||
|
|
settings = json.load(json_file)
|
||
|
|
|
||
|
|
assert type(settings) == list
|
||
|
|
|
||
|
|
for sets in settings:
|
||
|
|
logger.info(f"Working set: {sets}")
|
||
|
|
assert type(sets) == dict
|
||
|
|
assert 'src' in sets
|
||
|
|
assert 'rss' in sets
|
||
|
|
s = Session()
|
||
|
|
retries = Retry(
|
||
|
|
total=3,
|
||
|
|
backoff_factor=0.1,
|
||
|
|
status_forcelist=[502, 503, 504],
|
||
|
|
allowed_methods={'GET'},
|
||
|
|
)
|
||
|
|
s.mount(sets['src'], HTTPAdapter(max_retries=retries))
|
||
|
|
try:
|
||
|
|
r = s.get(sets['src'], timeout=3, stream=True)
|
||
|
|
if r.status_code == 200:
|
||
|
|
logger.info(f"Saving to file: {sets['rss']}")
|
||
|
|
with open("{0}/{1}".format(args.directory, sets['rss']), 'wb') as rss:
|
||
|
|
for chunk in r:
|
||
|
|
rss.write(chunk)
|
||
|
|
except requests.exceptions.ConnectionError as e:
|
||
|
|
logger.warning(f"Unable to fetch {sets['src']}", e)
|