d_mcknight/Varken
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improve MaxMind DB logging and error catching

Browse source
This commit is contained in:
Phillip Hernandez 2019-12-31 15:43:08 -05:00
parent 075780ea76
commit 855201160a
1 changed files with 26 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

41
varken/helpers.py
View file

@ -18,24 +18,26 @@ logger = getLogger()
class GeoIPHandler(object): class GeoIPHandler(object):
def __init__(self, data_folder): def __init__(self, data_folder, maxmind_license_key):
self.data_folder = data_folder self.data_folder = data_folder
self.maxmind_license_key = maxmind_license_key
self.dbfile = abspath(join(self.data_folder, 'GeoLite2-City.mmdb')) self.dbfile = abspath(join(self.data_folder, 'GeoLite2-City.mmdb'))
self.logger = getLogger() self.logger = getLogger()
self.reader = None self.reader = None
self.reader_manager(action='open') self.reader_manager(action='open')
self.logger.info('Opening persistent connection to GeoLite2 DB...') self.logger.info('Opening persistent connection to the MaxMind DB...')
def reader_manager(self, action=None): def reader_manager(self, action=None):
if action == 'open': if action == 'open':
try: try:
self.reader = Reader(self.dbfile) self.reader = Reader(self.dbfile)
except FileNotFoundError: except FileNotFoundError:
self.logger.error("Could not find GeoLite2 DB! Downloading!") self.logger.error("Could not find MaxMind DB! Downloading!")
result_status = self.download() result_status = self.download()
if result_status: if result_status:
self.logger.error("Could not download GeoLite2 DB!!!, You may need to manually install it.") self.logger.error("Could not download MaxMind DB! You may need to manually install it.")
exit(1) exit(1)
else: else:
self.reader = Reader(self.dbfile) self.reader = Reader(self.dbfile)
@ -56,22 +58,22 @@ class GeoIPHandler(object):
db_next_update = date.fromtimestamp(stat(self.dbfile).st_mtime) + timedelta(days=30) db_next_update = date.fromtimestamp(stat(self.dbfile).st_mtime) + timedelta(days=30)
except FileNotFoundError: except FileNotFoundError:
self.logger.error("Could not find GeoLite2 DB as: %s", self.dbfile) self.logger.error("Could not find MaxMind DB as: %s", self.dbfile)
self.download() self.download()
dbdate = date.fromtimestamp(stat(self.dbfile).st_mtime) dbdate = date.fromtimestamp(stat(self.dbfile).st_mtime)
db_next_update = date.fromtimestamp(stat(self.dbfile).st_mtime) + timedelta(days=30) db_next_update = date.fromtimestamp(stat(self.dbfile).st_mtime) + timedelta(days=30)
if db_next_update < today: if db_next_update < today:
self.logger.info("Newer GeoLite2 DB available, Updating...") self.logger.info("Newer MaxMind DB available, Updating...")
self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", self.logger.debug("MaxMind DB date %s, DB updates after: %s, Today: %s",
dbdate, db_next_update, today) dbdate, db_next_update, today)
self.reader_manager(action='close') self.reader_manager(action='close')
self.download() self.download()
self.reader_manager(action='open') self.reader_manager(action='open')
else: else:
db_days_update = db_next_update - today db_days_update = db_next_update - today
self.logger.debug("Geolite2 DB will update in %s days", abs(db_days_update.days)) self.logger.debug("MaxMind DB will update in %s days", abs(db_days_update.days))
self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", self.logger.debug("MaxMind DB date %s, DB updates after: %s, Today: %s",
dbdate, db_next_update, today) dbdate, db_next_update, today)
def download(self): def download(self):
@ -83,25 +85,30 @@ class GeoIPHandler(object):
retry_counter = 0 retry_counter = 0
while not downloaded: while not downloaded:
self.logger.info('Downloading GeoLite2 from %s', url) self.logger.info('Downloading GeoLite2 DB from MaxMind...')
try: try:
urlretrieve(url, tar_dbfile) urlretrieve(maxmind_url, tar_dbfile)
downloaded = True downloaded = True
except HTTPError as e: except HTTPError as e:
self.logger.error("Problem downloading new GeoLite2 DB... Trying again. Error: %s", e) if e.code == 401:
self.logger.error("Your MaxMind license key is incorect! Check your config: %s", e)
result_status = 1
return result_status
else:
self.logger.error("Problem downloading new MaxMind DB... Trying again: %s", e)
sleep(2) sleep(2)
retry_counter = (retry_counter + 1) retry_counter = (retry_counter + 1)
if retry_counter >= 3: if retry_counter >= 3:
self.logger.error("Retried downloading the new GeoLite2 DB 3 times and failed... Aborting!") self.logger.error("Retried downloading the new MaxMind DB 3 times and failed... Aborting!")
result_status = 1 result_status = 1
return result_status return result_status
try: try:
remove(self.dbfile) remove(self.dbfile)
except FileNotFoundError: except FileNotFoundError:
self.logger.warning("Cannot remove GeoLite2 DB as it does not exist!") self.logger.warning("Cannot remove MaxMind DB as it does not exist!")
self.logger.debug("Opening GeoLite2 tar file : %s", tar_dbfile) self.logger.debug("Opening MaxMind tar file : %s", tar_dbfile)
tar = taropen(tar_dbfile, 'r:gz') tar = taropen(tar_dbfile, 'r:gz')
@ -114,9 +121,9 @@ class GeoIPHandler(object):
tar.close() tar.close()
try: try:
remove(tar_dbfile) remove(tar_dbfile)
self.logger.debug('Removed the GeoLite2 DB TAR file.') self.logger.debug('Removed the MaxMind DB tar file.')
except FileNotFoundError: except FileNotFoundError:
self.logger.warning("Cannot remove GeoLite2 DB TAR file as it does not exist!") self.logger.warning("Cannot remove MaxMind DB TAR file as it does not exist!")
def hashit(string): def hashit(string):