208 lines
7.4 KiB
Python
208 lines
7.4 KiB
Python
from hashlib import md5
|
|
from datetime import date, timedelta
|
|
from time import sleep
|
|
from logging import getLogger
|
|
from ipaddress import IPv4Address
|
|
from urllib.error import HTTPError
|
|
from geoip2.database import Reader
|
|
from tarfile import open as taropen
|
|
from urllib3 import disable_warnings
|
|
from os import stat, remove, makedirs
|
|
from urllib.request import urlretrieve
|
|
from json.decoder import JSONDecodeError
|
|
from os.path import abspath, join, basename, isdir
|
|
from urllib3.exceptions import InsecureRequestWarning
|
|
from requests.exceptions import InvalidSchema, SSLError, ConnectionError, ChunkedEncodingError
|
|
|
|
logger = getLogger()
|
|
|
|
|
|
class GeoIPHandler(object):
|
|
def __init__(self, data_folder):
|
|
self.data_folder = data_folder
|
|
self.dbfile = abspath(join(self.data_folder, 'GeoLite2-City.mmdb'))
|
|
self.logger = getLogger()
|
|
self.reader = None
|
|
self.reader_manager(action='open')
|
|
|
|
self.logger.info('Opening persistent connection to GeoLite2 DB...')
|
|
|
|
def reader_manager(self, action=None):
|
|
if action == 'open':
|
|
try:
|
|
self.reader = Reader(self.dbfile)
|
|
except FileNotFoundError:
|
|
self.logger.error("Could not find GeoLite2 DB! Downloading!")
|
|
result_status = self.download()
|
|
if result_status:
|
|
self.logger.error("Could not download GeoLite2 DB!!!, You may need to manually install it.")
|
|
exit(1)
|
|
else:
|
|
self.reader = Reader(self.dbfile)
|
|
else:
|
|
self.reader.close()
|
|
|
|
def lookup(self, ipaddress):
|
|
ip = ipaddress
|
|
self.logger.debug('Getting lat/long for Tautulli stream using ip with last octet ending in %s',
|
|
ip.split('.')[-1:][0])
|
|
return self.reader.city(ip)
|
|
|
|
def update(self):
|
|
today = date.today()
|
|
|
|
try:
|
|
dbdate = date.fromtimestamp(stat(self.dbfile).st_mtime)
|
|
db_next_update = date.fromtimestamp(stat(self.dbfile).st_mtime) + timedelta(days=60)
|
|
|
|
except FileNotFoundError:
|
|
self.logger.error("Could not find GeoLite2 DB as: %s", self.dbfile)
|
|
self.download()
|
|
dbdate = date.fromtimestamp(stat(self.dbfile).st_mtime)
|
|
db_next_update = date.fromtimestamp(stat(self.dbfile).st_mtime) + timedelta(days=60)
|
|
|
|
if db_next_update < today:
|
|
self.logger.info("Newer GeoLite2 DB available, Updating...")
|
|
self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s",
|
|
dbdate, db_next_update, today)
|
|
self.reader_manager(action='close')
|
|
self.download()
|
|
self.reader_manager(action='open')
|
|
else:
|
|
db_days_update = db_next_update - today
|
|
self.logger.debug("Geolite2 DB will update in %s days", abs(db_days_update.days))
|
|
self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s",
|
|
dbdate, db_next_update, today)
|
|
|
|
def download(self):
|
|
tar_dbfile = abspath(join(self.data_folder, 'GeoLite2-City.tar.gz'))
|
|
url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz'
|
|
downloaded = False
|
|
|
|
retry_counter = 0
|
|
|
|
while not downloaded:
|
|
self.logger.info('Downloading GeoLite2 from %s', url)
|
|
try:
|
|
urlretrieve(url, tar_dbfile)
|
|
downloaded = True
|
|
except HTTPError as e:
|
|
self.logger.error("Problem downloading new GeoLite2 DB... Trying again. Error: %s", e)
|
|
sleep(2)
|
|
retry_counter = (retry_counter + 1)
|
|
|
|
if retry_counter >= 3:
|
|
self.logger.error("Retried downloading the new GeoLite2 DB 3 times and failed... Aborting!")
|
|
result_status = 1
|
|
return result_status
|
|
try:
|
|
remove(self.dbfile)
|
|
except FileNotFoundError:
|
|
self.logger.warning("Cannot remove GeoLite2 DB as it does not exist!")
|
|
|
|
self.logger.debug("Opening GeoLite2 tar file : %s", tar_dbfile)
|
|
|
|
tar = taropen(tar_dbfile, 'r:gz')
|
|
|
|
for files in tar.getmembers():
|
|
if 'GeoLite2-City.mmdb' in files.name:
|
|
self.logger.debug('"GeoLite2-City.mmdb" FOUND in tar file')
|
|
files.name = basename(files.name)
|
|
tar.extract(files, self.data_folder)
|
|
self.logger.debug('%s has been extracted to %s', files, self.data_folder)
|
|
tar.close()
|
|
try:
|
|
remove(tar_dbfile)
|
|
self.logger.debug('Removed the GeoLite2 DB TAR file.')
|
|
except FileNotFoundError:
|
|
self.logger.warning("Cannot remove GeoLite2 DB TAR file as it does not exist!")
|
|
|
|
|
|
def hashit(string):
|
|
encoded = string.encode()
|
|
hashed = md5(encoded).hexdigest()
|
|
|
|
return hashed
|
|
|
|
|
|
def rfc1918_ip_check(ip):
|
|
rfc1918_ip = IPv4Address(ip).is_private
|
|
|
|
return rfc1918_ip
|
|
|
|
|
|
def connection_handler(session, request, verify, as_is_reply=False):
|
|
air = as_is_reply
|
|
s = session
|
|
r = request
|
|
v = verify
|
|
return_json = False
|
|
|
|
disable_warnings(InsecureRequestWarning)
|
|
|
|
try:
|
|
get = s.send(r, verify=v)
|
|
if get.status_code == 401:
|
|
if 'NoSiteContext' in str(get.content):
|
|
logger.info('Your Site is incorrect for %s', r.url)
|
|
elif 'LoginRequired' in str(get.content):
|
|
logger.info('Your login credentials are incorrect for %s', r.url)
|
|
else:
|
|
logger.info('Your api key is incorrect for %s', r.url)
|
|
elif get.status_code == 404:
|
|
logger.info('This url doesnt even resolve: %s', r.url)
|
|
elif get.status_code == 200:
|
|
try:
|
|
return_json = get.json()
|
|
except JSONDecodeError:
|
|
logger.error('No JSON response. Response is: %s', get.text)
|
|
if air:
|
|
return get
|
|
except InvalidSchema:
|
|
logger.error("You added http(s):// in the config file. Don't do that.")
|
|
except SSLError as e:
|
|
logger.error('Either your host is unreachable or you have an SSL issue. : %s', e)
|
|
except ConnectionError as e:
|
|
logger.error('Cannot resolve the url/ip/port. Check connectivity. Error: %s', e)
|
|
except ChunkedEncodingError as e:
|
|
logger.error('Broken connection during request... oops? Error: %s', e)
|
|
|
|
return return_json
|
|
|
|
|
|
def mkdir_p(path):
|
|
templogger = getLogger('temp')
|
|
try:
|
|
if not isdir(path):
|
|
templogger.info('Creating folder %s ', path)
|
|
makedirs(path, exist_ok=True)
|
|
except Exception as e:
|
|
templogger.error('Could not create folder %s : %s ', path, e)
|
|
|
|
|
|
def clean_sid_check(server_id_list, server_type=None):
|
|
t = server_type
|
|
sid_list = server_id_list
|
|
cleaned_list = sid_list.replace(' ', '').split(',')
|
|
valid_sids = []
|
|
for sid in cleaned_list:
|
|
try:
|
|
valid_sids.append(int(sid))
|
|
except ValueError:
|
|
logger.error("%s is not a valid server id number", sid)
|
|
if valid_sids:
|
|
logger.info('%s : %s', t.upper(), valid_sids)
|
|
return valid_sids
|
|
else:
|
|
logger.error('No valid %s', t.upper())
|
|
return False
|
|
|
|
|
|
def boolcheck(var):
|
|
if var.lower() in ['true', 'yes']:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def itemgetter_with_default(defaults):
|
|
return lambda obj: {k: obj.get(k, v) for k, v in defaults.items()}
|