From 556109e8c5d683eade727ddf1315cd6a22c3cb14 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 17 Dec 2018 23:47:29 -0600 Subject: [PATCH] v --- Varken.py | 6 +++- varken/__init__.py | 2 +- varken/helpers.py | 88 +++++++++++++++++++++++++-------------------- varken/iniparser.py | 2 -- varken/tautulli.py | 12 +++---- 5 files changed, 62 insertions(+), 48 deletions(-) diff --git a/Varken.py b/Varken.py index cfc6f05..bb7e03e 100644 --- a/Varken.py +++ b/Varken.py @@ -17,10 +17,12 @@ from varken.sonarr import SonarrAPI from varken.radarr import RadarrAPI from varken.iniparser import INIParser from varken.dbmanager import DBManager +from varken.helpers import GeoIPHandler from varken.tautulli import TautulliAPI from varken.sickchill import SickChillAPI from varken.varkenlogger import VarkenLogger + PLATFORM_LINUX_DISTRO = ' '.join(x for x in linux_distribution() if x) @@ -95,8 +97,10 @@ if __name__ == "__main__": schedule.every(server.future_days_run_seconds).seconds.do(threaded, SONARR.get_future) if CONFIG.tautulli_enabled: + GEOIPHANDLER = GeoIPHandler(DATA_FOLDER) + schedule.every(12).to(24).hours.do(threaded, GEOIPHANDLER.update) for server in CONFIG.tautulli_servers: - TAUTULLI = TautulliAPI(server, DBMANAGER, DATA_FOLDER) + TAUTULLI = TautulliAPI(server, DBMANAGER, GEOIPHANDLER) if server.get_activity: schedule.every(server.get_activity_run_seconds).seconds.do(threaded, TAUTULLI.get_activity) diff --git a/varken/__init__.py b/varken/__init__.py index 314c9c8..ba78dc4 100644 --- a/varken/__init__.py +++ b/varken/__init__.py @@ -1,2 +1,2 @@ VERSION = 1.3 -BRANCH = 'pre-nightly' +BRANCH = 'nightly' diff --git a/varken/helpers.py b/varken/helpers.py index eec1ec8..88426b3 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -1,7 +1,8 @@ -from time import time from hashlib import md5 -from tarfile import open +from tarfile import open as taropen +from datetime import date from logging import getLogger +from calendar import monthcalendar from geoip2.database import Reader from urllib3 import disable_warnings from os import stat, remove, makedirs @@ -14,52 +15,63 @@ from requests.exceptions import InvalidSchema, SSLError, ConnectionError logger = getLogger() -def geoip_download(data_folder): - datafolder = data_folder +class GeoIPHandler(object): + def __init__(self, data_folder): + self.data_folder = data_folder + self.dbfile = abspath(join(self.data_folder, 'GeoLite2-City.mmdb')) + self.logger = getLogger() + self.update() - tar_dbfile = abspath(join(datafolder, 'GeoLite2-City.tar.gz')) + self.logger.info('Opening persistent connection to GeoLite2 DB...') + self.reader = Reader(self.dbfile) - url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz' - logger.info('Downloading GeoLite2 from %s', url) - urlretrieve(url, tar_dbfile) + def lookup(self, ipaddress): + ip = ipaddress + self.logger.debug('Getting lat/long for Tautulli stream') + return self.reader.city(ip) - tar = open(tar_dbfile, 'r:gz') - logger.debug('Opening GeoLite2 tar file : %s', tar_dbfile) + def update(self): + today = date.today() + dbdate = None + try: + dbdate = date.fromtimestamp(stat(self.dbfile).st_ctime) + except FileNotFoundError: + self.logger.error("Could not find GeoLite2 DB as: %s", self.dbfile) + self.download() + first_wednesday_day = [week[2:3][0] for week in monthcalendar(today.year, today.month) if week[2:3][0] != 0][0] + first_wednesday_date = date(today.year, today.month, first_wednesday_day) - for files in tar.getmembers(): - if 'GeoLite2-City.mmdb' in files.name: - logger.debug('"GeoLite2-City.mmdb" FOUND in tar file') - files.name = basename(files.name) - - tar.extract(files, datafolder) - logger.debug('%s has been extracted to %s', files, datafolder) - - remove(tar_dbfile) + if dbdate < first_wednesday_date < today: + self.logger.info("Newer GeoLite2 DB available, Updating...") + remove(self.dbfile) + self.download() + else: + td = first_wednesday_date - today + if td.days < 0: + self.logger.debug('Geolite2 DB is only %s days old. Keeping current copy', abs(td.days)) + else: + self.logger.debug('Geolite2 DB will update in %s days', abs(td.days)) -def geo_lookup(ipaddress, data_folder): - datafolder = data_folder - logger.debug('Reading GeoLite2 DB from %s', datafolder) + def download(self): + tar_dbfile = abspath(join(self.data_folder, 'GeoLite2-City.tar.gz')) + url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz' - dbfile = abspath(join(datafolder, 'GeoLite2-City.mmdb')) - now = time() + self.logger.info('Downloading GeoLite2 from %s', url) + urlretrieve(url, tar_dbfile) - try: - dbinfo = stat(dbfile) - db_age = now - dbinfo.st_ctime - if db_age > (35 * 86400): - logger.info('GeoLite2 DB is older than 35 days. Attempting to re-download...') + self.logger.debug('Opening GeoLite2 tar file : %s', tar_dbfile) - remove(dbfile) + tar = taropen(tar_dbfile, 'r:gz') - geoip_download(datafolder) - except FileNotFoundError: - logger.error('GeoLite2 DB not found. Attempting to download...') - geoip_download(datafolder) - - reader = Reader(dbfile) - - return reader.city(ipaddress) + for files in tar.getmembers(): + if 'GeoLite2-City.mmdb' in files.name: + self.logger.debug('"GeoLite2-City.mmdb" FOUND in tar file') + files.name = basename(files.name) + tar.extract(files, self.data_folder) + self.logger.debug('%s has been extracted to %s', files, self.data_folder) + tar.close() + remove(tar_dbfile) def hashit(string): diff --git a/varken/iniparser.py b/varken/iniparser.py index b813af0..29144fc 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -75,8 +75,6 @@ class INIParser(object): regex = compile('{}'.format(search), IGNORECASE) - print(match(regex, url_check)) - valid = match(regex, url_check) is not None if not valid: if inc_port: diff --git a/varken/tautulli.py b/varken/tautulli.py index f71ce81..896278f 100644 --- a/varken/tautulli.py +++ b/varken/tautulli.py @@ -4,18 +4,18 @@ from datetime import datetime, timezone from geoip2.errors import AddressNotFoundError from varken.structures import TautulliStream -from varken.helpers import geo_lookup, hashit, connection_handler +from varken.helpers import hashit, connection_handler class TautulliAPI(object): - def __init__(self, server, dbmanager, data_folder): + def __init__(self, server, dbmanager, geoiphandler): self.dbmanager = dbmanager self.server = server + self.geoiphandler = geoiphandler self.session = Session() self.session.params = {'apikey': self.server.api_key, 'cmd': 'get_activity'} self.endpoint = '/api/v2' self.logger = getLogger() - self.data_folder = data_folder def __repr__(self): return f"" @@ -47,13 +47,13 @@ class TautulliAPI(object): exit(1) try: - geodata = geo_lookup(session.ip_address_public, self.data_folder) + geodata = self.geoiphandler.lookup(session.ip_address_public) except (ValueError, AddressNotFoundError): if self.server.fallback_ip: - geodata = geo_lookup(self.server.fallback_ip, self.data_folder) + geodata = self.geoiphandler.lookup(self.server.fallback_ip) else: my_ip = self.session.get('http://ip.42.pl/raw').text - geodata = geo_lookup(my_ip, self.data_folder) + geodata = self.geoiphandler.lookup(my_ip) if not all([geodata.location.latitude, geodata.location.longitude]): latitude = 37.234332396