From 693e25c7b6e27cce59ff25b6a8cfc90d74c74769 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Sun, 9 Dec 2018 21:57:16 -0600 Subject: [PATCH 01/76] final v1.0 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d017bd1..715bb76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ - Add cisco asa from legacy [\#44](https://github.com/Boerderij/Varken/issues/44) - Add server ID to ombi to differenciate [\#43](https://github.com/Boerderij/Varken/issues/43) +**Merged pull requests:** + +- v1.0 Merge [\#45](https://github.com/Boerderij/Varken/pull/45) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) + ## [v0.3-nightly](https://github.com/Boerderij/Varken/tree/v0.3-nightly) (2018-12-07) [Full Changelog](https://github.com/Boerderij/Varken/compare/v0.2-nightly...v0.3-nightly) From 78ce399f33571e8813a03b677907c64079f2e49d Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Sun, 9 Dec 2018 22:56:09 -0600 Subject: [PATCH 02/76] passed data folder to tautulli for helper functions. Fixes #46 --- Varken.py | 2 +- varken/cisco.py | 1 - varken/helpers.py | 17 +++++++++-------- varken/tautulli.py | 9 +++++---- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Varken.py b/Varken.py index 6fe4ec3..bd6c1b8 100644 --- a/Varken.py +++ b/Varken.py @@ -79,7 +79,7 @@ if __name__ == "__main__": if CONFIG.tautulli_enabled: for server in CONFIG.tautulli_servers: - TAUTULLI = TautulliAPI(server, DBMANAGER) + TAUTULLI = TautulliAPI(server, DBMANAGER, DATA_FOLDER) if server.get_activity: schedule.every(server.get_activity_run_seconds).seconds.do(threaded, TAUTULLI.get_activity) diff --git a/varken/cisco.py b/varken/cisco.py index 6ce3392..750ad89 100644 --- a/varken/cisco.py +++ b/varken/cisco.py @@ -39,7 +39,6 @@ class CiscoAPI(object): return req = self.session.prepare_request(Request('GET', self.firewall.url + endpoint)) - print(req.headers) get = connection_handler(self.session, req, self.firewall.verify_ssl) if not get: diff --git a/varken/helpers.py b/varken/helpers.py index 25f99d9..b746085 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -14,21 +14,22 @@ from urllib.request import urlretrieve logger = logging.getLogger('varken') -def geoip_download(): - tar_dbfile = abspath(join('.', 'data', 'GeoLite2-City.tar.gz')) +def geoip_download(data_folder): + datafolder = data_folder + tar_dbfile = abspath(join(datafolder, 'GeoLite2-City.tar.gz')) url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz' urlretrieve(url, tar_dbfile) tar = tarfile.open(tar_dbfile, 'r:gz') for files in tar.getmembers(): if 'GeoLite2-City.mmdb' in files.name: files.name = os.path.basename(files.name) - tar.extract(files, abspath(join('.', 'data'))) + tar.extract(files, datafolder) os.remove(tar_dbfile) -def geo_lookup(ipaddress): - - dbfile = abspath(join('.', 'data', 'GeoLite2-City.mmdb')) +def geo_lookup(ipaddress, data_folder): + datafolder = data_folder + dbfile = abspath(join(datafolder, 'GeoLite2-City.mmdb')) now = time.time() try: @@ -36,9 +37,9 @@ def geo_lookup(ipaddress): db_age = now - dbinfo.st_ctime if db_age > (35 * 86400): os.remove(dbfile) - geoip_download() + geoip_download(datafolder) except FileNotFoundError: - geoip_download() + geoip_download(datafolder) reader = geoip2.database.Reader(dbfile) diff --git a/varken/tautulli.py b/varken/tautulli.py index 5ce6773..9112f83 100644 --- a/varken/tautulli.py +++ b/varken/tautulli.py @@ -8,7 +8,7 @@ from varken.structures import TautulliStream class TautulliAPI(object): - def __init__(self, server, dbmanager): + def __init__(self, server, dbmanager, data_folder): # Set Time of initialization self.now = datetime.now(timezone.utc).astimezone().isoformat() self.dbmanager = dbmanager @@ -17,6 +17,7 @@ class TautulliAPI(object): self.session.params = {'apikey': self.server.api_key, 'cmd': 'get_activity'} self.endpoint = '/api/v2' self.logger = logging.getLogger() + self.data_folder = data_folder def __repr__(self): return "".format(self.server.id) @@ -41,13 +42,13 @@ class TautulliAPI(object): for session in sessions: try: - geodata = geo_lookup(session.ip_address_public) + geodata = geo_lookup(session.ip_address_public, self.data_folder) except (ValueError, AddressNotFoundError): if self.server.fallback_ip: - geodata = geo_lookup(self.server.fallback_ip) + geodata = geo_lookup(self.server.fallback_ip, self.data_folder) else: my_ip = self.session.get('http://ip.42.pl/raw').text - geodata = geo_lookup(my_ip) + geodata = geo_lookup(my_ip, self.data_folder) if not all([geodata.location.latitude, geodata.location.longitude]): latitude = 37.234332396 From a2b01b2b1ca03c776a99308d26db7ee25a4868c2 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 10 Dec 2018 01:16:01 -0600 Subject: [PATCH 03/76] changed Missing available to 0/1 for grafana colorization --- varken/radarr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/varken/radarr.py b/varken/radarr.py index db4dd2e..e04ea49 100644 --- a/varken/radarr.py +++ b/varken/radarr.py @@ -40,9 +40,9 @@ class RadarrAPI(object): for movie in movies: if not movie.downloaded: if movie.isAvailable: - ma = True + ma = 0 else: - ma = False + ma = 1 movie_name = '{} ({})'.format(movie.title, movie.year) missing.append((movie_name, ma, movie.tmdbId)) From 95c176167f0d9031c5199e1b01eec61b2aa48933 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 10 Dec 2018 12:21:02 -0800 Subject: [PATCH 04/76] Handle invalid config better and log it --- varken/iniparser.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/varken/iniparser.py b/varken/iniparser.py index 30629a0..d163423 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -29,7 +29,10 @@ class INIParser(object): self.ciscoasa_enabled = False self.ciscoasa_firewalls = [] - self.parse_opts() + try: + self.parse_opts() + except configparser.NoOptionError as e: + logger.error(e) def enable_check(self, server_type=None): t = server_type From 2d3b094eca042403891b4d57211bde0c9f174e89 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 10 Dec 2018 12:40:20 -0800 Subject: [PATCH 05/76] Handle invalid config better and log it --- varken/iniparser.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/varken/iniparser.py b/varken/iniparser.py index 30629a0..d163423 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -29,7 +29,10 @@ class INIParser(object): self.ciscoasa_enabled = False self.ciscoasa_firewalls = [] - self.parse_opts() + try: + self.parse_opts() + except configparser.NoOptionError as e: + logger.error(e) def enable_check(self, server_type=None): t = server_type From 63aad2b320ccfe6ba3b1aeffa36a0e689c05b077 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 10 Dec 2018 15:06:43 -0800 Subject: [PATCH 06/76] Added titleSlug to radarrEnhancement for #50 --- varken/radarr.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/varken/radarr.py b/varken/radarr.py index e04ea49..d4652c6 100644 --- a/varken/radarr.py +++ b/varken/radarr.py @@ -43,10 +43,15 @@ class RadarrAPI(object): ma = 0 else: ma = 1 - movie_name = '{} ({})'.format(movie.title, movie.year) - missing.append((movie_name, ma, movie.tmdbId)) - for title, ma, mid in missing: + movie_name = '{} ({})'.format(movie.title, movie.year) + + title_slug = movie.titleSlug + + missing.append((movie_name, ma, movie.tmdbId, title_slug)) + + + for title, ma, mid, title_slug in missing: hash_id = hashit('{}{}{}'.format(self.server.id, title, mid)) influx_payload.append( { @@ -56,7 +61,8 @@ class RadarrAPI(object): "Missing_Available": ma, "tmdbId": mid, "server": self.server.id, - "name": title + "name": title, + "titleSlug": title_slug }, "time": self.now, "fields": { @@ -94,17 +100,20 @@ class RadarrAPI(object): for queue_item in download_queue: movie = queue_item.movie + name = '{} ({})'.format(movie.title, movie.year) + title_slug = movie.titleSlug + if queue_item.protocol.upper() == 'USENET': protocol_id = 1 else: protocol_id = 0 queue.append((name, queue_item.quality['quality']['name'], queue_item.protocol.upper(), - protocol_id, queue_item.id)) + protocol_id, queue_item.id, title_slug)) - for name, quality, protocol, protocol_id, qid in queue: + for name, quality, protocol, protocol_id, qid, title_slug in queue: hash_id = hashit('{}{}{}'.format(self.server.id, name, quality)) influx_payload.append( { @@ -116,7 +125,8 @@ class RadarrAPI(object): "name": name, "quality": quality, "protocol": protocol, - "protocol_id": protocol_id + "protocol_id": protocol_id, + "titleSlug": title_slug }, "time": self.now, "fields": { From 09c6a1d029799128c312f41ecf41c62293e5162d Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 10 Dec 2018 16:49:22 -0800 Subject: [PATCH 07/76] Handle exceptions in a differnt place --- varken/iniparser.py | 22 +++++++++++----------- varken/radarr.py | 8 ++------ 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/varken/iniparser.py b/varken/iniparser.py index d163423..b242329 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -29,20 +29,20 @@ class INIParser(object): self.ciscoasa_enabled = False self.ciscoasa_firewalls = [] - try: - self.parse_opts() - except configparser.NoOptionError as e: - logger.error(e) + self.parse_opts() def enable_check(self, server_type=None): t = server_type - global_server_ids = self.config.get('global', t) - if global_server_ids.lower() in ['false', 'no', '0']: - logger.info('%s disabled.', t.upper()) - return False - else: - sids = self.clean_check(global_server_ids, t) - return sids + try: + global_server_ids = self.config.get('global', t) + if global_server_ids.lower() in ['false', 'no', '0']: + logger.info('%s disabled.', t.upper()) + return False + else: + sids = self.clean_check(global_server_ids, t) + return sids + except configparser.NoOptionError as e: + logger.error(e) @staticmethod def clean_check(server_id_list, server_type=None): diff --git a/varken/radarr.py b/varken/radarr.py index d4652c6..e19623c 100644 --- a/varken/radarr.py +++ b/varken/radarr.py @@ -46,9 +46,7 @@ class RadarrAPI(object): movie_name = '{} ({})'.format(movie.title, movie.year) - title_slug = movie.titleSlug - - missing.append((movie_name, ma, movie.tmdbId, title_slug)) + missing.append((movie_name, ma, movie.tmdbId, movie.titleSlug)) for title, ma, mid, title_slug in missing: @@ -103,15 +101,13 @@ class RadarrAPI(object): name = '{} ({})'.format(movie.title, movie.year) - title_slug = movie.titleSlug - if queue_item.protocol.upper() == 'USENET': protocol_id = 1 else: protocol_id = 0 queue.append((name, queue_item.quality['quality']['name'], queue_item.protocol.upper(), - protocol_id, queue_item.id, title_slug)) + protocol_id, queue_item.id, movie.titleSlug)) for name, quality, protocol, protocol_id, qid, title_slug in queue: hash_id = hashit('{}{}{}'.format(self.server.id, name, quality)) From 164f91976963dedd6a0b4f0a4f6754a7838c1930 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 10 Dec 2018 16:55:47 -0800 Subject: [PATCH 08/76] Add extra_type to Tautulli structures --- varken/structures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/varken/structures.py b/varken/structures.py index 44c202a..34fa91c 100644 --- a/varken/structures.py +++ b/varken/structures.py @@ -274,6 +274,7 @@ class TautulliStream(NamedTuple): subtitle_language: str = None stream_subtitle_container: str = None sub_type: str = None + extra_type: str = None class TVShow(NamedTuple): From 18285f191497bf155cf3d787058d4ded35db42a4 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 10 Dec 2018 17:27:50 -0800 Subject: [PATCH 09/76] Convert missing available to True False --- varken/radarr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/varken/radarr.py b/varken/radarr.py index e19623c..11d0e14 100644 --- a/varken/radarr.py +++ b/varken/radarr.py @@ -40,9 +40,9 @@ class RadarrAPI(object): for movie in movies: if not movie.downloaded: if movie.isAvailable: - ma = 0 + ma = True else: - ma = 1 + ma = False movie_name = '{} ({})'.format(movie.title, movie.year) From 4c1c60c1523f2b995ee1cf02b42fc1aebc08b8c0 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 10 Dec 2018 19:55:45 -0600 Subject: [PATCH 10/76] Revert "Convert missing available to True False" This reverts commit 273706d1 --- varken/radarr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/varken/radarr.py b/varken/radarr.py index 11d0e14..e19623c 100644 --- a/varken/radarr.py +++ b/varken/radarr.py @@ -40,9 +40,9 @@ class RadarrAPI(object): for movie in movies: if not movie.downloaded: if movie.isAvailable: - ma = True + ma = 0 else: - ma = False + ma = 1 movie_name = '{} ({})'.format(movie.title, movie.year) From 5d3ec2db11e5babc8b11254ea2878a18a1bb07ad Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 10 Dec 2018 17:59:03 -0800 Subject: [PATCH 11/76] Add logs to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f97a8aa..cc45351 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ data/varken.ini .idea/ Legacy/configuration.py varken-venv/ +logs/ From 30984a1e79c140518e226fbba5117a59ec88ecca Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 10 Dec 2018 20:27:54 -0600 Subject: [PATCH 12/76] Travis-CI Dockerbuild test --- .travis.yml | 12 ++++++++++++ Dockerfile | 16 ++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 .travis.yml create mode 100644 Dockerfile diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..6e28370 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,12 @@ +services: + - docker + +script: + - docker build -t boerderij/varken:$TRAVIS_BRANCH . + - docker ps -a + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - docker push boerderij/varken:$TRAVIS_BRANCH + +env: + global: + - secure: c91+zQXN28Anr2q94vhJAzN36teKovgniNmUI0MaejYfwcShN8TVeojkxbP+AxZuvGh1AfvB26LQeuPqAAntoNxvtPg3PLxv93rRSkAf0jk9apm9biYDAJNAM3OSiqCGfzfNhtUHmPmybRy2UmRXpHc6ZU1GmOdX2yyXCC2S6wjJOGabRpCA2Lw1vNnQuSJMDZ78amybZNmqAkK+rxe9hH2TGwcSImW8dlW2Ryt8H4a2s9VW9rbebQF+PzY4pw+OlIarpVUXZzUyEq8PS2EmJTuhrNA+RtZWJ4yRZ33jK4UqZRJzfC4FniZzSqtV/P3YGgSFNzhM87y5VhNiauX6QmtIDfLUV6c86cWCy24O41SrAJQOi4CLszJVkYfyggVFoRFegNS2+njN+f2Bbbx3rHtmNds0cDSfFuK3XhtTe0EhNgHLXOCX4IyAGzYWO+afmbqm/8S+m/QjCT28+0GgxYSqD2qO3FuPRA7woWucrKl2xa/tYikkurkDif0yBHxPac8mB8KLPLrjGzHlBG6SYYpTlpjWJrddbYhm0EZVmMkkFHRHLcOK8AOHKQipQBHmP+wvTGouwaZ8Uv5+uDNZ76st4BZR1tfXCtZ6A0RLRspo0wJ5EKlrNr8OIQGdj1G4TJ0H029mycqkAQ5yFPlvF/wAZ0shVFb/uMejpQw+2ks= \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..023a197 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM lsiobase/alpine.python3 + +LABEL maintainer="dirtycajunrice" + +ENV branch="master" + +RUN \ + git clone --single-branch -b $branch https://github.com/Boerderij/Varken.git /app && \ + python3 -m pip install -r /app/requirements.txt && \ + chown -R abc:abc \ + /config \ + /app + +CMD cp /app/data/varken.example.ini /config/varken.example.ini && python3 /app/Varken.py --data-folder /config + +VOLUME /config \ No newline at end of file From 5d284a525925c8e4079fa8c22c2d89bb4e718465 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 10 Dec 2018 19:10:59 -0800 Subject: [PATCH 13/76] Added logging to the GeoLite2 downloader --- varken/helpers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/varken/helpers.py b/varken/helpers.py index b746085..f3c02e4 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -16,14 +16,24 @@ logger = logging.getLogger('varken') def geoip_download(data_folder): datafolder = data_folder + tar_dbfile = abspath(join(datafolder, 'GeoLite2-City.tar.gz')) + url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz' + logger.info('Downloading GeoLite2 from %s', url) urlretrieve(url, tar_dbfile) + tar = tarfile.open(tar_dbfile, 'r:gz') + logging.debug('Opening GeoLite2 tar file : %s', tar_dbfile) + for files in tar.getmembers(): if 'GeoLite2-City.mmdb' in files.name: + logging.debug('"GeoLite2-City.mmdb" FOUND in tar file') files.name = os.path.basename(files.name) + tar.extract(files, datafolder) + logging.debug('%s has been extracted to %s', files, datafolder) + os.remove(tar_dbfile) From 0e844d5a156f4d56a863fcefc123d2a74f79ba12 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 10 Dec 2018 19:16:13 -0800 Subject: [PATCH 14/76] Add logging to geo_lookup. --- varken/helpers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/varken/helpers.py b/varken/helpers.py index f3c02e4..0fb4d6a 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -33,12 +33,14 @@ def geoip_download(data_folder): tar.extract(files, datafolder) logging.debug('%s has been extracted to %s', files, datafolder) - + os.remove(tar_dbfile) def geo_lookup(ipaddress, data_folder): datafolder = data_folder + logging.debug('Reading GeoLite2 from %s', datafolder) + dbfile = abspath(join(datafolder, 'GeoLite2-City.mmdb')) now = time.time() @@ -46,9 +48,13 @@ def geo_lookup(ipaddress, data_folder): dbinfo = os.stat(dbfile) db_age = now - dbinfo.st_ctime if db_age > (35 * 86400): + logging.info('GeoLite2 DB is older than 35 days. Attempting to re-download...') + os.remove(dbfile) + geoip_download(datafolder) except FileNotFoundError: + logging.error('GeoLite2 DB not found. Attempting to download...') geoip_download(datafolder) reader = geoip2.database.Reader(dbfile) From 074d1c0a595ba1aaf54b97f6ebbea4fff18e63a2 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 10 Dec 2018 19:35:36 -0800 Subject: [PATCH 15/76] Log data folder on start --- Varken.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Varken.py b/Varken.py index bd6c1b8..8d6e5eb 100644 --- a/Varken.py +++ b/Varken.py @@ -58,6 +58,8 @@ if __name__ == "__main__": vl = VarkenLogger(data_folder=DATA_FOLDER, debug=opts.debug) vl.logger.info('Starting Varken...') + vl.logger.info('Data folder is "%s"', DATA_FOLDER) + vl.logger.info(u"{} {} ({}{})".format( platform.system(), platform.release(), platform.version(), ' - {}'.format(PLATFORM_LINUX_DISTRO) if PLATFORM_LINUX_DISTRO else '' From 46299bef765e04be3adafa704a6f8f7ee088a928 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 10 Dec 2018 21:54:01 -0600 Subject: [PATCH 16/76] updated readme + travis. Also addresses #53 --- .travis.yml | 2 +- README.md | 74 ++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 60 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6e28370..f746698 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ services: script: - docker build -t boerderij/varken:$TRAVIS_BRANCH . - - docker ps -a + - docker images - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - docker push boerderij/varken:$TRAVIS_BRANCH diff --git a/README.md b/README.md index eb16bc3..838724d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Varken +[![Build Status](https://travis-ci.org/Boerderij/Varken.svg?branch=master)](https://travis-ci.org/Boerderij/Varken) [![Discord](https://img.shields.io/badge/Discord-Varken-7289DA.svg?logo=discord&style=flat-square)](https://discord.gg/AGTG44H) [![BuyMeACoffee](https://img.shields.io/badge/BuyMeACoffee-Donate-ff813f.svg?logo=CoffeeScript&style=flat-square)](https://www.buymeacoffee.com/varken) -[![Docker Pulls](https://img.shields.io/docker/pulls/boerderij/varken.svg?style=flat-square)](https://hub.docker.com/r/boerderij/varken/) Dutch for PIG. PIG is an Acronym for Plex/InfluxDB/Grafana @@ -12,28 +12,47 @@ frontend Requirements: * Python3.6+ * Python3-pip +* InfluxDB

-## Quick Setup -1. Clone the repository `sudo git clone https://github.com/Boerderij/Varken.git /opt/Varken` -1. Follow the systemd install instructions located in `varken.systemd` -1. Create venv in project `cd /opt/Varken && /usr/bin/python3 -m venv varken-venv` -1. Install requirements `/opt/Varken/varken-venv/bin/python -m pip install -r requirements.txt` -1. Make a copy of `varken.example.ini` to `varken.ini` in the `data` folder - `cp /opt/Varken/data/varken.example.ini /opt/Varken/data/varken.ini` -1. Make the appropriate changes to `varken.ini` - ie.`nano /opt/Varken/data/varken.ini` -1. Make sure all the files have the appropriate permissions `sudo chown varken:varken -R /opt/Varken` -1. After completing the [getting started](http://docs.grafana.org/guides/getting_started/) portion of grafana, create your datasource for influxdb. -1. Install `grafana-cli plugins install grafana-worldmap-panel` +## Quick Setup (Git Clone) +``` +# Clone the repository +git clone https://github.com/Boerderij/Varken.git /opt/Varken +# Follow the systemd install instructions located in varken.systemd +nano /opt/Varken/varken.systemd +cp /opt/Varken/varken.systemd /etc/systemd/system/varken.service + +# Create venv in project +/usr/bin/python3 -m venv /opt/Varken/varken-venv + +# Install requirements +/opt/Varken/varken-venv/bin/python -m pip install -r requirements.txt + +# Make a copy of varken.example.ini to varken.ini in the data folder +cp /opt/Varken/data/varken.example.ini /opt/Varken/data/varken.ini + +# Make the appropriate changes to varken.ini +nano /opt/Varken/data/varken.ini + +# Make sure all the files have the appropriate permissions +chown $USER:$USER -R /opt/Varken + +# Start the service and enable it +systemctl start varken +systemctl enable varken +``` ### Docker Repo is included in [Boerderij/docker-Varken](https://github.com/Boerderij/docker-Varken) - +[![Docker-Layers](https://images.microbadger.com/badges/image/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken") +[![Docker-Version](https://images.microbadger.com/badges/version/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken") +[![Docker Pulls](https://img.shields.io/docker/pulls/boerderij/varken.svg)](https://hub.docker.com/r/boerderij/varken/) +[![Docker Stars](https://img.shields.io/docker/stars/boerderij/varken.svg)](https://hub.docker.com/r/boerderij/varken/)
Example

@@ -42,7 +61,32 @@ docker run -d \ --name=varken \ -v :/config \ -e PGID= -e PUID= \ - boerderij/varken:nightly + boerderij/varken ```

+ +#### Tags +* **latest** +* **nightly** +* **release-tag** e.g. v1.0 + +#### Upgrading with docker +```sh +docker stop varken +docker rm varken +# Run deploy command above +``` + +### InfluxDB +[InfluxDB Installation documentation](https://docs.influxdata.com/influxdb/v1.7/introduction/installation/) +Influxdb is required but not packaged as part of Varken. Varken will create +its database on its own. If you choose to give varken user permissions that +do not include database creation, please ensure you create an influx database +named `varken` + +### Grafana +Grafana is used in our examples but not required, nor packaged as part of +Varken. Panel example pictures are pinned in the grafana-panels channel of +discord. Future releases may contain a json-generator, but it does not exist +as varken stands today. \ No newline at end of file From f1ba96d04fbe042a455fd988d2fdd1b4bc0024cb Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 10 Dec 2018 21:56:07 -0600 Subject: [PATCH 17/76] sigh... spaces and such --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 838724d..2628902 100644 --- a/README.md +++ b/README.md @@ -79,13 +79,14 @@ docker rm varken ``` ### InfluxDB -[InfluxDB Installation documentation](https://docs.influxdata.com/influxdb/v1.7/introduction/installation/) +[InfluxDB Installation Documentation](https://docs.influxdata.com/influxdb/v1.7/introduction/installation/) Influxdb is required but not packaged as part of Varken. Varken will create its database on its own. If you choose to give varken user permissions that do not include database creation, please ensure you create an influx database named `varken` ### Grafana +[Grafana Installation Documentation(http://docs.grafana.org/installation/) Grafana is used in our examples but not required, nor packaged as part of Varken. Panel example pictures are pinned in the grafana-panels channel of discord. Future releases may contain a json-generator, but it does not exist From 381656812ed7b3a58141bac5acdef8ba13360d84 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 10 Dec 2018 21:59:10 -0600 Subject: [PATCH 18/76] sigh... spaces and such --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 2628902..e01046b 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ systemctl enable varken ### Docker Repo is included in [Boerderij/docker-Varken](https://github.com/Boerderij/docker-Varken) + [![Docker-Layers](https://images.microbadger.com/badges/image/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken") [![Docker-Version](https://images.microbadger.com/badges/version/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken") [![Docker Pulls](https://img.shields.io/docker/pulls/boerderij/varken.svg)](https://hub.docker.com/r/boerderij/varken/) @@ -80,6 +81,7 @@ docker rm varken ### InfluxDB [InfluxDB Installation Documentation](https://docs.influxdata.com/influxdb/v1.7/introduction/installation/) + Influxdb is required but not packaged as part of Varken. Varken will create its database on its own. If you choose to give varken user permissions that do not include database creation, please ensure you create an influx database @@ -87,6 +89,7 @@ named `varken` ### Grafana [Grafana Installation Documentation(http://docs.grafana.org/installation/) + Grafana is used in our examples but not required, nor packaged as part of Varken. Panel example pictures are pinned in the grafana-panels channel of discord. Future releases may contain a json-generator, but it does not exist From a2f34ca349bc6e9c9d2c4bc4c06ab5e31e86d7da Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 10 Dec 2018 22:03:04 -0600 Subject: [PATCH 19/76] sigh... spaces and such --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index e01046b..b0ceab1 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,6 @@ systemctl enable varken ``` ### Docker -Repo is included in [Boerderij/docker-Varken](https://github.com/Boerderij/docker-Varken) - [![Docker-Layers](https://images.microbadger.com/badges/image/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken") [![Docker-Version](https://images.microbadger.com/badges/version/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken") [![Docker Pulls](https://img.shields.io/docker/pulls/boerderij/varken.svg)](https://hub.docker.com/r/boerderij/varken/) @@ -88,7 +86,7 @@ do not include database creation, please ensure you create an influx database named `varken` ### Grafana -[Grafana Installation Documentation(http://docs.grafana.org/installation/) +[Grafana Installation Documentation](http://docs.grafana.org/installation/) Grafana is used in our examples but not required, nor packaged as part of Varken. Panel example pictures are pinned in the grafana-panels channel of From e5713887f8100c8e6231210b97eb7a1a92a00eef Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 10 Dec 2018 22:40:23 -0600 Subject: [PATCH 20/76] split out stages for travis --- .travis.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index f746698..89e7b26 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,17 @@ services: - docker -script: - - docker build -t boerderij/varken:$TRAVIS_BRANCH . - - docker images - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker push boerderij/varken:$TRAVIS_BRANCH +jobs: + include: + - stage: "Build and Push" + name: "Build" + script: + - docker build -t boerderij/varken:$TRAVIS_BRANCH . + - docker images + - name: "Push" + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - docker push boerderij/varken:$TRAVIS_BRANCH env: global: From e9febb96d61003e21e35ceab965d32aa57acd9a8 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 10 Dec 2018 22:43:41 -0600 Subject: [PATCH 21/76] split out stages for travis --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 89e7b26..f4ed0fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,9 @@ services: jobs: include: - stage: "Build and Push" - name: "Build" script: - docker build -t boerderij/varken:$TRAVIS_BRANCH . - docker images - - name: "Push" - script: - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - docker push boerderij/varken:$TRAVIS_BRANCH From a25a9b23b406e44a1376c7c91f798f3bd4aa27ea Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 10 Dec 2018 22:50:28 -0600 Subject: [PATCH 22/76] added licence --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..59cd926 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Boerderij + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 4be1b8d5b18a9fcfb3948e6c3a92263776c09258 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 10 Dec 2018 22:56:37 -0600 Subject: [PATCH 23/76] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 31 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 +++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..0364d15 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,31 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG]" +labels: awaiting-approval +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. ... +2. ... +3. ... +4. ... + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Environment (please complete the following information):** + - OS: [e.g. Ubuntu 18.04.1 or Docker:Tag] + - Version [e.g. v1.1] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..54cdb66 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[Feature Request]" +labels: awaiting-approval +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. From b0022523e07a464f2619ad5843f5e30ee3900af6 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 10 Dec 2018 23:03:21 -0600 Subject: [PATCH 24/76] test travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f4ed0fe..4653958 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ jobs: include: - stage: "Build and Push" script: - - docker build -t boerderij/varken:$TRAVIS_BRANCH . + - docker build --build-arg branch=$TRAVIS_BRANCH -t boerderij/varken:$TRAVIS_BRANCH . - docker images - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - docker push boerderij/varken:$TRAVIS_BRANCH From f74b7a1830149d315b1eca87118ffab667960317 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 10 Dec 2018 23:11:54 -0600 Subject: [PATCH 25/76] test2 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 023a197..16ccdab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM lsiobase/alpine.python3 LABEL maintainer="dirtycajunrice" -ENV branch="master" +#ENV branch="master" RUN \ git clone --single-branch -b $branch https://github.com/Boerderij/Varken.git /app && \ From 51d488517541c42d9e2825ce5bce9908ad5c631e Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 10 Dec 2018 23:16:30 -0600 Subject: [PATCH 26/76] test3? --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 16ccdab..facaef6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,9 @@ FROM lsiobase/alpine.python3 LABEL maintainer="dirtycajunrice" -#ENV branch="master" +ARG branch="master" + +ENV branch=$branch RUN \ git clone --single-branch -b $branch https://github.com/Boerderij/Varken.git /app && \ From 2ec2eda9712f7ef2e609430500d700d069674588 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 11 Dec 2018 00:01:24 -0600 Subject: [PATCH 27/76] made time local to the function. Fixes #56 --- varken/ombi.py | 9 ++++----- varken/radarr.py | 11 ++++------- varken/sonarr.py | 21 ++++++++++----------- varken/tautulli.py | 8 +++----- 4 files changed, 21 insertions(+), 28 deletions(-) diff --git a/varken/ombi.py b/varken/ombi.py index f38ca49..7ea33fc 100644 --- a/varken/ombi.py +++ b/varken/ombi.py @@ -8,7 +8,6 @@ from varken.structures import OmbiRequestCounts class OmbiAPI(object): def __init__(self, server, dbmanager): - self.now = datetime.now(timezone.utc).astimezone().isoformat() self.dbmanager = dbmanager self.server = server # Create session to reduce server web thread load, and globally define pageSize for all requests @@ -20,7 +19,7 @@ class OmbiAPI(object): return "".format(self.server.id) def get_total_requests(self): - self.now = datetime.now(timezone.utc).astimezone().isoformat() + now = datetime.now(timezone.utc).astimezone().isoformat() tv_endpoint = '/api/v1/Request/tv' movie_endpoint = "/api/v1/Request/movie" @@ -42,7 +41,7 @@ class OmbiAPI(object): "type": "Request_Total", "server": self.server.id }, - "time": self.now, + "time": now, "fields": { "total": movie_requests + tv_requests, "movies": movie_requests, @@ -54,7 +53,7 @@ class OmbiAPI(object): self.dbmanager.write_points(influx_payload) def get_request_counts(self): - self.now = datetime.now(timezone.utc).astimezone().isoformat() + now = datetime.now(timezone.utc).astimezone().isoformat() endpoint = '/api/v1/Request/count' req = self.session.prepare_request(Request('GET', self.server.url + endpoint)) @@ -70,7 +69,7 @@ class OmbiAPI(object): "tags": { "type": "Request_Counts" }, - "time": self.now, + "time": now, "fields": { "pending": requests.pending, "approved": requests.approved, diff --git a/varken/radarr.py b/varken/radarr.py index e19623c..1b23923 100644 --- a/varken/radarr.py +++ b/varken/radarr.py @@ -8,7 +8,6 @@ from varken.structures import Movie, Queue class RadarrAPI(object): def __init__(self, server, dbmanager): - self.now = datetime.now(timezone.utc).astimezone().isoformat() self.dbmanager = dbmanager self.server = server # Create session to reduce server web thread load, and globally define pageSize for all requests @@ -21,7 +20,7 @@ class RadarrAPI(object): def get_missing(self): endpoint = '/api/movie' - self.now = datetime.now(timezone.utc).astimezone().isoformat() + now = datetime.now(timezone.utc).astimezone().isoformat() influx_payload = [] missing = [] @@ -45,10 +44,8 @@ class RadarrAPI(object): ma = 1 movie_name = '{} ({})'.format(movie.title, movie.year) - missing.append((movie_name, ma, movie.tmdbId, movie.titleSlug)) - for title, ma, mid, title_slug in missing: hash_id = hashit('{}{}{}'.format(self.server.id, title, mid)) influx_payload.append( @@ -62,7 +59,7 @@ class RadarrAPI(object): "name": title, "titleSlug": title_slug }, - "time": self.now, + "time": now, "fields": { "hash": hash_id } @@ -73,7 +70,7 @@ class RadarrAPI(object): def get_queue(self): endpoint = '/api/queue' - self.now = datetime.now(timezone.utc).astimezone().isoformat() + now = datetime.now(timezone.utc).astimezone().isoformat() influx_payload = [] queue = [] @@ -124,7 +121,7 @@ class RadarrAPI(object): "protocol_id": protocol_id, "titleSlug": title_slug }, - "time": self.now, + "time": now, "fields": { "hash": hash_id } diff --git a/varken/sonarr.py b/varken/sonarr.py index 8f71817..117136a 100644 --- a/varken/sonarr.py +++ b/varken/sonarr.py @@ -8,10 +8,7 @@ from varken.structures import Queue, TVShow class SonarrAPI(object): def __init__(self, server, dbmanager): - # Set Time of initialization - self.now = datetime.now(timezone.utc).astimezone().isoformat() self.dbmanager = dbmanager - self.today = str(date.today()) self.server = server # Create session to reduce server web thread load, and globally define pageSize for all requests self.session = Session() @@ -24,9 +21,10 @@ class SonarrAPI(object): def get_missing(self): endpoint = '/api/calendar' + today = str(date.today()) last_days = str(date.today() + timedelta(days=-self.server.missing_days)) - self.now = datetime.now(timezone.utc).astimezone().isoformat() - params = {'start': last_days, 'end': self.today} + now = datetime.now(timezone.utc).astimezone().isoformat() + params = {'start': last_days, 'end': today} influx_payload = [] missing = [] @@ -63,7 +61,7 @@ class SonarrAPI(object): "sxe": sxe, "airs": air_date }, - "time": self.now, + "time": now, "fields": { "hash": hash_id @@ -75,11 +73,12 @@ class SonarrAPI(object): def get_future(self): endpoint = '/api/calendar/' - self.now = datetime.now(timezone.utc).astimezone().isoformat() + today = str(date.today()) + now = datetime.now(timezone.utc).astimezone().isoformat() future = str(date.today() + timedelta(days=self.server.future_days)) influx_payload = [] air_days = [] - params = {'start': self.today, 'end': future} + params = {'start': today, 'end': future} req = self.session.prepare_request(Request('GET', self.server.url + endpoint, params=params)) get = connection_handler(self.session, req, self.server.verify_ssl) @@ -116,7 +115,7 @@ class SonarrAPI(object): "airs": air_date, "downloaded": dl_status }, - "time": self.now, + "time": now, "fields": { "hash": hash_id } @@ -128,7 +127,7 @@ class SonarrAPI(object): def get_queue(self): influx_payload = [] endpoint = '/api/queue' - self.now = datetime.now(timezone.utc).astimezone().isoformat() + now = datetime.now(timezone.utc).astimezone().isoformat() queue = [] req = self.session.prepare_request(Request('GET', self.server.url + endpoint)) @@ -168,7 +167,7 @@ class SonarrAPI(object): "protocol": protocol, "protocol_id": protocol_id }, - "time": self.now, + "time": now, "fields": { "hash": hash_id } diff --git a/varken/tautulli.py b/varken/tautulli.py index 9112f83..a62da8f 100644 --- a/varken/tautulli.py +++ b/varken/tautulli.py @@ -9,8 +9,6 @@ from varken.structures import TautulliStream class TautulliAPI(object): def __init__(self, server, dbmanager, data_folder): - # Set Time of initialization - self.now = datetime.now(timezone.utc).astimezone().isoformat() self.dbmanager = dbmanager self.server = server self.session = Session() @@ -23,7 +21,7 @@ class TautulliAPI(object): return "".format(self.server.id) def get_activity(self): - self.now = datetime.now(timezone.utc).astimezone().isoformat() + now = datetime.now(timezone.utc).astimezone().isoformat() influx_payload = [] req = self.session.prepare_request(Request('GET', self.server.url + self.endpoint)) @@ -119,7 +117,7 @@ class TautulliAPI(object): "device_type": session.platform, "server": self.server.id }, - "time": self.now, + "time": now, "fields": { "hash": hash_id } @@ -133,7 +131,7 @@ class TautulliAPI(object): "type": "current_stream_stats", "server": self.server.id }, - "time": self.now, + "time": now, "fields": { "stream_count": int(get['stream_count']), "total_bandwidth": int(get['total_bandwidth']), From b8fdaad7f2c879a97bd55fda84b641bd87fa6cb3 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 11 Dec 2018 00:05:37 -0600 Subject: [PATCH 28/76] test for cibuild --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index facaef6..6fab470 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,10 @@ ARG branch="master" ENV branch=$branch +#COPY / /app/Varken + RUN \ + echo $PWD && ls && \ git clone --single-branch -b $branch https://github.com/Boerderij/Varken.git /app && \ python3 -m pip install -r /app/requirements.txt && \ chown -R abc:abc \ From b33755204d9be566daaea7f2a5ce01c1f3b28121 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 11 Dec 2018 00:13:17 -0600 Subject: [PATCH 29/76] test4 --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6fab470..259cb60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,11 +6,9 @@ ARG branch="master" ENV branch=$branch -#COPY / /app/Varken +COPY / /app/Varken RUN \ - echo $PWD && ls && \ - git clone --single-branch -b $branch https://github.com/Boerderij/Varken.git /app && \ python3 -m pip install -r /app/requirements.txt && \ chown -R abc:abc \ /config \ From 250949e3d1ba214a42c88b038bf3190fd7cdd488 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 11 Dec 2018 00:15:38 -0600 Subject: [PATCH 30/76] test5 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 259cb60..c0d4517 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ARG branch="master" ENV branch=$branch -COPY / /app/Varken +COPY / /app RUN \ python3 -m pip install -r /app/requirements.txt && \ From 5cb1fe71617120d472179aaeeeb6b16f58547be2 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 11 Dec 2018 00:25:52 -0600 Subject: [PATCH 31/76] Changelog + Version bump --- CHANGELOG.md | 27 ++++++++++++++++++++++++++- varken/__init__.py | 2 +- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 715bb76..61c3fe2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,31 @@ # Change Log -## [v1.0](https://github.com/Boerderij/Varken/tree/v1.0) (2018-12-09) +## [v1.1](https://github.com/Boerderij/Varken/tree/v1.1) (2018-12-11) +[Full Changelog](https://github.com/Boerderij/Varken/compare/v1.0...v1.1) + +**Implemented enhancements:** + +- Convert missing available to True False [\#54](https://github.com/Boerderij/Varken/issues/54) +- Handle invalid config better and log it [\#51](https://github.com/Boerderij/Varken/issues/51) +- Feature Request - Include value from Radarr [\#50](https://github.com/Boerderij/Varken/issues/50) +- Change true/false to 0/1 for missing movies [\#47](https://github.com/Boerderij/Varken/issues/47) + +**Fixed bugs:** + +- \[BUG\] Time does not update from "today" [\#56](https://github.com/Boerderij/Varken/issues/56) +- geoip\_download does not account for moving data folder [\#46](https://github.com/Boerderij/Varken/issues/46) + +**Closed issues:** + +- Initial startup requires admin access to InfluxDB [\#53](https://github.com/Boerderij/Varken/issues/53) +- Ability to add custom tautulli port [\#49](https://github.com/Boerderij/Varken/issues/49) + +**Merged pull requests:** + +- v1.1 Merge [\#57](https://github.com/Boerderij/Varken/pull/57) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) +- Update issue templates [\#55](https://github.com/Boerderij/Varken/pull/55) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) + +## [v1.0](https://github.com/Boerderij/Varken/tree/v1.0) (2018-12-10) [Full Changelog](https://github.com/Boerderij/Varken/compare/v0.3-nightly...v1.0) **Implemented enhancements:** diff --git a/varken/__init__.py b/varken/__init__.py index 341988c..c3ef0d4 100644 --- a/varken/__init__.py +++ b/varken/__init__.py @@ -1 +1 @@ -VERSION = 1.0 +VERSION = 1.1 From 9995a3bf0ed161547c637f4356c17432cdb5d187 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 11 Dec 2018 00:29:00 -0600 Subject: [PATCH 32/76] remove need for branches --- .travis.yml | 2 +- Dockerfile | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4653958..f4ed0fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ jobs: include: - stage: "Build and Push" script: - - docker build --build-arg branch=$TRAVIS_BRANCH -t boerderij/varken:$TRAVIS_BRANCH . + - docker build -t boerderij/varken:$TRAVIS_BRANCH . - docker images - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - docker push boerderij/varken:$TRAVIS_BRANCH diff --git a/Dockerfile b/Dockerfile index c0d4517..aa7fcf9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,6 @@ FROM lsiobase/alpine.python3 LABEL maintainer="dirtycajunrice" -ARG branch="master" - -ENV branch=$branch - COPY / /app RUN \ From 1252c1af8dfe5aaf51df494503e0853589eff42b Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 11 Dec 2018 00:40:38 -0600 Subject: [PATCH 33/76] sam said i was dumb --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b0ceab1..b7d4381 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ Requirements: git clone https://github.com/Boerderij/Varken.git /opt/Varken # Follow the systemd install instructions located in varken.systemd -nano /opt/Varken/varken.systemd cp /opt/Varken/varken.systemd /etc/systemd/system/varken.service +nano /etc/systemd/system/varken.service # Create venv in project /usr/bin/python3 -m venv /opt/Varken/varken-venv From 176be580b7e48a933bf015b1a9efc2ea460acbf1 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 11 Dec 2018 10:30:52 -0600 Subject: [PATCH 34/76] tweak logger responses in helper --- varken/helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index 0fb4d6a..8e184fb 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -80,21 +80,21 @@ def connection_handler(session, request, verify): try: get = s.send(r, verify=v) if get.status_code == 401: - logger.info('Your api key is incorrect for {}'.format(r.url)) + logger.info('Your api key is incorrect for %s', r.url) elif get.status_code == 404: - logger.info('This url doesnt even resolve: {}'.format(r.url)) + logger.info('This url doesnt even resolve: %s', r.url) elif get.status_code == 200: try: return_json = get.json() except JSONDecodeError: - logger.error('No JSON response... BORKED! Let us know in discord') + logger.error('No JSON response. Response is: %s', get.text) # 204 No Content is for ASA only elif get.status_code == 204: if get.headers['X-Auth-Token']: return get.headers['X-Auth-Token'] except InvalidSchema: - logger.error('You added http(s):// in the config file. Don\'t do that.') + logger.error("You added http(s):// in the config file. Don't do that.") except SSLError as e: logger.error('Either your host is unreachable or you have an SSL issue. : %s', e) From bb12b74555e6a21f23d3e84a4209173ec2b28a42 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Tue, 11 Dec 2018 11:39:27 -0500 Subject: [PATCH 35/76] Add ports to the example config --- data/varken.example.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/varken.example.ini b/data/varken.example.ini index 392ed80..a8e4b80 100644 --- a/data/varken.example.ini +++ b/data/varken.example.ini @@ -19,7 +19,7 @@ username = password = [tautulli-1] -url = tautulli.domain.tld +url = tautulli.domain.tld:8181 fallback_ip = 0.0.0.0 apikey = xxxxxxxxxxxxxxxx ssl = false @@ -28,7 +28,7 @@ get_activity = true get_activity_run_seconds = 30 [sonarr-1] -url = sonarr1.domain.tld +url = sonarr1.domain.tld:8989 apikey = xxxxxxxxxxxxxxxx ssl = false verify_ssl = true @@ -40,7 +40,7 @@ queue = true queue_run_seconds = 300 [sonarr-2] -url = sonarr2.domain.tld +url = sonarr2.domain.tld:8989 apikey = yyyyyyyyyyyyyyyy ssl = false verify_ssl = true From 1cfcbe6d1e643cb61f5c09ebf63b0a409c331ec4 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 11 Dec 2018 11:45:43 -0600 Subject: [PATCH 36/76] move static clean_check to helpers --- varken/helpers.py | 18 ++++++++++++++++++ varken/iniparser.py | 42 ++++++++---------------------------------- 2 files changed, 26 insertions(+), 34 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index 8e184fb..baf2357 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -109,3 +109,21 @@ def mkdir_p(path): os.makedirs(path, exist_ok=True) except Exception as e: logger.error('Could not create folder %s : %s ', path, e) + + +def clean_sid_check(server_id_list, server_type=None): + t = server_type + sid_list = server_id_list + cleaned_list = sid_list.replace(' ', '').split(',') + valid_sids = [] + for sid in cleaned_list: + try: + valid_sids.append(int(sid)) + except ValueError: + logger.error("{} is not a valid server id number".format(sid)) + if valid_sids: + logger.info('%s : %s', t.upper(), valid_sids) + return valid_sids + else: + logger.error('No valid %s', t.upper()) + return False diff --git a/varken/iniparser.py b/varken/iniparser.py index b242329..8586020 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -2,6 +2,8 @@ import configparser import logging from sys import exit from os.path import join, exists + +from varken.helpers import clean_sid_check from varken.structures import SonarrServer, RadarrServer, OmbiServer, TautulliServer, InfluxServer, CiscoASAFirewall logger = logging.getLogger() @@ -37,32 +39,12 @@ class INIParser(object): global_server_ids = self.config.get('global', t) if global_server_ids.lower() in ['false', 'no', '0']: logger.info('%s disabled.', t.upper()) - return False else: - sids = self.clean_check(global_server_ids, t) + sids = clean_sid_check(global_server_ids, t) return sids except configparser.NoOptionError as e: logger.error(e) - @staticmethod - def clean_check(server_id_list, server_type=None): - t = server_type - sid_list = server_id_list - cleaned_list = sid_list.replace(' ', '').split(',') - valid_sids = [] - for sid in cleaned_list: - try: - valid_sids.append(int(sid)) - except ValueError: - logger.error("{} is not a valid server id number".format(sid)) - - if valid_sids: - logger.info('%s : %s', t.upper(), valid_sids) - return valid_sids - else: - logger.error('No valid %s', t.upper()) - return False - def read_file(self): file_path = join(self.data_folder, 'varken.ini') if exists(file_path): @@ -85,9 +67,7 @@ class INIParser(object): self.sonarr_enabled = self.enable_check('sonarr_server_ids') if self.sonarr_enabled: - sids = self.config.get('global', 'sonarr_server_ids').strip(' ').split(',') - - for server_id in sids: + for server_id in self.sonarr_enabled: sonarr_section = 'sonarr-' + server_id url = self.config.get(sonarr_section, 'url') apikey = self.config.get(sonarr_section, 'apikey') @@ -111,9 +91,7 @@ class INIParser(object): self.radarr_enabled = self.enable_check('radarr_server_ids') if self.radarr_enabled: - sids = self.config.get('global', 'radarr_server_ids').strip(' ').split(',') - - for server_id in sids: + for server_id in self.radarr_enabled: radarr_section = 'radarr-' + server_id url = self.config.get(radarr_section, 'url') apikey = self.config.get(radarr_section, 'apikey') @@ -134,9 +112,7 @@ class INIParser(object): self.tautulli_enabled = self.enable_check('tautulli_server_ids') if self.tautulli_enabled: - sids = self.config.get('global', 'tautulli_server_ids').strip(' ').split(',') - - for server_id in sids: + for server_id in self.tautulli_enabled: tautulli_section = 'tautulli-' + server_id url = self.config.get(tautulli_section, 'url') fallback_ip = self.config.get(tautulli_section, 'fallback_ip') @@ -156,8 +132,7 @@ class INIParser(object): self.ombi_enabled = self.enable_check('ombi_server_ids') if self.ombi_enabled: - sids = self.config.get('global', 'ombi_server_ids').strip(' ').split(',') - for server_id in sids: + for server_id in self.ombi_enabled: ombi_section = 'ombi-' + server_id url = self.config.get(ombi_section, 'url') apikey = self.config.get(ombi_section, 'apikey') @@ -178,8 +153,7 @@ class INIParser(object): self.ciscoasa_enabled = self.enable_check('ciscoasa_firewall_ids') if self.ciscoasa_enabled: - fids = self.config.get('global', 'ciscoasa_firewall_ids').strip(' ').split(',') - for firewall_id in fids: + for firewall_id in self.ciscoasa_enabled: ciscoasa_section = 'ciscoasa-' + firewall_id url = self.config.get(ciscoasa_section, 'url') username = self.config.get(ciscoasa_section, 'username') From dde29c66203afa3478403ddea97cf698031f402e Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 11 Dec 2018 11:58:11 -0600 Subject: [PATCH 37/76] add key:value section options basic logging --- varken/iniparser.py | 116 +++++++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 50 deletions(-) diff --git a/varken/iniparser.py b/varken/iniparser.py index 8586020..a530bcc 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -93,20 +93,24 @@ class INIParser(object): if self.radarr_enabled: for server_id in self.radarr_enabled: radarr_section = 'radarr-' + server_id - url = self.config.get(radarr_section, 'url') - apikey = self.config.get(radarr_section, 'apikey') - scheme = 'https://' if self.config.getboolean(radarr_section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean(radarr_section, 'verify_ssl') - if scheme != 'https://': - verify_ssl = False - queue = self.config.getboolean(radarr_section, 'queue') - queue_run_seconds = self.config.getint(radarr_section, 'queue_run_seconds') - get_missing = self.config.getboolean(radarr_section, 'get_missing') - get_missing_run_seconds = self.config.getint(radarr_section, 'get_missing_run_seconds') + try: + url = self.config.get(radarr_section, 'url') + apikey = self.config.get(radarr_section, 'apikey') + scheme = 'https://' if self.config.getboolean(radarr_section, 'ssl') else 'http://' + verify_ssl = self.config.getboolean(radarr_section, 'verify_ssl') + if scheme != 'https://': + verify_ssl = False + queue = self.config.getboolean(radarr_section, 'queue') + queue_run_seconds = self.config.getint(radarr_section, 'queue_run_seconds') + get_missing = self.config.getboolean(radarr_section, 'get_missing') + get_missing_run_seconds = self.config.getint(radarr_section, 'get_missing_run_seconds') - server = RadarrServer(server_id, scheme + url, apikey, verify_ssl, queue, queue_run_seconds, - get_missing, get_missing_run_seconds) - self.radarr_servers.append(server) + server = RadarrServer(server_id, scheme + url, apikey, verify_ssl, queue, queue_run_seconds, + get_missing, get_missing_run_seconds) + self.radarr_servers.append(server) + except configparser.NoOptionError as e: + self.radarr_enabled = False + logger.error('%s disabled. Error: %s', radarr_section, e) # Parse Tautulli options self.tautulli_enabled = self.enable_check('tautulli_server_ids') @@ -114,19 +118,23 @@ class INIParser(object): if self.tautulli_enabled: for server_id in self.tautulli_enabled: tautulli_section = 'tautulli-' + server_id - url = self.config.get(tautulli_section, 'url') - fallback_ip = self.config.get(tautulli_section, 'fallback_ip') - apikey = self.config.get(tautulli_section, 'apikey') - scheme = 'https://' if self.config.getboolean(tautulli_section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean(tautulli_section, 'verify_ssl') - if scheme != 'https://': - verify_ssl = False - get_activity = self.config.getboolean(tautulli_section, 'get_activity') - get_activity_run_seconds = self.config.getint(tautulli_section, 'get_activity_run_seconds') + try: + url = self.config.get(tautulli_section, 'url') + fallback_ip = self.config.get(tautulli_section, 'fallback_ip') + apikey = self.config.get(tautulli_section, 'apikey') + scheme = 'https://' if self.config.getboolean(tautulli_section, 'ssl') else 'http://' + verify_ssl = self.config.getboolean(tautulli_section, 'verify_ssl') + if scheme != 'https://': + verify_ssl = False + get_activity = self.config.getboolean(tautulli_section, 'get_activity') + get_activity_run_seconds = self.config.getint(tautulli_section, 'get_activity_run_seconds') - server = TautulliServer(server_id, scheme + url, fallback_ip, apikey, verify_ssl, get_activity, - get_activity_run_seconds) - self.tautulli_servers.append(server) + server = TautulliServer(server_id, scheme + url, fallback_ip, apikey, verify_ssl, get_activity, + get_activity_run_seconds) + self.tautulli_servers.append(server) + except configparser.NoOptionError as e: + self.tautulli_enabled = False + logger.error('%s disabled. Error: %s', tautulli_section, e) # Parse Ombi options self.ombi_enabled = self.enable_check('ombi_server_ids') @@ -134,20 +142,24 @@ class INIParser(object): if self.ombi_enabled: for server_id in self.ombi_enabled: ombi_section = 'ombi-' + server_id - url = self.config.get(ombi_section, 'url') - apikey = self.config.get(ombi_section, 'apikey') - scheme = 'https://' if self.config.getboolean(ombi_section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean(ombi_section, 'verify_ssl') - if scheme != 'https://': - verify_ssl = False - request_type_counts = self.config.getboolean(ombi_section, 'get_request_type_counts') - request_type_run_seconds = self.config.getint(ombi_section, 'request_type_run_seconds') - request_total_counts = self.config.getboolean(ombi_section, 'get_request_total_counts') - request_total_run_seconds = self.config.getint(ombi_section, 'request_total_run_seconds') + try: + url = self.config.get(ombi_section, 'url') + apikey = self.config.get(ombi_section, 'apikey') + scheme = 'https://' if self.config.getboolean(ombi_section, 'ssl') else 'http://' + verify_ssl = self.config.getboolean(ombi_section, 'verify_ssl') + if scheme != 'https://': + verify_ssl = False + request_type_counts = self.config.getboolean(ombi_section, 'get_request_type_counts') + request_type_run_seconds = self.config.getint(ombi_section, 'request_type_run_seconds') + request_total_counts = self.config.getboolean(ombi_section, 'get_request_total_counts') + request_total_run_seconds = self.config.getint(ombi_section, 'request_total_run_seconds') - server = OmbiServer(server_id, scheme + url, apikey, verify_ssl, request_type_counts, - request_type_run_seconds, request_total_counts, request_total_run_seconds) - self.ombi_servers.append(server) + server = OmbiServer(server_id, scheme + url, apikey, verify_ssl, request_type_counts, + request_type_run_seconds, request_total_counts, request_total_run_seconds) + self.ombi_servers.append(server) + except configparser.NoOptionError as e: + self.ombi_enabled = False + logger.error('%s disabled. Error: %s', ombi_section, e) # Parse ASA opts self.ciscoasa_enabled = self.enable_check('ciscoasa_firewall_ids') @@ -155,16 +167,20 @@ class INIParser(object): if self.ciscoasa_enabled: for firewall_id in self.ciscoasa_enabled: ciscoasa_section = 'ciscoasa-' + firewall_id - url = self.config.get(ciscoasa_section, 'url') - username = self.config.get(ciscoasa_section, 'username') - password = self.config.get(ciscoasa_section, 'password') - scheme = 'https://' if self.config.getboolean(ciscoasa_section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean(ciscoasa_section, 'verify_ssl') - if scheme != 'https://': - verify_ssl = False - outside_interface = self.config.get(ciscoasa_section, 'outside_interface') - get_bandwidth_run_seconds = self.config.getint(ciscoasa_section, 'get_bandwidth_run_seconds') + try: + url = self.config.get(ciscoasa_section, 'url') + username = self.config.get(ciscoasa_section, 'username') + password = self.config.get(ciscoasa_section, 'password') + scheme = 'https://' if self.config.getboolean(ciscoasa_section, 'ssl') else 'http://' + verify_ssl = self.config.getboolean(ciscoasa_section, 'verify_ssl') + if scheme != 'https://': + verify_ssl = False + outside_interface = self.config.get(ciscoasa_section, 'outside_interface') + get_bandwidth_run_seconds = self.config.getint(ciscoasa_section, 'get_bandwidth_run_seconds') - firewall = CiscoASAFirewall(firewall_id, scheme + url, username, password, outside_interface, - verify_ssl, get_bandwidth_run_seconds) - self.ciscoasa_firewalls.append(firewall) + firewall = CiscoASAFirewall(firewall_id, scheme + url, username, password, outside_interface, + verify_ssl, get_bandwidth_run_seconds) + self.ciscoasa_firewalls.append(firewall) + except configparser.NoOptionError as e: + self.ciscoasa_enabled = False + logger.error('%s disabled. Error: %s', ciscoasa_section, e) From 88f5e22cecde1d40688c2a45039f733376210085 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 11 Dec 2018 13:30:54 -0600 Subject: [PATCH 38/76] typecast to string server_ids --- varken/iniparser.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/varken/iniparser.py b/varken/iniparser.py index a530bcc..a2eb171 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -68,7 +68,7 @@ class INIParser(object): if self.sonarr_enabled: for server_id in self.sonarr_enabled: - sonarr_section = 'sonarr-' + server_id + sonarr_section = 'sonarr-' + str(server_id) url = self.config.get(sonarr_section, 'url') apikey = self.config.get(sonarr_section, 'apikey') scheme = 'https://' if self.config.getboolean(sonarr_section, 'ssl') else 'http://' @@ -92,7 +92,7 @@ class INIParser(object): if self.radarr_enabled: for server_id in self.radarr_enabled: - radarr_section = 'radarr-' + server_id + radarr_section = 'radarr-' + str(server_id) try: url = self.config.get(radarr_section, 'url') apikey = self.config.get(radarr_section, 'apikey') @@ -117,7 +117,7 @@ class INIParser(object): if self.tautulli_enabled: for server_id in self.tautulli_enabled: - tautulli_section = 'tautulli-' + server_id + tautulli_section = 'tautulli-' + str(server_id) try: url = self.config.get(tautulli_section, 'url') fallback_ip = self.config.get(tautulli_section, 'fallback_ip') @@ -141,7 +141,7 @@ class INIParser(object): if self.ombi_enabled: for server_id in self.ombi_enabled: - ombi_section = 'ombi-' + server_id + ombi_section = 'ombi-' + str(server_id) try: url = self.config.get(ombi_section, 'url') apikey = self.config.get(ombi_section, 'apikey') @@ -166,7 +166,7 @@ class INIParser(object): if self.ciscoasa_enabled: for firewall_id in self.ciscoasa_enabled: - ciscoasa_section = 'ciscoasa-' + firewall_id + ciscoasa_section = 'ciscoasa-' + str(firewall_id) try: url = self.config.get(ciscoasa_section, 'url') username = self.config.get(ciscoasa_section, 'username') From c89322909823af2f2c59d70c2b14cf5db7a25fd5 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 11 Dec 2018 13:33:08 -0600 Subject: [PATCH 39/76] add sonarr sectionchecking --- varken/iniparser.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/varken/iniparser.py b/varken/iniparser.py index a2eb171..c818f3a 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -69,23 +69,27 @@ class INIParser(object): if self.sonarr_enabled: for server_id in self.sonarr_enabled: sonarr_section = 'sonarr-' + str(server_id) - url = self.config.get(sonarr_section, 'url') - apikey = self.config.get(sonarr_section, 'apikey') - scheme = 'https://' if self.config.getboolean(sonarr_section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean(sonarr_section, 'verify_ssl') - if scheme != 'https://': - verify_ssl = False - queue = self.config.getboolean(sonarr_section, 'queue') - missing_days = self.config.getint(sonarr_section, 'missing_days') - future_days = self.config.getint(sonarr_section, 'future_days') - missing_days_run_seconds = self.config.getint(sonarr_section, 'missing_days_run_seconds') - future_days_run_seconds = self.config.getint(sonarr_section, 'future_days_run_seconds') - queue_run_seconds = self.config.getint(sonarr_section, 'queue_run_seconds') + try: + url = self.config.get(sonarr_section, 'url') + apikey = self.config.get(sonarr_section, 'apikey') + scheme = 'https://' if self.config.getboolean(sonarr_section, 'ssl') else 'http://' + verify_ssl = self.config.getboolean(sonarr_section, 'verify_ssl') + if scheme != 'https://': + verify_ssl = False + queue = self.config.getboolean(sonarr_section, 'queue') + missing_days = self.config.getint(sonarr_section, 'missing_days') + future_days = self.config.getint(sonarr_section, 'future_days') + missing_days_run_seconds = self.config.getint(sonarr_section, 'missing_days_run_seconds') + future_days_run_seconds = self.config.getint(sonarr_section, 'future_days_run_seconds') + queue_run_seconds = self.config.getint(sonarr_section, 'queue_run_seconds') - server = SonarrServer(server_id, scheme + url, apikey, verify_ssl, missing_days, - missing_days_run_seconds, future_days, future_days_run_seconds, - queue, queue_run_seconds) - self.sonarr_servers.append(server) + server = SonarrServer(server_id, scheme + url, apikey, verify_ssl, missing_days, + missing_days_run_seconds, future_days, future_days_run_seconds, + queue, queue_run_seconds) + self.sonarr_servers.append(server) + except configparser.NoOptionError as e: + self.radarr_enabled = False + logger.error('%s disabled. Error: %s', sonarr_section, e) # Parse Radarr options self.radarr_enabled = self.enable_check('radarr_server_ids') From 9b860a0ca6f763e399bfbf12709d56e280420b35 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 11 Dec 2018 13:41:00 -0600 Subject: [PATCH 40/76] added exception handling for no route --- varken/helpers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/varken/helpers.py b/varken/helpers.py index baf2357..f53ed4b 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -8,7 +8,7 @@ import logging from json.decoder import JSONDecodeError from os.path import abspath, join -from requests.exceptions import InvalidSchema, SSLError +from requests.exceptions import InvalidSchema, SSLError, ConnectionError from urllib.request import urlretrieve logger = logging.getLogger('varken') @@ -99,6 +99,9 @@ def connection_handler(session, request, verify): except SSLError as e: logger.error('Either your host is unreachable or you have an SSL issue. : %s', e) + except ConnectionError as e: + logger.error('Cannot resolve the url/ip/port. Check connectivity. Error: %s', e) + return return_json From 7d5b11a6f8ca341e2938aa40e5b042d028f644f3 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 11 Dec 2018 19:43:22 -0600 Subject: [PATCH 41/76] added TZ to docker example --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b7d4381..3827917 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,8 @@ systemctl enable varken docker run -d \ --name=varken \ -v :/config \ - -e PGID= -e PUID= \ + -e PGID= -e PUID= \ + -e TZ=America/Chicago \ boerderij/varken ```

From 983467b0358520d573b27a3c1631cddc2f505ece Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Wed, 12 Dec 2018 00:19:25 -0500 Subject: [PATCH 42/76] Fix log message --- varken/dbmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varken/dbmanager.py b/varken/dbmanager.py index 4eee803..ff531ec 100644 --- a/varken/dbmanager.py +++ b/varken/dbmanager.py @@ -17,5 +17,5 @@ class DBManager(object): def write_points(self, data): d = data - logger.debug('Writing Data to InfluxDB {}'.format(d)) + logger.debug('Writing Data to InfluxDB %s', d) self.influx.write_points(d) From cb6249d31cbbba1b8af52d2f1a4d29a12d004bb1 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Wed, 12 Dec 2018 00:37:49 -0500 Subject: [PATCH 43/76] Add log filter for sensitive info such as API keys --- varken/iniparser.py | 118 ++++++++++++++++++++++++++++------------- varken/varkenlogger.py | 25 +++++++++ 2 files changed, 106 insertions(+), 37 deletions(-) diff --git a/varken/iniparser.py b/varken/iniparser.py index c818f3a..03c00e4 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -4,16 +4,17 @@ from sys import exit from os.path import join, exists from varken.helpers import clean_sid_check +from varken.varkenlogger import BlacklistFilter from varken.structures import SonarrServer, RadarrServer, OmbiServer, TautulliServer, InfluxServer, CiscoASAFirewall -logger = logging.getLogger() - class INIParser(object): def __init__(self, data_folder): self.config = configparser.ConfigParser(interpolation=None) self.data_folder = data_folder + self.logger = logging.getLogger() + self.influx_server = InfluxServer() self.sonarr_enabled = False @@ -33,25 +34,36 @@ class INIParser(object): self.parse_opts() + self.filtered_strings = None + + def config_blacklist(self): + filtered_strings = [section.get(k) for key, section in self.config.items() + for k in section if k in BlacklistFilter.blacklisted_strings] + self.filtered_strings = list(filter(None, filtered_strings)) + + for handler in self.logger.handlers: + handler.addFilter(BlacklistFilter(set(self.filtered_strings))) + def enable_check(self, server_type=None): t = server_type try: global_server_ids = self.config.get('global', t) if global_server_ids.lower() in ['false', 'no', '0']: - logger.info('%s disabled.', t.upper()) + self.logger.info('%s disabled.', t.upper()) else: sids = clean_sid_check(global_server_ids, t) return sids except configparser.NoOptionError as e: - logger.error(e) + self.logger.error(e) def read_file(self): file_path = join(self.data_folder, 'varken.ini') if exists(file_path): with open(file_path) as config_ini: self.config.read_file(config_ini) + self.config_blacklist() else: - exit('Config file missing (varken.ini) in {}'.format(self.data_folder)) + exit('Config file missing (varken.ini) in %s', self.data_folder) def parse_opts(self): self.read_file() @@ -72,16 +84,23 @@ class INIParser(object): try: url = self.config.get(sonarr_section, 'url') apikey = self.config.get(sonarr_section, 'apikey') - scheme = 'https://' if self.config.getboolean(sonarr_section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean(sonarr_section, 'verify_ssl') + scheme = 'https://' if self.config.getboolean( + sonarr_section, 'ssl') else 'http://' + verify_ssl = self.config.getboolean( + sonarr_section, 'verify_ssl') if scheme != 'https://': verify_ssl = False queue = self.config.getboolean(sonarr_section, 'queue') - missing_days = self.config.getint(sonarr_section, 'missing_days') - future_days = self.config.getint(sonarr_section, 'future_days') - missing_days_run_seconds = self.config.getint(sonarr_section, 'missing_days_run_seconds') - future_days_run_seconds = self.config.getint(sonarr_section, 'future_days_run_seconds') - queue_run_seconds = self.config.getint(sonarr_section, 'queue_run_seconds') + missing_days = self.config.getint( + sonarr_section, 'missing_days') + future_days = self.config.getint( + sonarr_section, 'future_days') + missing_days_run_seconds = self.config.getint( + sonarr_section, 'missing_days_run_seconds') + future_days_run_seconds = self.config.getint( + sonarr_section, 'future_days_run_seconds') + queue_run_seconds = self.config.getint( + sonarr_section, 'queue_run_seconds') server = SonarrServer(server_id, scheme + url, apikey, verify_ssl, missing_days, missing_days_run_seconds, future_days, future_days_run_seconds, @@ -89,7 +108,8 @@ class INIParser(object): self.sonarr_servers.append(server) except configparser.NoOptionError as e: self.radarr_enabled = False - logger.error('%s disabled. Error: %s', sonarr_section, e) + self.logger.error( + '%s disabled. Error: %s', sonarr_section, e) # Parse Radarr options self.radarr_enabled = self.enable_check('radarr_server_ids') @@ -100,21 +120,27 @@ class INIParser(object): try: url = self.config.get(radarr_section, 'url') apikey = self.config.get(radarr_section, 'apikey') - scheme = 'https://' if self.config.getboolean(radarr_section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean(radarr_section, 'verify_ssl') + scheme = 'https://' if self.config.getboolean( + radarr_section, 'ssl') else 'http://' + verify_ssl = self.config.getboolean( + radarr_section, 'verify_ssl') if scheme != 'https://': verify_ssl = False queue = self.config.getboolean(radarr_section, 'queue') - queue_run_seconds = self.config.getint(radarr_section, 'queue_run_seconds') - get_missing = self.config.getboolean(radarr_section, 'get_missing') - get_missing_run_seconds = self.config.getint(radarr_section, 'get_missing_run_seconds') + queue_run_seconds = self.config.getint( + radarr_section, 'queue_run_seconds') + get_missing = self.config.getboolean( + radarr_section, 'get_missing') + get_missing_run_seconds = self.config.getint( + radarr_section, 'get_missing_run_seconds') server = RadarrServer(server_id, scheme + url, apikey, verify_ssl, queue, queue_run_seconds, get_missing, get_missing_run_seconds) self.radarr_servers.append(server) except configparser.NoOptionError as e: self.radarr_enabled = False - logger.error('%s disabled. Error: %s', radarr_section, e) + self.logger.error( + '%s disabled. Error: %s', radarr_section, e) # Parse Tautulli options self.tautulli_enabled = self.enable_check('tautulli_server_ids') @@ -124,21 +150,27 @@ class INIParser(object): tautulli_section = 'tautulli-' + str(server_id) try: url = self.config.get(tautulli_section, 'url') - fallback_ip = self.config.get(tautulli_section, 'fallback_ip') + fallback_ip = self.config.get( + tautulli_section, 'fallback_ip') apikey = self.config.get(tautulli_section, 'apikey') - scheme = 'https://' if self.config.getboolean(tautulli_section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean(tautulli_section, 'verify_ssl') + scheme = 'https://' if self.config.getboolean( + tautulli_section, 'ssl') else 'http://' + verify_ssl = self.config.getboolean( + tautulli_section, 'verify_ssl') if scheme != 'https://': verify_ssl = False - get_activity = self.config.getboolean(tautulli_section, 'get_activity') - get_activity_run_seconds = self.config.getint(tautulli_section, 'get_activity_run_seconds') + get_activity = self.config.getboolean( + tautulli_section, 'get_activity') + get_activity_run_seconds = self.config.getint( + tautulli_section, 'get_activity_run_seconds') server = TautulliServer(server_id, scheme + url, fallback_ip, apikey, verify_ssl, get_activity, get_activity_run_seconds) self.tautulli_servers.append(server) except configparser.NoOptionError as e: self.tautulli_enabled = False - logger.error('%s disabled. Error: %s', tautulli_section, e) + self.logger.error( + '%s disabled. Error: %s', tautulli_section, e) # Parse Ombi options self.ombi_enabled = self.enable_check('ombi_server_ids') @@ -149,21 +181,28 @@ class INIParser(object): try: url = self.config.get(ombi_section, 'url') apikey = self.config.get(ombi_section, 'apikey') - scheme = 'https://' if self.config.getboolean(ombi_section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean(ombi_section, 'verify_ssl') + scheme = 'https://' if self.config.getboolean( + ombi_section, 'ssl') else 'http://' + verify_ssl = self.config.getboolean( + ombi_section, 'verify_ssl') if scheme != 'https://': verify_ssl = False - request_type_counts = self.config.getboolean(ombi_section, 'get_request_type_counts') - request_type_run_seconds = self.config.getint(ombi_section, 'request_type_run_seconds') - request_total_counts = self.config.getboolean(ombi_section, 'get_request_total_counts') - request_total_run_seconds = self.config.getint(ombi_section, 'request_total_run_seconds') + request_type_counts = self.config.getboolean( + ombi_section, 'get_request_type_counts') + request_type_run_seconds = self.config.getint( + ombi_section, 'request_type_run_seconds') + request_total_counts = self.config.getboolean( + ombi_section, 'get_request_total_counts') + request_total_run_seconds = self.config.getint( + ombi_section, 'request_total_run_seconds') server = OmbiServer(server_id, scheme + url, apikey, verify_ssl, request_type_counts, request_type_run_seconds, request_total_counts, request_total_run_seconds) self.ombi_servers.append(server) except configparser.NoOptionError as e: self.ombi_enabled = False - logger.error('%s disabled. Error: %s', ombi_section, e) + self.logger.error( + '%s disabled. Error: %s', ombi_section, e) # Parse ASA opts self.ciscoasa_enabled = self.enable_check('ciscoasa_firewall_ids') @@ -175,16 +214,21 @@ class INIParser(object): url = self.config.get(ciscoasa_section, 'url') username = self.config.get(ciscoasa_section, 'username') password = self.config.get(ciscoasa_section, 'password') - scheme = 'https://' if self.config.getboolean(ciscoasa_section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean(ciscoasa_section, 'verify_ssl') + scheme = 'https://' if self.config.getboolean( + ciscoasa_section, 'ssl') else 'http://' + verify_ssl = self.config.getboolean( + ciscoasa_section, 'verify_ssl') if scheme != 'https://': verify_ssl = False - outside_interface = self.config.get(ciscoasa_section, 'outside_interface') - get_bandwidth_run_seconds = self.config.getint(ciscoasa_section, 'get_bandwidth_run_seconds') + outside_interface = self.config.get( + ciscoasa_section, 'outside_interface') + get_bandwidth_run_seconds = self.config.getint( + ciscoasa_section, 'get_bandwidth_run_seconds') firewall = CiscoASAFirewall(firewall_id, scheme + url, username, password, outside_interface, verify_ssl, get_bandwidth_run_seconds) self.ciscoasa_firewalls.append(firewall) except configparser.NoOptionError as e: self.ciscoasa_enabled = False - logger.error('%s disabled. Error: %s', ciscoasa_section, e) + self.logger.error( + '%s disabled. Error: %s', ciscoasa_section, e) diff --git a/varken/varkenlogger.py b/varken/varkenlogger.py index 92d097b..3890a14 100644 --- a/varken/varkenlogger.py +++ b/varken/varkenlogger.py @@ -3,12 +3,37 @@ import logging from logging.handlers import RotatingFileHandler from varken.helpers import mkdir_p + FILENAME = "varken.log" MAX_SIZE = 5000000 # 5 MB MAX_FILES = 5 LOG_FOLDER = 'logs' +# Taken from Hellowlol/HTPC-Manager/Tautulli +class BlacklistFilter(logging.Filter): + """ + Log filter for blacklisted tokens and passwords + """ + blacklisted_strings = ['apikey', 'username', 'password'] + + def __init__(self, filteredstrings): + self.filtered_strings = filteredstrings + + def filter(self, record): + for item in self.filtered_strings: + try: + if item in record.msg: + record.msg = record.msg.replace(item, 8 * '*' + item[-2:]) + if any(item in str(arg) for arg in record.args): + record.args = tuple(arg.replace(item, 8 * '*' + item[-2:]) if isinstance(arg, str) else arg + for arg in record.args) + + except: + pass + return True + + class VarkenLogger(object): """docstring for .""" def __init__(self, log_path=None, debug=None, data_folder=None): From 1d348b5b705a8fee1b38d912b3102ffba44d0392 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Wed, 12 Dec 2018 12:38:58 -0500 Subject: [PATCH 44/76] Add the ability to toggle debug in docker --- Dockerfile | 4 +++- Varken.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aa7fcf9..ccfb4b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,8 @@ RUN \ /config \ /app +ENV DEBUG="False" + CMD cp /app/data/varken.example.ini /config/varken.example.ini && python3 /app/Varken.py --data-folder /config -VOLUME /config \ No newline at end of file +VOLUME /config diff --git a/Varken.py b/Varken.py index 8d6e5eb..a37b494 100644 --- a/Varken.py +++ b/Varken.py @@ -8,6 +8,7 @@ import schedule import threading import platform import distro +import os from sys import exit from time import sleep @@ -54,6 +55,9 @@ if __name__ == "__main__": else: exit("{} does not exist".format(ARG_FOLDER)) + if os.getenv('DEBUG', False) == 'True': + opts.debug = True + # Initiate the logger vl = VarkenLogger(data_folder=DATA_FOLDER, debug=opts.debug) vl.logger.info('Starting Varken...') From 74a256fc84962ece927fb2a813e09c28e467b623 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Wed, 12 Dec 2018 12:50:35 -0500 Subject: [PATCH 45/76] Move ENV --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ccfb4b0..33fe8b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,8 @@ FROM lsiobase/alpine.python3 LABEL maintainer="dirtycajunrice" +ENV DEBUG="False" + COPY / /app RUN \ @@ -10,8 +12,6 @@ RUN \ /config \ /app -ENV DEBUG="False" - CMD cp /app/data/varken.example.ini /config/varken.example.ini && python3 /app/Varken.py --data-folder /config VOLUME /config From d8517aeb1d0bb3a053401bd89eb985cec8b08dd0 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Wed, 12 Dec 2018 19:34:54 -0600 Subject: [PATCH 46/76] fixed logger error --- varken/iniparser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/varken/iniparser.py b/varken/iniparser.py index 03c00e4..f9dbc29 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -63,7 +63,8 @@ class INIParser(object): self.config.read_file(config_ini) self.config_blacklist() else: - exit('Config file missing (varken.ini) in %s', self.data_folder) + self.logger.error('Config file missing (varken.ini) in %s', self.data_folder) + exit(1) def parse_opts(self): self.read_file() From eb3f6bffa8fb496e570c25cf462680120f1ff4e9 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Wed, 12 Dec 2018 22:26:02 -0500 Subject: [PATCH 47/76] Update log message --- varken/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index f53ed4b..6c3e6fa 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -33,13 +33,13 @@ def geoip_download(data_folder): tar.extract(files, datafolder) logging.debug('%s has been extracted to %s', files, datafolder) - + os.remove(tar_dbfile) def geo_lookup(ipaddress, data_folder): datafolder = data_folder - logging.debug('Reading GeoLite2 from %s', datafolder) + logging.debug('Reading GeoLite2 DB from %s', datafolder) dbfile = abspath(join(datafolder, 'GeoLite2-City.mmdb')) now = time.time() From c16262aba6c4f0604141a00412bf5a9ef7468037 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Wed, 12 Dec 2018 22:26:56 -0500 Subject: [PATCH 48/76] Add sanity check for ip_address_public --- varken/tautulli.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/varken/tautulli.py b/varken/tautulli.py index a62da8f..096191d 100644 --- a/varken/tautulli.py +++ b/varken/tautulli.py @@ -1,4 +1,6 @@ +import os import logging + from requests import Session, Request from datetime import datetime, timezone from geoip2.errors import AddressNotFoundError @@ -38,7 +40,16 @@ class TautulliAPI(object): self.logger.error('TypeError has occurred : %s while creating TautulliStream structure', e) return + + for session in sessions: + # Check to see if ip_address_public atribute exists as it was introduced in v2 + try: + getattr(session, 'ip_address_public') + except AttributeError: + self.logger.error('Public IP attribute missing!!! Do you have an old version of Tautulli (v1)?') + os._exit(1) + try: geodata = geo_lookup(session.ip_address_public, self.data_folder) except (ValueError, AddressNotFoundError): From e70414abc7b4cb6735e09471d1394190bd22916c Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Wed, 12 Dec 2018 22:39:48 -0500 Subject: [PATCH 49/76] Fix spacing --- varken/tautulli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/varken/tautulli.py b/varken/tautulli.py index 096191d..b2cd549 100644 --- a/varken/tautulli.py +++ b/varken/tautulli.py @@ -40,8 +40,6 @@ class TautulliAPI(object): self.logger.error('TypeError has occurred : %s while creating TautulliStream structure', e) return - - for session in sessions: # Check to see if ip_address_public atribute exists as it was introduced in v2 try: From d357351651875d03af9a81d654e8c909776200d3 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Thu, 13 Dec 2018 14:53:18 -0500 Subject: [PATCH 50/76] Sanitize DEBUG ENV --- Varken.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Varken.py b/Varken.py index a37b494..04897d2 100644 --- a/Varken.py +++ b/Varken.py @@ -55,8 +55,12 @@ if __name__ == "__main__": else: exit("{} does not exist".format(ARG_FOLDER)) - if os.getenv('DEBUG', False) == 'True': - opts.debug = True + # Set Debug to True if DEBUG env is set + enable_opts = ['True', 'true', 'yes'] + debug_opts = ['debug', 'Debug', 'DEBUG'] + + opts.debug = True if any([os.getenv(string, False) for true in enable_opts + for string in debug_opts if os.getenv(string, False) == true]) else False # Initiate the logger vl = VarkenLogger(data_folder=DATA_FOLDER, debug=opts.debug) From 78d2e22e583176f2974fdf33bbf857cbacba11d2 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Thu, 13 Dec 2018 14:55:07 -0500 Subject: [PATCH 51/76] Add URL config check --- varken/iniparser.py | 77 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/varken/iniparser.py b/varken/iniparser.py index f9dbc29..f74b923 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -1,5 +1,7 @@ import configparser import logging +import re + from sys import exit from os.path import join, exists @@ -40,7 +42,7 @@ class INIParser(object): filtered_strings = [section.get(k) for key, section in self.config.items() for k in section if k in BlacklistFilter.blacklisted_strings] self.filtered_strings = list(filter(None, filtered_strings)) - + for handler in self.logger.handlers: handler.addFilter(BlacklistFilter(set(self.filtered_strings))) @@ -66,12 +68,34 @@ class INIParser(object): self.logger.error('Config file missing (varken.ini) in %s', self.data_folder) exit(1) + def url_check(self, url=None): + url_check = url + + regex = re.compile( + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain... + r'localhost|' #localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE + ) + + valid = re.match(regex, url_check) is not None + if not valid: + self.logger.error('%s is invalid! URL must host/IP and port if not 80 or 443. ie. localhost:8080', url_check) + exit(1) + else: + self.logger.debug('%s is a vlaid URL in the config', url_check) + return url_check + def parse_opts(self): self.read_file() # Parse InfluxDB options url = self.config.get('influxdb', 'url') + port = self.config.getint('influxdb', 'port') + username = self.config.get('influxdb', 'username') + password = self.config.get('influxdb', 'password') self.influx_server = InfluxServer(url, port, username, password) @@ -83,29 +107,40 @@ class INIParser(object): for server_id in self.sonarr_enabled: sonarr_section = 'sonarr-' + str(server_id) try: - url = self.config.get(sonarr_section, 'url') + url = self.url_check(self.config.get(sonarr_section, 'url')) + apikey = self.config.get(sonarr_section, 'apikey') + scheme = 'https://' if self.config.getboolean( sonarr_section, 'ssl') else 'http://' + verify_ssl = self.config.getboolean( sonarr_section, 'verify_ssl') + if scheme != 'https://': verify_ssl = False + queue = self.config.getboolean(sonarr_section, 'queue') + missing_days = self.config.getint( sonarr_section, 'missing_days') + future_days = self.config.getint( sonarr_section, 'future_days') + missing_days_run_seconds = self.config.getint( sonarr_section, 'missing_days_run_seconds') + future_days_run_seconds = self.config.getint( sonarr_section, 'future_days_run_seconds') + queue_run_seconds = self.config.getint( sonarr_section, 'queue_run_seconds') server = SonarrServer(server_id, scheme + url, apikey, verify_ssl, missing_days, missing_days_run_seconds, future_days, future_days_run_seconds, queue, queue_run_seconds) + self.sonarr_servers.append(server) except configparser.NoOptionError as e: self.radarr_enabled = False @@ -119,19 +154,27 @@ class INIParser(object): for server_id in self.radarr_enabled: radarr_section = 'radarr-' + str(server_id) try: - url = self.config.get(radarr_section, 'url') + url = self.url_check(self.config.get(radarr_section, 'url')) + apikey = self.config.get(radarr_section, 'apikey') + scheme = 'https://' if self.config.getboolean( radarr_section, 'ssl') else 'http://' + verify_ssl = self.config.getboolean( radarr_section, 'verify_ssl') + if scheme != 'https://': verify_ssl = False + queue = self.config.getboolean(radarr_section, 'queue') + queue_run_seconds = self.config.getint( radarr_section, 'queue_run_seconds') + get_missing = self.config.getboolean( radarr_section, 'get_missing') + get_missing_run_seconds = self.config.getint( radarr_section, 'get_missing_run_seconds') @@ -150,18 +193,25 @@ class INIParser(object): for server_id in self.tautulli_enabled: tautulli_section = 'tautulli-' + str(server_id) try: - url = self.config.get(tautulli_section, 'url') + url = self.url_check(self.config.get(tautulli_section, 'url')) + fallback_ip = self.config.get( tautulli_section, 'fallback_ip') + apikey = self.config.get(tautulli_section, 'apikey') + scheme = 'https://' if self.config.getboolean( tautulli_section, 'ssl') else 'http://' + verify_ssl = self.config.getboolean( tautulli_section, 'verify_ssl') + if scheme != 'https://': verify_ssl = False + get_activity = self.config.getboolean( tautulli_section, 'get_activity') + get_activity_run_seconds = self.config.getint( tautulli_section, 'get_activity_run_seconds') @@ -180,20 +230,28 @@ class INIParser(object): for server_id in self.ombi_enabled: ombi_section = 'ombi-' + str(server_id) try: - url = self.config.get(ombi_section, 'url') + url = self.url_check(self.config.get(ombi_section, 'url')) + apikey = self.config.get(ombi_section, 'apikey') + scheme = 'https://' if self.config.getboolean( ombi_section, 'ssl') else 'http://' + verify_ssl = self.config.getboolean( ombi_section, 'verify_ssl') + if scheme != 'https://': verify_ssl = False + request_type_counts = self.config.getboolean( ombi_section, 'get_request_type_counts') + request_type_run_seconds = self.config.getint( ombi_section, 'request_type_run_seconds') + request_total_counts = self.config.getboolean( ombi_section, 'get_request_total_counts') + request_total_run_seconds = self.config.getint( ombi_section, 'request_total_run_seconds') @@ -212,17 +270,24 @@ class INIParser(object): for firewall_id in self.ciscoasa_enabled: ciscoasa_section = 'ciscoasa-' + str(firewall_id) try: - url = self.config.get(ciscoasa_section, 'url') + url = self.url_check(self.config.get(ciscoasa_section, 'url')) + username = self.config.get(ciscoasa_section, 'username') + password = self.config.get(ciscoasa_section, 'password') + scheme = 'https://' if self.config.getboolean( ciscoasa_section, 'ssl') else 'http://' + verify_ssl = self.config.getboolean( ciscoasa_section, 'verify_ssl') + if scheme != 'https://': verify_ssl = False + outside_interface = self.config.get( ciscoasa_section, 'outside_interface') + get_bandwidth_run_seconds = self.config.getint( ciscoasa_section, 'get_bandwidth_run_seconds') From cb966f2cc660abb0fab73cba14250581451c2e2b Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Thu, 13 Dec 2018 14:59:53 -0500 Subject: [PATCH 52/76] Fix command line arg --- Varken.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Varken.py b/Varken.py index 04897d2..2f1e7be 100644 --- a/Varken.py +++ b/Varken.py @@ -59,8 +59,9 @@ if __name__ == "__main__": enable_opts = ['True', 'true', 'yes'] debug_opts = ['debug', 'Debug', 'DEBUG'] - opts.debug = True if any([os.getenv(string, False) for true in enable_opts - for string in debug_opts if os.getenv(string, False) == true]) else False + if not opts.debug: + opts.debug = True if any([os.getenv(string, False) for true in enable_opts + for string in debug_opts if os.getenv(string, False) == true]) else False # Initiate the logger vl = VarkenLogger(data_folder=DATA_FOLDER, debug=opts.debug) From 7b8403c9ba140d06e9b86c446828ad3bd43da8b8 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Fri, 14 Dec 2018 15:25:17 -0500 Subject: [PATCH 53/76] Add URL check to InfluxDB --- varken/iniparser.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/varken/iniparser.py b/varken/iniparser.py index f74b923..cbd03df 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -68,29 +68,40 @@ class INIParser(object): self.logger.error('Config file missing (varken.ini) in %s', self.data_folder) exit(1) - def url_check(self, url=None): + def url_check(self, url=None, include_port=True): url_check = url + inc_port = include_port - regex = re.compile( - r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain... - r'localhost|' #localhost... - r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip - r'(?::\d+)?' # optional port - r'(?:/?|[/?]\S+)$', re.IGNORECASE - ) + search = (r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain... + r'localhost|' #localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip + ) + + if inc_port: + search = (search + r'(?::\d+)?' + r'(?:/?|[/?]\S+)$') + else: + search = (search + r'(?:/?|[/?]\S+)$') + + regex = re.compile('{}'.format(search), re.IGNORECASE) + + print(re.match(regex, url_check)) valid = re.match(regex, url_check) is not None if not valid: - self.logger.error('%s is invalid! URL must host/IP and port if not 80 or 443. ie. localhost:8080', url_check) - exit(1) + if inc_port: + self.logger.error('%s is invalid! URL must host/IP and port if not 80 or 443. ie. localhost:8080', url_check) + exit(1) + else: + self.logger.error('%s is invalid! URL must host/IP. ie. localhost', url_check) + exit(1) else: - self.logger.debug('%s is a vlaid URL in the config', url_check) + self.logger.debug('%s is a vlaid URL in the config.', url_check) return url_check def parse_opts(self): self.read_file() # Parse InfluxDB options - url = self.config.get('influxdb', 'url') + url = self.url_check(self.config.get('influxdb', 'url'), include_port=False) port = self.config.getint('influxdb', 'port') From 810c0b64bbc51e0050f7784d0dd489689426d51a Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Fri, 14 Dec 2018 15:37:43 -0500 Subject: [PATCH 54/76] Update readme --- README.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3827917..38e0e8c 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,26 @@ frontend Requirements: * Python3.6+ * Python3-pip -* InfluxDB +* [InfluxDB](https://www.influxdata.com/)

+_Example Dashboard_

+Supported Modules: +* [Sonarr](https://sonarr.tv/) - Smart PVR for newsgroup and bittorrent users. +* [Radarr](https://radarr.video/) - A fork of Sonarr to work with movies à la Couchpotato. +* [Tautulli](https://tautulli.com/) - A Python based monitoring and tracking tool for Plex Media Server. +* [Ombi](https://ombi.io/) - Want a Movie or TV Show on Plex or Emby? Use Ombi! +* Cisco ASA + +Key features: +* Multiple server support for all modules +* Geolocation mapping from [GeoLite2](https://dev.maxmind.com/geoip/geoip2/geolite2/) +* Grafana [Worldmap Panel](https://grafana.com/plugins/grafana-worldmap-panel/installation) support + + ## Quick Setup (Git Clone) ``` # Clone the repository @@ -92,4 +106,4 @@ named `varken` Grafana is used in our examples but not required, nor packaged as part of Varken. Panel example pictures are pinned in the grafana-panels channel of discord. Future releases may contain a json-generator, but it does not exist -as varken stands today. \ No newline at end of file +as varken stands today. From 5f1728bd5effa704bcc9e51ff64a8492720e3d0b Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Fri, 14 Dec 2018 15:38:19 -0500 Subject: [PATCH 55/76] Add Space --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 38e0e8c..aa6a8de 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Requirements:

+ _Example Dashboard_

From 06308f7311201b9b9f75d1ca76a0823f73fd5b18 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Fri, 14 Dec 2018 15:42:43 -0500 Subject: [PATCH 56/76] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index aa6a8de..e91a4bd 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,9 @@ Requirements: * [InfluxDB](https://www.influxdata.com/)

- +Example Dashboard -_Example Dashboard_ +

Supported Modules: From 4392a2f3de168f04aa8e246c96c803b265b22b48 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Fri, 14 Dec 2018 22:47:27 -0500 Subject: [PATCH 57/76] Convert Air date to UTC --- varken/sonarr.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/varken/sonarr.py b/varken/sonarr.py index 117136a..54f027d 100644 --- a/varken/sonarr.py +++ b/varken/sonarr.py @@ -45,9 +45,9 @@ class SonarrAPI(object): for show in tv_shows: if not show.hasFile: sxe = 'S{:0>2}E{:0>2}'.format(show.seasonNumber, show.episodeNumber) - missing.append((show.series['title'], sxe, show.airDate, show.title, show.id)) + missing.append((show.series['title'], sxe, show.airDateUtc, show.title, show.id)) - for series_title, sxe, air_date, episode_title, sonarr_id in missing: + for series_title, sxe, air_date_utc, episode_title, sonarr_id in missing: hash_id = hashit('{}{}{}'.format(self.server.id, series_title, sxe)) influx_payload.append( { @@ -59,7 +59,7 @@ class SonarrAPI(object): "name": series_title, "epname": episode_title, "sxe": sxe, - "airs": air_date + "airsUTC": air_date_utc }, "time": now, "fields": { @@ -98,9 +98,9 @@ class SonarrAPI(object): downloaded = 1 else: downloaded = 0 - air_days.append((show.series['title'], downloaded, sxe, show.title, show.airDate, show.id)) + air_days.append((show.series['title'], downloaded, sxe, show.title, show.airDateUtc, show.id)) - for series_title, dl_status, sxe, episode_title, air_date, sonarr_id in air_days: + for series_title, dl_status, sxe, episode_title, air_date_utc, sonarr_id in air_days: hash_id = hashit('{}{}{}'.format(self.server.id, series_title, sxe)) influx_payload.append( { @@ -112,7 +112,7 @@ class SonarrAPI(object): "name": series_title, "epname": episode_title, "sxe": sxe, - "airs": air_date, + "airsUTC": air_date_utc, "downloaded": dl_status }, "time": now, From 056d211d99f4b7aa214ad305f8021c3a9dab144e Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Sat, 15 Dec 2018 21:37:42 -0600 Subject: [PATCH 58/76] added the ability to pull per request chart. Fixes #58 --- Varken.py | 2 +- varken/ombi.py | 89 ++++++++++++++++++++++++++++++++++++++++---- varken/structures.py | 47 +++++++++++++++++++++++ 3 files changed, 129 insertions(+), 9 deletions(-) diff --git a/Varken.py b/Varken.py index 2f1e7be..b03bc9e 100644 --- a/Varken.py +++ b/Varken.py @@ -108,7 +108,7 @@ if __name__ == "__main__": if server.request_type_counts: schedule.every(server.request_type_run_seconds).seconds.do(threaded, OMBI.get_request_counts) if server.request_total_counts: - schedule.every(server.request_total_run_seconds).seconds.do(threaded, OMBI.get_total_requests) + schedule.every(server.request_total_run_seconds).seconds.do(threaded, OMBI.get_all_requests) if CONFIG.ciscoasa_enabled: for firewall in CONFIG.ciscoasa_firewalls: diff --git a/varken/ombi.py b/varken/ombi.py index 7ea33fc..3b8d56b 100644 --- a/varken/ombi.py +++ b/varken/ombi.py @@ -2,8 +2,8 @@ import logging from requests import Session, Request from datetime import datetime, timezone -from varken.helpers import connection_handler -from varken.structures import OmbiRequestCounts +from varken.helpers import connection_handler, hashit +from varken.structures import OmbiRequestCounts, OmbiMovieRequest, OmbiTVRequest class OmbiAPI(object): @@ -18,7 +18,7 @@ class OmbiAPI(object): def __repr__(self): return "".format(self.server.id) - def get_total_requests(self): + def get_all_requests(self): now = datetime.now(timezone.utc).astimezone().isoformat() tv_endpoint = '/api/v1/Request/tv' movie_endpoint = "/api/v1/Request/movie" @@ -31,8 +31,20 @@ class OmbiAPI(object): if not all([get_tv, get_movie]): return - movie_requests = len(get_movie) - tv_requests = len(get_tv) + movie_request_count = len(get_movie) + tv_request_count = len(get_tv) + + try: + tv_show_requests = [OmbiTVRequest(**show) for show in get_tv] + except TypeError as e: + self.logger.error('TypeError has occurred : %s while creating OmbiTVRequest structure', e) + return + + try: + movie_requests = [OmbiMovieRequest(**movie) for movie in get_movie] + except TypeError as e: + self.logger.error('TypeError has occurred : %s while creating OmbiMovieRequest structure', e) + return influx_payload = [ { @@ -43,12 +55,73 @@ class OmbiAPI(object): }, "time": now, "fields": { - "total": movie_requests + tv_requests, - "movies": movie_requests, - "tv_shows": tv_requests + "total": movie_request_count + tv_request_count, + "movies": movie_request_count, + "tv_shows": tv_request_count } } ] + # Request Type: Movie = 1, TV Show = 0 + for movie in movie_requests: + hash_id = hashit(f'{movie.id}{movie.theMovieDbId}{movie.title}') + status = None + # Denied = 0, Approved = 1, Completed = 2 + if movie.denied: + status = 0 + elif movie.approved and movie.available: + status = 2 + elif movie.approved: + status = 1 + + influx_payload.append( + { + "measurement": "Ombi", + "tags": { + "type": "Requests", + "server": self.server.id, + "request_type": 1, + "status": status, + "title": movie.title, + "requested_user": movie.requestedUser['userAlias'], + "requested_date": movie.requestedDate + }, + "time": now, + "fields": { + "hash": hash_id + } + } + ) + + for show in tv_show_requests: + hash_id = hashit(f'{show.id}{show.tvDbId}{show.title}') + status = None + # Denied = 0, Approved = 1, Completed = 2 + if show.childRequests[0]['denied']: + status = 0 + elif show.childRequests[0]['approved'] and show.childRequests[0]['available']: + status = 2 + elif show.childRequests[0]['approved']: + status = 1 + + influx_payload.append( + { + "measurement": "Ombi", + "tags": { + "type": "Requests", + "server": self.server.id, + "request_type": 0, + "status": status, + "title": show.title, + "requested_user": show.childRequests[0]['requestedUser']['userAlias'], + "requested_date": show.childRequests[0]['requestedDate'] + }, + "time": now, + "fields": { + "hash": hash_id + } + } + ) + self.dbmanager.write_points(influx_payload) diff --git a/varken/structures.py b/varken/structures.py index 34fa91c..3e1a583 100644 --- a/varken/structures.py +++ b/varken/structures.py @@ -337,3 +337,50 @@ class Movie(NamedTuple): physicalReleaseNote: str = None website: str = None id: int = None + +class OmbiMovieRequest(NamedTuple): + theMovieDbId: int = None + issueId: None = None + issues: None = None + subscribed: bool = None + showSubscribe: bool = None + rootPathOverride: int = None + qualityOverride: int = None + imdbId: str = None + overview: str = None + posterPath: str = None + releaseDate: str = None + digitalReleaseDate: None = None + status: str = None + background: str = None + released: bool = None + digitalRelease: bool = None + title: str = None + approved: bool = None + markedAsApproved: str = None + requestedDate: str = None + available: bool = None + markedAsAvailable: None = None + requestedUserId: str = None + denied: bool = None + markedAsDenied: str = None + deniedReason: None = None + requestType: int = None + requestedUser: dict = None + canApprove: bool = None + id: int = None + +class OmbiTVRequest(NamedTuple): + tvDbId: int = None + imdbId: str = None + qualityOverride: None = None + rootFolder: None = None + overview: str = None + title: str = None + posterPath: str = None + background: str = None + releaseDate: str = None + status: str = None + totalSeasons: int = None + childRequests: list = None + id: int = None From 78b7765553db8e9037a6df7b1c1abac803753bd6 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Sun, 16 Dec 2018 00:55:59 -0600 Subject: [PATCH 59/76] added sickchill. fixes #48 --- Varken.py | 9 +- data/varken.example.ini | 25 ++++-- varken/iniparser.py | 176 +++++++++++++++++++++------------------- varken/sickchill.py | 64 +++++++++++++++ varken/structures.py | 26 ++++++ 5 files changed, 208 insertions(+), 92 deletions(-) create mode 100644 varken/sickchill.py diff --git a/Varken.py b/Varken.py index b03bc9e..8734e12 100644 --- a/Varken.py +++ b/Varken.py @@ -21,6 +21,7 @@ from varken.sonarr import SonarrAPI from varken.tautulli import TautulliAPI from varken.radarr import RadarrAPI from varken.ombi import OmbiAPI +from varken.sickchill import SickChillAPI from varken.cisco import CiscoAPI from varken.dbmanager import DBManager from varken.varkenlogger import VarkenLogger @@ -110,6 +111,12 @@ if __name__ == "__main__": if server.request_total_counts: schedule.every(server.request_total_run_seconds).seconds.do(threaded, OMBI.get_all_requests) + if CONFIG.sickchill_enabled: + for server in CONFIG.sickchill_servers: + SICKCHILL = SickChillAPI(server, DBMANAGER) + if server.get_missing: + schedule.every(server.get_missing_run_seconds).seconds.do(threaded, SICKCHILL.get_missing) + if CONFIG.ciscoasa_enabled: for firewall in CONFIG.ciscoasa_firewalls: ASA = CiscoAPI(firewall, DBMANAGER) @@ -117,7 +124,7 @@ if __name__ == "__main__": # Run all on startup SERVICES_ENABLED = [CONFIG.ombi_enabled, CONFIG.radarr_enabled, CONFIG.tautulli_enabled, - CONFIG.sonarr_enabled, CONFIG.ciscoasa_enabled] + CONFIG.sonarr_enabled, CONFIG.ciscoasa_enabled, CONFIG.sickchill_enabled] if not [enabled for enabled in SERVICES_ENABLED if enabled]: exit("All services disabled. Exiting") schedule.run_all() diff --git a/data/varken.example.ini b/data/varken.example.ini index a8e4b80..51675ad 100644 --- a/data/varken.example.ini +++ b/data/varken.example.ini @@ -11,6 +11,7 @@ radarr_server_ids = 1,2 tautulli_server_ids = 1 ombi_server_ids = 1 ciscoasa_firewall_ids = false +sickchill_server_ids = false [influxdb] url = influxdb.domain.tld @@ -23,7 +24,7 @@ url = tautulli.domain.tld:8181 fallback_ip = 0.0.0.0 apikey = xxxxxxxxxxxxxxxx ssl = false -verify_ssl = true +verify_ssl = false get_activity = true get_activity_run_seconds = 30 @@ -31,7 +32,7 @@ get_activity_run_seconds = 30 url = sonarr1.domain.tld:8989 apikey = xxxxxxxxxxxxxxxx ssl = false -verify_ssl = true +verify_ssl = false missing_days = 7 missing_days_run_seconds = 300 future_days = 1 @@ -43,7 +44,7 @@ queue_run_seconds = 300 url = sonarr2.domain.tld:8989 apikey = yyyyyyyyyyyyyyyy ssl = false -verify_ssl = true +verify_ssl = false missing_days = 7 missing_days_run_seconds = 300 future_days = 1 @@ -55,7 +56,7 @@ queue_run_seconds = 300 url = radarr1.domain.tld apikey = xxxxxxxxxxxxxxxx ssl = false -verify_ssl = true +verify_ssl = false queue = true queue_run_seconds = 300 get_missing = true @@ -65,7 +66,7 @@ get_missing_run_seconds = 300 url = radarr2.domain.tld apikey = yyyyyyyyyyyyyyyy ssl = false -verify_ssl = true +verify_ssl = false queue = true queue_run_seconds = 300 get_missing = true @@ -75,17 +76,27 @@ get_missing_run_seconds = 300 url = ombi.domain.tld apikey = xxxxxxxxxxxxxxxx ssl = false -verify_ssl = true +verify_ssl = false get_request_type_counts = true request_type_run_seconds = 300 get_request_total_counts = true request_total_run_seconds = 300 +[sickchill-1] +url = sickchill.domain.tld:8081 +apikey = xxxxxxxxxxxxxxxx +ssl = false +verify_ssl = false +get_missing = true +get_missing_run_seconds = 300 + + + [ciscoasa-1] url = firewall.domain.tld username = cisco password = cisco outside_interface = WAN ssl = false -verify_ssl = true +verify_ssl = false get_bandwidth_run_seconds = 300 diff --git a/varken/iniparser.py b/varken/iniparser.py index cbd03df..846de52 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -8,6 +8,7 @@ from os.path import join, exists from varken.helpers import clean_sid_check from varken.varkenlogger import BlacklistFilter from varken.structures import SonarrServer, RadarrServer, OmbiServer, TautulliServer, InfluxServer, CiscoASAFirewall +from varken.structures import SickChillServer class INIParser(object): @@ -31,6 +32,9 @@ class INIParser(object): self.tautulli_enabled = False self.tautulli_servers = [] + self.sickchill_enabled = False + self.sickchill_servers = [] + self.ciscoasa_enabled = False self.ciscoasa_firewalls = [] @@ -40,7 +44,7 @@ class INIParser(object): def config_blacklist(self): filtered_strings = [section.get(k) for key, section in self.config.items() - for k in section if k in BlacklistFilter.blacklisted_strings] + for k in section if k in BlacklistFilter.blacklisted_strings] self.filtered_strings = list(filter(None, filtered_strings)) for handler in self.logger.handlers: @@ -72,9 +76,9 @@ class INIParser(object): url_check = url inc_port = include_port - search = (r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain... - r'localhost|' #localhost... - r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip + search = (r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... + r'localhost|' # localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip ) if inc_port: @@ -89,13 +93,14 @@ class INIParser(object): valid = re.match(regex, url_check) is not None if not valid: if inc_port: - self.logger.error('%s is invalid! URL must host/IP and port if not 80 or 443. ie. localhost:8080', url_check) + self.logger.error('%s is invalid! URL must host/IP and port if not 80 or 443. ie. localhost:8080', + url_check) exit(1) else: self.logger.error('%s is invalid! URL must host/IP. ie. localhost', url_check) exit(1) else: - self.logger.debug('%s is a vlaid URL in the config.', url_check) + self.logger.debug('%s is a valid URL in the config.', url_check) return url_check def parse_opts(self): @@ -116,37 +121,30 @@ class INIParser(object): if self.sonarr_enabled: for server_id in self.sonarr_enabled: - sonarr_section = 'sonarr-' + str(server_id) + section = 'sonarr-' + str(server_id) try: - url = self.url_check(self.config.get(sonarr_section, 'url')) + url = self.url_check(self.config.get(section, 'url')) - apikey = self.config.get(sonarr_section, 'apikey') + apikey = self.config.get(section, 'apikey') - scheme = 'https://' if self.config.getboolean( - sonarr_section, 'ssl') else 'http://' + scheme = 'https://' if self.config.getboolean(section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean( - sonarr_section, 'verify_ssl') + verify_ssl = self.config.getboolean(section, 'verify_ssl') if scheme != 'https://': verify_ssl = False - queue = self.config.getboolean(sonarr_section, 'queue') + queue = self.config.getboolean(section, 'queue') - missing_days = self.config.getint( - sonarr_section, 'missing_days') + missing_days = self.config.getint(section, 'missing_days') - future_days = self.config.getint( - sonarr_section, 'future_days') + future_days = self.config.getint(section, 'future_days') - missing_days_run_seconds = self.config.getint( - sonarr_section, 'missing_days_run_seconds') + missing_days_run_seconds = self.config.getint(section, 'missing_days_run_seconds') - future_days_run_seconds = self.config.getint( - sonarr_section, 'future_days_run_seconds') + future_days_run_seconds = self.config.getint(section, 'future_days_run_seconds') - queue_run_seconds = self.config.getint( - sonarr_section, 'queue_run_seconds') + queue_run_seconds = self.config.getint(section, 'queue_run_seconds') server = SonarrServer(server_id, scheme + url, apikey, verify_ssl, missing_days, missing_days_run_seconds, future_days, future_days_run_seconds, @@ -154,40 +152,35 @@ class INIParser(object): self.sonarr_servers.append(server) except configparser.NoOptionError as e: - self.radarr_enabled = False + self.sonarr_enabled = False self.logger.error( - '%s disabled. Error: %s', sonarr_section, e) + '%s disabled. Error: %s', section, e) # Parse Radarr options self.radarr_enabled = self.enable_check('radarr_server_ids') if self.radarr_enabled: for server_id in self.radarr_enabled: - radarr_section = 'radarr-' + str(server_id) + section = 'radarr-' + str(server_id) try: - url = self.url_check(self.config.get(radarr_section, 'url')) + url = self.url_check(self.config.get(section, 'url')) - apikey = self.config.get(radarr_section, 'apikey') + apikey = self.config.get(section, 'apikey') - scheme = 'https://' if self.config.getboolean( - radarr_section, 'ssl') else 'http://' + scheme = 'https://' if self.config.getboolean(section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean( - radarr_section, 'verify_ssl') + verify_ssl = self.config.getboolean(section, 'verify_ssl') if scheme != 'https://': verify_ssl = False - queue = self.config.getboolean(radarr_section, 'queue') + queue = self.config.getboolean(section, 'queue') - queue_run_seconds = self.config.getint( - radarr_section, 'queue_run_seconds') + queue_run_seconds = self.config.getint(section, 'queue_run_seconds') - get_missing = self.config.getboolean( - radarr_section, 'get_missing') + get_missing = self.config.getboolean(section, 'get_missing') - get_missing_run_seconds = self.config.getint( - radarr_section, 'get_missing_run_seconds') + get_missing_run_seconds = self.config.getint(section, 'get_missing_run_seconds') server = RadarrServer(server_id, scheme + url, apikey, verify_ssl, queue, queue_run_seconds, get_missing, get_missing_run_seconds) @@ -195,36 +188,31 @@ class INIParser(object): except configparser.NoOptionError as e: self.radarr_enabled = False self.logger.error( - '%s disabled. Error: %s', radarr_section, e) + '%s disabled. Error: %s', section, e) # Parse Tautulli options self.tautulli_enabled = self.enable_check('tautulli_server_ids') if self.tautulli_enabled: for server_id in self.tautulli_enabled: - tautulli_section = 'tautulli-' + str(server_id) + section = 'tautulli-' + str(server_id) try: - url = self.url_check(self.config.get(tautulli_section, 'url')) + url = self.url_check(self.config.get(section, 'url')) - fallback_ip = self.config.get( - tautulli_section, 'fallback_ip') + fallback_ip = self.config.get(section, 'fallback_ip') - apikey = self.config.get(tautulli_section, 'apikey') + apikey = self.config.get(section, 'apikey') - scheme = 'https://' if self.config.getboolean( - tautulli_section, 'ssl') else 'http://' + scheme = 'https://' if self.config.getboolean(section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean( - tautulli_section, 'verify_ssl') + verify_ssl = self.config.getboolean(section, 'verify_ssl') if scheme != 'https://': verify_ssl = False - get_activity = self.config.getboolean( - tautulli_section, 'get_activity') + get_activity = self.config.getboolean(section, 'get_activity') - get_activity_run_seconds = self.config.getint( - tautulli_section, 'get_activity_run_seconds') + get_activity_run_seconds = self.config.getint(section, 'get_activity_run_seconds') server = TautulliServer(server_id, scheme + url, fallback_ip, apikey, verify_ssl, get_activity, get_activity_run_seconds) @@ -232,39 +220,33 @@ class INIParser(object): except configparser.NoOptionError as e: self.tautulli_enabled = False self.logger.error( - '%s disabled. Error: %s', tautulli_section, e) + '%s disabled. Error: %s', section, e) # Parse Ombi options self.ombi_enabled = self.enable_check('ombi_server_ids') if self.ombi_enabled: for server_id in self.ombi_enabled: - ombi_section = 'ombi-' + str(server_id) + section = 'ombi-' + str(server_id) try: - url = self.url_check(self.config.get(ombi_section, 'url')) + url = self.url_check(self.config.get(section, 'url')) - apikey = self.config.get(ombi_section, 'apikey') + apikey = self.config.get(section, 'apikey') - scheme = 'https://' if self.config.getboolean( - ombi_section, 'ssl') else 'http://' + scheme = 'https://' if self.config.getboolean(section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean( - ombi_section, 'verify_ssl') + verify_ssl = self.config.getboolean(section, 'verify_ssl') if scheme != 'https://': verify_ssl = False - request_type_counts = self.config.getboolean( - ombi_section, 'get_request_type_counts') + request_type_counts = self.config.getboolean(section, 'get_request_type_counts') - request_type_run_seconds = self.config.getint( - ombi_section, 'request_type_run_seconds') + request_type_run_seconds = self.config.getint(section, 'request_type_run_seconds') - request_total_counts = self.config.getboolean( - ombi_section, 'get_request_total_counts') + request_total_counts = self.config.getboolean(section, 'get_request_total_counts') - request_total_run_seconds = self.config.getint( - ombi_section, 'request_total_run_seconds') + request_total_run_seconds = self.config.getint(section, 'request_total_run_seconds') server = OmbiServer(server_id, scheme + url, apikey, verify_ssl, request_type_counts, request_type_run_seconds, request_total_counts, request_total_run_seconds) @@ -272,35 +254,61 @@ class INIParser(object): except configparser.NoOptionError as e: self.ombi_enabled = False self.logger.error( - '%s disabled. Error: %s', ombi_section, e) + '%s disabled. Error: %s', section, e) + + # Parse SickChill options + self.sickchill_enabled = self.enable_check('sickchill_server_ids') + + if self.sickchill_enabled: + for server_id in self.sickchill_enabled: + section = 'sickchill-' + str(server_id) + try: + url = self.url_check(self.config.get(section, 'url')) + + apikey = self.config.get(section, 'apikey') + + scheme = 'https://' if self.config.getboolean(section, 'ssl') else 'http://' + + verify_ssl = self.config.getboolean(section, 'verify_ssl') + + if scheme != 'https://': + verify_ssl = False + + get_missing = self.config.getboolean(section, 'get_missing') + + get_missing_run_seconds = self.config.getint(section, 'get_missing_run_seconds') + + server = SickChillServer(server_id, scheme + url, apikey, verify_ssl, + get_missing, get_missing_run_seconds) + self.sickchill_servers.append(server) + except configparser.NoOptionError as e: + self.sickchill_enabled = False + self.logger.error( + '%s disabled. Error: %s', section, e) # Parse ASA opts self.ciscoasa_enabled = self.enable_check('ciscoasa_firewall_ids') if self.ciscoasa_enabled: for firewall_id in self.ciscoasa_enabled: - ciscoasa_section = 'ciscoasa-' + str(firewall_id) + section = 'ciscoasa-' + str(firewall_id) try: - url = self.url_check(self.config.get(ciscoasa_section, 'url')) + url = self.url_check(self.config.get(section, 'url')) - username = self.config.get(ciscoasa_section, 'username') + username = self.config.get(section, 'username') - password = self.config.get(ciscoasa_section, 'password') + password = self.config.get(section, 'password') - scheme = 'https://' if self.config.getboolean( - ciscoasa_section, 'ssl') else 'http://' + scheme = 'https://' if self.config.getboolean(section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean( - ciscoasa_section, 'verify_ssl') + verify_ssl = self.config.getboolean(section, 'verify_ssl') if scheme != 'https://': verify_ssl = False - outside_interface = self.config.get( - ciscoasa_section, 'outside_interface') + outside_interface = self.config.get(section, 'outside_interface') - get_bandwidth_run_seconds = self.config.getint( - ciscoasa_section, 'get_bandwidth_run_seconds') + get_bandwidth_run_seconds = self.config.getint(section, 'get_bandwidth_run_seconds') firewall = CiscoASAFirewall(firewall_id, scheme + url, username, password, outside_interface, verify_ssl, get_bandwidth_run_seconds) @@ -308,4 +316,4 @@ class INIParser(object): except configparser.NoOptionError as e: self.ciscoasa_enabled = False self.logger.error( - '%s disabled. Error: %s', ciscoasa_section, e) + '%s disabled. Error: %s', section, e) diff --git a/varken/sickchill.py b/varken/sickchill.py new file mode 100644 index 0000000..3af3684 --- /dev/null +++ b/varken/sickchill.py @@ -0,0 +1,64 @@ +import logging +from requests import Session, Request +from datetime import datetime, timezone, date, timedelta + +from varken.helpers import hashit, connection_handler +from varken.structures import SickChillTVShow + + +class SickChillAPI(object): + def __init__(self, server, dbmanager): + self.dbmanager = dbmanager + self.server = server + # Create session to reduce server web thread load, and globally define pageSize for all requests + self.session = Session() + self.session.params = {'limit': 1000} + self.endpoint = f"/api/{self.server.api_key}" + self.logger = logging.getLogger() + + def __repr__(self): + return "".format(self.server.id) + + def get_missing(self): + now = datetime.now(timezone.utc).astimezone().isoformat() + influx_payload = [] + params = {'cmd': 'future', 'paused': 1, 'type': 'missed|today|soon|later|snatched'} + + req = self.session.prepare_request(Request('GET', self.server.url + self.endpoint, params=params)) + get = connection_handler(self.session, req, self.server.verify_ssl) + + if not get: + return + + try: + for key, section in get['data'].items(): + get['data'][key] = [SickChillTVShow(**show) for show in section] + except TypeError as e: + self.logger.error('TypeError has occurred : %s while creating SickChillTVShow structure', e) + return + + for key, section in get['data'].items(): + for show in section: + sxe = 'S{:0>2}E{:0>2}'.format(show.season, show.episode) + hash_id = hashit('{}{}{}'.format(self.server.id, show.show_name, sxe)) + missing_types = [(0, 'future'), (1, 'later'), (2, 'soon'), (3, 'today'), (4, 'missed')] + influx_payload.append( + { + "measurement": "SickChill", + "tags": { + "type": [item[0] for item in missing_types if key in item][0], + "indexerid": show.indexerid, + "server": self.server.id, + "name": show.show_name, + "epname": show.ep_name, + "sxe": sxe, + "airdate": show.airdate, + }, + "time": now, + "fields": { + "hash": hash_id + } + } + ) + + self.dbmanager.write_points(influx_payload) diff --git a/varken/structures.py b/varken/structures.py index 3e1a583..6140770 100644 --- a/varken/structures.py +++ b/varken/structures.py @@ -70,6 +70,16 @@ class InfluxServer(NamedTuple): username: str = 'root' password: str = 'root' + +class SickChillServer(NamedTuple): + id: int = None + url: str = None + api_key: str = None + verify_ssl: bool = False + get_missing: bool = False + get_missing_run_seconds: int = 30 + + class CiscoASAFirewall(NamedTuple): id: int = None url: str = '192.168.1.1' @@ -384,3 +394,19 @@ class OmbiTVRequest(NamedTuple): totalSeasons: int = None childRequests: list = None id: int = None + +class SickChillTVShow(NamedTuple): + airdate: str = None + airs: str = None + ep_name: str = None + ep_plot: str = None + episode: int = None + indexerid: int = None + network: str = None + paused: int = None + quality: str = None + season: int = None + show_name: str = None + show_status: str = None + tvdbid: int = None + weekday: int = None \ No newline at end of file From 73e203d6ca6d9fb530494dedae96ddaaae75b416 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Sun, 16 Dec 2018 00:59:58 -0600 Subject: [PATCH 60/76] Changelog + version bump --- CHANGELOG.md | 8 ++++++++ varken/__init__.py | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61c3fe2..ff3fafd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Change Log +## [v1.2-nightly](https://github.com/Boerderij/Varken/tree/v1.2-nightly) (2018-12-16) +[Full Changelog](https://github.com/Boerderij/Varken/compare/v1.1...v1.2-nightly) + +**Implemented enhancements:** + +- \[Feature Request\]: Pull list of requests \(instead of just counts\) [\#58](https://github.com/Boerderij/Varken/issues/58) +- Feature Request , Add Sickchill [\#48](https://github.com/Boerderij/Varken/issues/48) + ## [v1.1](https://github.com/Boerderij/Varken/tree/v1.1) (2018-12-11) [Full Changelog](https://github.com/Boerderij/Varken/compare/v1.0...v1.1) diff --git a/varken/__init__.py b/varken/__init__.py index c3ef0d4..b5a808d 100644 --- a/varken/__init__.py +++ b/varken/__init__.py @@ -1 +1,2 @@ -VERSION = 1.1 +VERSION = 1.2 +BRANCH = 'nightly' From 371d53d99d2cb4db3283cba10af0b493657f4b86 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Sun, 16 Dec 2018 09:39:33 -0500 Subject: [PATCH 61/76] Add Sickchill to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e91a4bd..8017b17 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Example Dashboard Supported Modules: * [Sonarr](https://sonarr.tv/) - Smart PVR for newsgroup and bittorrent users. +* [SickChill](https://sickchill.github.io/) - SickChill is an automatic Video Library Manager for TV Shows. * [Radarr](https://radarr.video/) - A fork of Sonarr to work with movies à la Couchpotato. * [Tautulli](https://tautulli.com/) - A Python based monitoring and tracking tool for Plex Media Server. * [Ombi](https://ombi.io/) - Want a Movie or TV Show on Plex or Emby? Use Ombi! From ecc2a8abdf224498d9ebe7233b46c7d777e0b7e8 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 17 Dec 2018 13:28:14 -0500 Subject: [PATCH 62/76] Remove some misc logging config --- varken/helpers.py | 2 +- varken/varkenlogger.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index 6c3e6fa..5fdf8fe 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -11,7 +11,7 @@ from os.path import abspath, join from requests.exceptions import InvalidSchema, SSLError, ConnectionError from urllib.request import urlretrieve -logger = logging.getLogger('varken') +logger = logging.getLogger() def geoip_download(data_folder): diff --git a/varken/varkenlogger.py b/varken/varkenlogger.py index 3890a14..cd3e0d8 100644 --- a/varken/varkenlogger.py +++ b/varken/varkenlogger.py @@ -36,7 +36,7 @@ class BlacklistFilter(logging.Filter): class VarkenLogger(object): """docstring for .""" - def __init__(self, log_path=None, debug=None, data_folder=None): + def __init__(self, debug=None, data_folder=None): self.data_folder = data_folder self.log_level = debug From 47f2f6108266fc15116d46a0682839e95fd0b6fb Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 17 Dec 2018 18:00:01 -0600 Subject: [PATCH 63/76] QOL and PEP8 Cleanup for logger --- README.md | 4 ++-- varken/varkenlogger.py | 32 ++++++++++++++------------------ 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 8017b17..e4ccef7 100644 --- a/README.md +++ b/README.md @@ -64,8 +64,8 @@ systemctl enable varken ``` ### Docker -[![Docker-Layers](https://images.microbadger.com/badges/image/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken") -[![Docker-Version](https://images.microbadger.com/badges/version/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken") +[![Docker-Layers](https://images.microbadger.com/badges/image/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken) +[![Docker-Version](https://images.microbadger.com/badges/version/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken) [![Docker Pulls](https://img.shields.io/docker/pulls/boerderij/varken.svg)](https://hub.docker.com/r/boerderij/varken/) [![Docker Stars](https://img.shields.io/docker/stars/boerderij/varken.svg)](https://hub.docker.com/r/boerderij/varken/)
Example diff --git a/varken/varkenlogger.py b/varken/varkenlogger.py index cd3e0d8..acdb4ec 100644 --- a/varken/varkenlogger.py +++ b/varken/varkenlogger.py @@ -4,20 +4,19 @@ from logging.handlers import RotatingFileHandler from varken.helpers import mkdir_p -FILENAME = "varken.log" -MAX_SIZE = 5000000 # 5 MB -MAX_FILES = 5 -LOG_FOLDER = 'logs' - - -# Taken from Hellowlol/HTPC-Manager/Tautulli class BlacklistFilter(logging.Filter): """ Log filter for blacklisted tokens and passwords """ + filename = "varken.log" + max_size = 5000000 # 5 MB + max_files = 5 + log_folder = 'logs' + blacklisted_strings = ['apikey', 'username', 'password'] def __init__(self, filteredstrings): + super().__init__() self.filtered_strings = filteredstrings def filter(self, record): @@ -28,14 +27,12 @@ class BlacklistFilter(logging.Filter): if any(item in str(arg) for arg in record.args): record.args = tuple(arg.replace(item, 8 * '*' + item[-2:]) if isinstance(arg, str) else arg for arg in record.args) - - except: + except TypeError: pass return True class VarkenLogger(object): - """docstring for .""" def __init__(self, debug=None, data_folder=None): self.data_folder = data_folder self.log_level = debug @@ -47,23 +44,22 @@ class VarkenLogger(object): else: self.log_level = logging.INFO - # Make the log directory if it does not exist - mkdir_p('{}/{}'.format(self.data_folder, LOG_FOLDER)) + mkdir_p('{}/{}'.format(self.data_folder, BlacklistFilter.log_folder)) # Create the Logger self.logger = logging.getLogger() self.logger.setLevel(logging.DEBUG) # Create a Formatter for formatting the log messages - logger_formatter = logging.Formatter('%(asctime)s : %(levelname)s : %(module)s : %(message)s', '%Y-%m-%d %H:%M:%S') + logger_formatter = logging.Formatter('%(asctime)s : %(levelname)s : %(module)s : %(message)s', + '%Y-%m-%d %H:%M:%S') # Create the Handler for logging data to a file - file_logger = RotatingFileHandler('{}/{}/{}'.format(self.data_folder, LOG_FOLDER, FILENAME), - mode='a', maxBytes=MAX_SIZE, - backupCount=MAX_FILES, - encoding=None, delay=0 - ) + file_logger = RotatingFileHandler('{}/{}/{}'.format(self.data_folder, BlacklistFilter.log_folder, + BlacklistFilter.filename), mode='a', + maxBytes=BlacklistFilter.max_size, backupCount=BlacklistFilter.max_files, + encoding=None, delay=0) file_logger.setLevel(self.log_level) From 61fcf3b80c3bd5d867624b9f3da8007cd7449ece Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 17 Dec 2018 19:12:37 -0600 Subject: [PATCH 64/76] major pep8 cleanup. funneled scope of imports where possible fixed #59 Added fstrings where possible added logging both temporarily and in secondary places --- .gitignore | 1 - Varken.py | 63 ++++++++++++++++++++++-------------------- varken/cisco.py | 6 ++-- varken/dbmanager.py | 10 ++++--- varken/helpers.py | 8 +++--- varken/iniparser.py | 34 +++++++++++------------ varken/ombi.py | 13 +++++---- varken/radarr.py | 16 +++++------ varken/sickchill.py | 14 +++++----- varken/sonarr.py | 20 +++++++------- varken/structures.py | 15 +++++++++- varken/tautulli.py | 20 ++++++-------- varken/varkenlogger.py | 32 ++++++++++----------- 13 files changed, 131 insertions(+), 121 deletions(-) diff --git a/.gitignore b/.gitignore index cc45351..6b64cc4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,5 @@ GeoLite2-City.mmdb GeoLite2-City.tar.gz data/varken.ini .idea/ -Legacy/configuration.py varken-venv/ logs/ diff --git a/Varken.py b/Varken.py index 8734e12..b669c5b 100644 --- a/Varken.py +++ b/Varken.py @@ -1,36 +1,30 @@ -import sys - -# Check for python3.6 or newer to resolve erroneous typing.NamedTuple issues -if sys.version_info < (3, 6): - exit('Varken requires python3.6 or newer') - -import schedule -import threading import platform -import distro -import os +import schedule -from sys import exit from time import sleep -from os import access, R_OK +from sys import version +from threading import Thread +from os import access, R_OK, getenv +from distro import linux_distribution from os.path import isdir, abspath, dirname, join from argparse import ArgumentParser, RawTextHelpFormatter +from logging import getLogger, StreamHandler, Formatter, DEBUG -from varken.iniparser import INIParser -from varken.sonarr import SonarrAPI -from varken.tautulli import TautulliAPI -from varken.radarr import RadarrAPI from varken.ombi import OmbiAPI -from varken.sickchill import SickChillAPI from varken.cisco import CiscoAPI +from varken.sonarr import SonarrAPI +from varken.radarr import RadarrAPI +from varken.iniparser import INIParser from varken.dbmanager import DBManager +from varken.tautulli import TautulliAPI +from varken.sickchill import SickChillAPI from varken.varkenlogger import VarkenLogger -PLATFORM_LINUX_DISTRO = ' '.join(x for x in distro.linux_distribution() if x) +PLATFORM_LINUX_DISTRO = ' '.join(x for x in linux_distribution() if x) def threaded(job): - thread = threading.Thread(target=job) + thread = Thread(target=job) thread.start() @@ -46,23 +40,32 @@ if __name__ == "__main__": DATA_FOLDER = abspath(join(dirname(__file__), 'data')) + templogger = getLogger('temp') + templogger.setLevel(DEBUG) + tempch = StreamHandler() + tempformatter = Formatter('%(asctime)s : %(levelname)s : %(module)s : %(message)s', '%Y-%m-%d %H:%M:%S') + tempch.setFormatter(tempformatter) + templogger.addHandler(tempch) + if opts.data_folder: ARG_FOLDER = opts.data_folder if isdir(ARG_FOLDER): DATA_FOLDER = ARG_FOLDER - if not access(ARG_FOLDER, R_OK): - exit("Read permission error for {}".format(ARG_FOLDER)) + if not access(DATA_FOLDER, R_OK): + templogger.error("Read permission error for %s", DATA_FOLDER) + exit(1) else: - exit("{} does not exist".format(ARG_FOLDER)) + templogger.error("%s does not exist", ARG_FOLDER) + exit(1) # Set Debug to True if DEBUG env is set enable_opts = ['True', 'true', 'yes'] debug_opts = ['debug', 'Debug', 'DEBUG'] if not opts.debug: - opts.debug = True if any([os.getenv(string, False) for true in enable_opts - for string in debug_opts if os.getenv(string, False) == true]) else False + opts.debug = True if any([getenv(string, False) for true in enable_opts + for string in debug_opts if getenv(string, False) == true]) else False # Initiate the logger vl = VarkenLogger(data_folder=DATA_FOLDER, debug=opts.debug) @@ -70,11 +73,10 @@ if __name__ == "__main__": vl.logger.info('Data folder is "%s"', DATA_FOLDER) - vl.logger.info(u"{} {} ({}{})".format( - platform.system(), platform.release(), platform.version(), - ' - {}'.format(PLATFORM_LINUX_DISTRO) if PLATFORM_LINUX_DISTRO else '' - )) - vl.logger.info(u"Python {}".format(sys.version)) + vl.logger.info(u"%s %s (%s%s)", platform.system(), platform.release(), platform.version(), + f' - {PLATFORM_LINUX_DISTRO}' if PLATFORM_LINUX_DISTRO else '') + + vl.logger.info(u"Python %s", version) CONFIG = INIParser(DATA_FOLDER) DBMANAGER = DBManager(CONFIG.influx_server) @@ -126,7 +128,8 @@ if __name__ == "__main__": SERVICES_ENABLED = [CONFIG.ombi_enabled, CONFIG.radarr_enabled, CONFIG.tautulli_enabled, CONFIG.sonarr_enabled, CONFIG.ciscoasa_enabled, CONFIG.sickchill_enabled] if not [enabled for enabled in SERVICES_ENABLED if enabled]: - exit("All services disabled. Exiting") + vl.logger.error("All services disabled. Exiting") + exit(1) schedule.run_all() while True: diff --git a/varken/cisco.py b/varken/cisco.py index 750ad89..ac04166 100644 --- a/varken/cisco.py +++ b/varken/cisco.py @@ -1,4 +1,4 @@ -import logging +from logging import getLogger from requests import Session, Request from datetime import datetime, timezone @@ -13,12 +13,12 @@ class CiscoAPI(object): # Create session to reduce server web thread load, and globally define pageSize for all requests self.session = Session() self.session.auth = (self.firewall.username, self.firewall.password) - self.logger = logging.getLogger() + self.logger = getLogger() self.get_token() def __repr__(self): - return "".format(self.firewall.id) + return f"" def get_token(self): endpoint = '/api/tokenservices' diff --git a/varken/dbmanager.py b/varken/dbmanager.py index ff531ec..9035db2 100644 --- a/varken/dbmanager.py +++ b/varken/dbmanager.py @@ -1,8 +1,6 @@ -import logging - +from logging import getLogger from influxdb import InfluxDBClient -logger = logging.getLogger('varken') class DBManager(object): def __init__(self, server): @@ -10,12 +8,16 @@ class DBManager(object): self.influx = InfluxDBClient(self.server.url, self.server.port, self.server.username, self.server.password, 'varken') databases = [db['name'] for db in self.influx.get_list_database()] + self.logger = getLogger() if 'varken' not in databases: + self.logger.info("Creating varken database") self.influx.create_database('varken') + + self.logger.info("Creating varken retention policy (30d/1h)") self.influx.create_retention_policy('varken 30d/1h', '30d', '1', 'varken', False, '1h') def write_points(self, data): d = data - logger.debug('Writing Data to InfluxDB %s', d) + self.logger.debug('Writing Data to InfluxDB %s', d) self.influx.write_points(d) diff --git a/varken/helpers.py b/varken/helpers.py index 5fdf8fe..aaa8ccd 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -106,12 +106,12 @@ def connection_handler(session, request, verify): def mkdir_p(path): - """http://stackoverflow.com/a/600612/190597 (tzot)""" + templogger = logging.getLogger('temp') try: - logger.info('Creating folder %s ', path) + templogger.info('Creating folder %s ', path) os.makedirs(path, exist_ok=True) except Exception as e: - logger.error('Could not create folder %s : %s ', path, e) + templogger.error('Could not create folder %s : %s ', path, e) def clean_sid_check(server_id_list, server_type=None): @@ -123,7 +123,7 @@ def clean_sid_check(server_id_list, server_type=None): try: valid_sids.append(int(sid)) except ValueError: - logger.error("{} is not a valid server id number".format(sid)) + logger.error("%s is not a valid server id number", sid) if valid_sids: logger.info('%s : %s', t.upper(), valid_sids) return valid_sids diff --git a/varken/iniparser.py b/varken/iniparser.py index 846de52..24ea0b1 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -1,22 +1,20 @@ -import configparser -import logging -import re - -from sys import exit +from logging import getLogger from os.path import join, exists +from re import match, compile, IGNORECASE +from configparser import ConfigParser, NoOptionError from varken.helpers import clean_sid_check +from varken.structures import SickChillServer from varken.varkenlogger import BlacklistFilter from varken.structures import SonarrServer, RadarrServer, OmbiServer, TautulliServer, InfluxServer, CiscoASAFirewall -from varken.structures import SickChillServer class INIParser(object): def __init__(self, data_folder): - self.config = configparser.ConfigParser(interpolation=None) + self.config = ConfigParser(interpolation=None) self.data_folder = data_folder - self.logger = logging.getLogger() + self.logger = getLogger() self.influx_server = InfluxServer() @@ -59,7 +57,7 @@ class INIParser(object): else: sids = clean_sid_check(global_server_ids, t) return sids - except configparser.NoOptionError as e: + except NoOptionError as e: self.logger.error(e) def read_file(self): @@ -86,11 +84,11 @@ class INIParser(object): else: search = (search + r'(?:/?|[/?]\S+)$') - regex = re.compile('{}'.format(search), re.IGNORECASE) + regex = compile('{}'.format(search), IGNORECASE) - print(re.match(regex, url_check)) + print(match(regex, url_check)) - valid = re.match(regex, url_check) is not None + valid = match(regex, url_check) is not None if not valid: if inc_port: self.logger.error('%s is invalid! URL must host/IP and port if not 80 or 443. ie. localhost:8080', @@ -151,7 +149,7 @@ class INIParser(object): queue, queue_run_seconds) self.sonarr_servers.append(server) - except configparser.NoOptionError as e: + except NoOptionError as e: self.sonarr_enabled = False self.logger.error( '%s disabled. Error: %s', section, e) @@ -185,7 +183,7 @@ class INIParser(object): server = RadarrServer(server_id, scheme + url, apikey, verify_ssl, queue, queue_run_seconds, get_missing, get_missing_run_seconds) self.radarr_servers.append(server) - except configparser.NoOptionError as e: + except NoOptionError as e: self.radarr_enabled = False self.logger.error( '%s disabled. Error: %s', section, e) @@ -217,7 +215,7 @@ class INIParser(object): server = TautulliServer(server_id, scheme + url, fallback_ip, apikey, verify_ssl, get_activity, get_activity_run_seconds) self.tautulli_servers.append(server) - except configparser.NoOptionError as e: + except NoOptionError as e: self.tautulli_enabled = False self.logger.error( '%s disabled. Error: %s', section, e) @@ -251,7 +249,7 @@ class INIParser(object): server = OmbiServer(server_id, scheme + url, apikey, verify_ssl, request_type_counts, request_type_run_seconds, request_total_counts, request_total_run_seconds) self.ombi_servers.append(server) - except configparser.NoOptionError as e: + except NoOptionError as e: self.ombi_enabled = False self.logger.error( '%s disabled. Error: %s', section, e) @@ -281,7 +279,7 @@ class INIParser(object): server = SickChillServer(server_id, scheme + url, apikey, verify_ssl, get_missing, get_missing_run_seconds) self.sickchill_servers.append(server) - except configparser.NoOptionError as e: + except NoOptionError as e: self.sickchill_enabled = False self.logger.error( '%s disabled. Error: %s', section, e) @@ -313,7 +311,7 @@ class INIParser(object): firewall = CiscoASAFirewall(firewall_id, scheme + url, username, password, outside_interface, verify_ssl, get_bandwidth_run_seconds) self.ciscoasa_firewalls.append(firewall) - except configparser.NoOptionError as e: + except NoOptionError as e: self.ciscoasa_enabled = False self.logger.error( '%s disabled. Error: %s', section, e) diff --git a/varken/ombi.py b/varken/ombi.py index 3b8d56b..e0a13bb 100644 --- a/varken/ombi.py +++ b/varken/ombi.py @@ -1,4 +1,4 @@ -import logging +from logging import getLogger from requests import Session, Request from datetime import datetime, timezone @@ -13,10 +13,10 @@ class OmbiAPI(object): # Create session to reduce server web thread load, and globally define pageSize for all requests self.session = Session() self.session.headers = {'Apikey': self.server.api_key} - self.logger = logging.getLogger() + self.logger = getLogger() def __repr__(self): - return "".format(self.server.id) + return f"" def get_all_requests(self): now = datetime.now(timezone.utc).astimezone().isoformat() @@ -94,14 +94,16 @@ class OmbiAPI(object): for show in tv_show_requests: hash_id = hashit(f'{show.id}{show.tvDbId}{show.title}') - status = None - # Denied = 0, Approved = 1, Completed = 2 + + # Denied = 0, Approved = 1, Completed = 2, Pending = 3 if show.childRequests[0]['denied']: status = 0 elif show.childRequests[0]['approved'] and show.childRequests[0]['available']: status = 2 elif show.childRequests[0]['approved']: status = 1 + else: + status = 3 influx_payload.append( { @@ -122,7 +124,6 @@ class OmbiAPI(object): } ) - self.dbmanager.write_points(influx_payload) def get_request_counts(self): diff --git a/varken/radarr.py b/varken/radarr.py index 1b23923..d5ad514 100644 --- a/varken/radarr.py +++ b/varken/radarr.py @@ -1,9 +1,9 @@ -import logging +from logging import getLogger from requests import Session, Request from datetime import datetime, timezone -from varken.helpers import hashit, connection_handler from varken.structures import Movie, Queue +from varken.helpers import hashit, connection_handler class RadarrAPI(object): @@ -13,10 +13,10 @@ class RadarrAPI(object): # Create session to reduce server web thread load, and globally define pageSize for all requests self.session = Session() self.session.headers = {'X-Api-Key': self.server.api_key} - self.logger = logging.getLogger() + self.logger = getLogger() def __repr__(self): - return "".format(self.server.id) + return f"" def get_missing(self): endpoint = '/api/movie' @@ -43,11 +43,11 @@ class RadarrAPI(object): else: ma = 1 - movie_name = '{} ({})'.format(movie.title, movie.year) + movie_name = f'{movie.title} ({movie.year})' missing.append((movie_name, ma, movie.tmdbId, movie.titleSlug)) for title, ma, mid, title_slug in missing: - hash_id = hashit('{}{}{}'.format(self.server.id, title, mid)) + hash_id = hashit(f'{self.server.id}{title}{mid}') influx_payload.append( { "measurement": "Radarr", @@ -96,7 +96,7 @@ class RadarrAPI(object): for queue_item in download_queue: movie = queue_item.movie - name = '{} ({})'.format(movie.title, movie.year) + name = f'{movie.title} ({movie.year})' if queue_item.protocol.upper() == 'USENET': protocol_id = 1 @@ -107,7 +107,7 @@ class RadarrAPI(object): protocol_id, queue_item.id, movie.titleSlug)) for name, quality, protocol, protocol_id, qid, title_slug in queue: - hash_id = hashit('{}{}{}'.format(self.server.id, name, quality)) + hash_id = hashit(f'{self.server.id}{name}{quality}') influx_payload.append( { "measurement": "Radarr", diff --git a/varken/sickchill.py b/varken/sickchill.py index 3af3684..7be6acf 100644 --- a/varken/sickchill.py +++ b/varken/sickchill.py @@ -1,9 +1,9 @@ -import logging +from logging import getLogger from requests import Session, Request -from datetime import datetime, timezone, date, timedelta +from datetime import datetime, timezone -from varken.helpers import hashit, connection_handler from varken.structures import SickChillTVShow +from varken.helpers import hashit, connection_handler class SickChillAPI(object): @@ -14,10 +14,10 @@ class SickChillAPI(object): self.session = Session() self.session.params = {'limit': 1000} self.endpoint = f"/api/{self.server.api_key}" - self.logger = logging.getLogger() + self.logger = getLogger() def __repr__(self): - return "".format(self.server.id) + return f"" def get_missing(self): now = datetime.now(timezone.utc).astimezone().isoformat() @@ -39,8 +39,8 @@ class SickChillAPI(object): for key, section in get['data'].items(): for show in section: - sxe = 'S{:0>2}E{:0>2}'.format(show.season, show.episode) - hash_id = hashit('{}{}{}'.format(self.server.id, show.show_name, sxe)) + sxe = f'S{show.season:0>2}E{show.episode:0>2}' + hash_id = hashit(f'{self.server.id}{show.show_name}{sxe}') missing_types = [(0, 'future'), (1, 'later'), (2, 'soon'), (3, 'today'), (4, 'missed')] influx_payload.append( { diff --git a/varken/sonarr.py b/varken/sonarr.py index 54f027d..19a36c9 100644 --- a/varken/sonarr.py +++ b/varken/sonarr.py @@ -1,9 +1,9 @@ -import logging +from logging import getLogger from requests import Session, Request from datetime import datetime, timezone, date, timedelta -from varken.helpers import hashit, connection_handler from varken.structures import Queue, TVShow +from varken.helpers import hashit, connection_handler class SonarrAPI(object): @@ -14,10 +14,10 @@ class SonarrAPI(object): self.session = Session() self.session.headers = {'X-Api-Key': self.server.api_key} self.session.params = {'pageSize': 1000} - self.logger = logging.getLogger() + self.logger = getLogger() def __repr__(self): - return "".format(self.server.id) + return f"" def get_missing(self): endpoint = '/api/calendar' @@ -44,11 +44,11 @@ class SonarrAPI(object): # Add show to missing list if file does not exist for show in tv_shows: if not show.hasFile: - sxe = 'S{:0>2}E{:0>2}'.format(show.seasonNumber, show.episodeNumber) + sxe = f'S{show.seasonNumber:0>2}E{show.episodeNumber:0>2}' missing.append((show.series['title'], sxe, show.airDateUtc, show.title, show.id)) for series_title, sxe, air_date_utc, episode_title, sonarr_id in missing: - hash_id = hashit('{}{}{}'.format(self.server.id, series_title, sxe)) + hash_id = hashit(f'{self.server.id}{series_title}{sxe}') influx_payload.append( { "measurement": "Sonarr", @@ -93,7 +93,7 @@ class SonarrAPI(object): return for show in tv_shows: - sxe = 'S{:0>2}E{:0>2}'.format(show.seasonNumber, show.episodeNumber) + sxe = f'S{show.seasonNumber:0>2}E{show.episodeNumber:0>2}' if show.hasFile: downloaded = 1 else: @@ -101,7 +101,7 @@ class SonarrAPI(object): air_days.append((show.series['title'], downloaded, sxe, show.title, show.airDateUtc, show.id)) for series_title, dl_status, sxe, episode_title, air_date_utc, sonarr_id in air_days: - hash_id = hashit('{}{}{}'.format(self.server.id, series_title, sxe)) + hash_id = hashit(f'{self.server.id}{series_title}{sxe}') influx_payload.append( { "measurement": "Sonarr", @@ -143,7 +143,7 @@ class SonarrAPI(object): return for show in download_queue: - sxe = 'S{:0>2}E{:0>2}'.format(show.episode['seasonNumber'], show.episode['episodeNumber']) + sxe = f"S{show.episode['seasonNumber']:0>2}E{show.episode['episodeNumber']:0>2}" if show.protocol.upper() == 'USENET': protocol_id = 1 else: @@ -153,7 +153,7 @@ class SonarrAPI(object): protocol_id, sxe, show.id)) for series_title, episode_title, protocol, protocol_id, sxe, sonarr_id in queue: - hash_id = hashit('{}{}{}'.format(self.server.id, series_title, sxe)) + hash_id = hashit(f'{self.server.id}{series_title}{sxe}') influx_payload.append( { "measurement": "Sonarr", diff --git a/varken/structures.py b/varken/structures.py index 6140770..1c9b8a7 100644 --- a/varken/structures.py +++ b/varken/structures.py @@ -1,4 +1,13 @@ +from sys import version_info from typing import NamedTuple +from logging import getLogger + +logger = getLogger('temp') +# Check for python3.6 or newer to resolve erroneous typing.NamedTuple issues +if version_info < (3, 6): + logger.error('Varken requires python3.6 or newer. You are on python%s.%s - Exiting...', + version_info.major, version_info.minor) + exit(1) class Queue(NamedTuple): @@ -89,6 +98,7 @@ class CiscoASAFirewall(NamedTuple): verify_ssl: bool = False get_bandwidth_run_seconds: int = 30 + class OmbiRequestCounts(NamedTuple): pending: int = 0 approved: int = 0 @@ -348,6 +358,7 @@ class Movie(NamedTuple): website: str = None id: int = None + class OmbiMovieRequest(NamedTuple): theMovieDbId: int = None issueId: None = None @@ -380,6 +391,7 @@ class OmbiMovieRequest(NamedTuple): canApprove: bool = None id: int = None + class OmbiTVRequest(NamedTuple): tvDbId: int = None imdbId: str = None @@ -395,6 +407,7 @@ class OmbiTVRequest(NamedTuple): childRequests: list = None id: int = None + class SickChillTVShow(NamedTuple): airdate: str = None airs: str = None @@ -409,4 +422,4 @@ class SickChillTVShow(NamedTuple): show_name: str = None show_status: str = None tvdbid: int = None - weekday: int = None \ No newline at end of file + weekday: int = None diff --git a/varken/tautulli.py b/varken/tautulli.py index b2cd549..f71ce81 100644 --- a/varken/tautulli.py +++ b/varken/tautulli.py @@ -1,12 +1,10 @@ -import os -import logging - +from logging import getLogger from requests import Session, Request from datetime import datetime, timezone from geoip2.errors import AddressNotFoundError -from varken.helpers import geo_lookup, hashit, connection_handler from varken.structures import TautulliStream +from varken.helpers import geo_lookup, hashit, connection_handler class TautulliAPI(object): @@ -16,11 +14,11 @@ class TautulliAPI(object): self.session = Session() self.session.params = {'apikey': self.server.api_key, 'cmd': 'get_activity'} self.endpoint = '/api/v2' - self.logger = logging.getLogger() + self.logger = getLogger() self.data_folder = data_folder def __repr__(self): - return "".format(self.server.id) + return f"" def get_activity(self): now = datetime.now(timezone.utc).astimezone().isoformat() @@ -41,12 +39,12 @@ class TautulliAPI(object): return for session in sessions: - # Check to see if ip_address_public atribute exists as it was introduced in v2 + # Check to see if ip_address_public attribute exists as it was introduced in v2 try: getattr(session, 'ip_address_public') except AttributeError: self.logger.error('Public IP attribute missing!!! Do you have an old version of Tautulli (v1)?') - os._exit(1) + exit(1) try: geodata = geo_lookup(session.ip_address_public, self.data_folder) @@ -94,8 +92,7 @@ class TautulliAPI(object): if session.platform == 'Roku': product_version = session.product_version.split('-')[0] - hash_id = hashit('{}{}{}{}'.format(session.session_id, session.session_key, session.username, - session.full_title)) + hash_id = hashit(f'{session.session_id}{session.session_key}{session.username}{session.full_title}') influx_payload.append( { "measurement": "Tautulli", @@ -118,8 +115,7 @@ class TautulliAPI(object): "progress_percent": session.progress_percent, "region_code": geodata.subdivisions.most_specific.iso_code, "location": geodata.city.name, - "full_location": '{} - {}'.format(geodata.subdivisions.most_specific.name, - geodata.city.name), + "full_location": f'{geodata.subdivisions.most_specific.name} - {geodata.city.name}', "latitude": latitude, "longitude": longitude, "player_state": player_state, diff --git a/varken/varkenlogger.py b/varken/varkenlogger.py index acdb4ec..6816af4 100644 --- a/varken/varkenlogger.py +++ b/varken/varkenlogger.py @@ -1,10 +1,10 @@ -import logging - from logging.handlers import RotatingFileHandler +from logging import Filter, DEBUG, INFO, getLogger, Formatter, StreamHandler + from varken.helpers import mkdir_p -class BlacklistFilter(logging.Filter): +class BlacklistFilter(Filter): """ Log filter for blacklisted tokens and passwords """ @@ -13,7 +13,7 @@ class BlacklistFilter(logging.Filter): max_files = 5 log_folder = 'logs' - blacklisted_strings = ['apikey', 'username', 'password'] + blacklisted_strings = ['apikey', 'username', 'password', 'url'] def __init__(self, filteredstrings): super().__init__() @@ -23,9 +23,9 @@ class BlacklistFilter(logging.Filter): for item in self.filtered_strings: try: if item in record.msg: - record.msg = record.msg.replace(item, 8 * '*' + item[-2:]) + record.msg = record.msg.replace(item, 8 * '*' + item[-5:]) if any(item in str(arg) for arg in record.args): - record.args = tuple(arg.replace(item, 8 * '*' + item[-2:]) if isinstance(arg, str) else arg + record.args = tuple(arg.replace(item, 8 * '*' + item[-5:]) if isinstance(arg, str) else arg for arg in record.args) except TypeError: pass @@ -39,27 +39,25 @@ class VarkenLogger(object): # Set log level if self.log_level: - self.log_level = logging.DEBUG + self.log_level = DEBUG else: - self.log_level = logging.INFO + self.log_level = INFO # Make the log directory if it does not exist mkdir_p('{}/{}'.format(self.data_folder, BlacklistFilter.log_folder)) # Create the Logger - self.logger = logging.getLogger() - self.logger.setLevel(logging.DEBUG) + self.logger = getLogger() + self.logger.setLevel(DEBUG) # Create a Formatter for formatting the log messages - logger_formatter = logging.Formatter('%(asctime)s : %(levelname)s : %(module)s : %(message)s', - '%Y-%m-%d %H:%M:%S') + logger_formatter = Formatter('%(asctime)s : %(levelname)s : %(module)s : %(message)s', '%Y-%m-%d %H:%M:%S') # Create the Handler for logging data to a file - file_logger = RotatingFileHandler('{}/{}/{}'.format(self.data_folder, BlacklistFilter.log_folder, - BlacklistFilter.filename), mode='a', - maxBytes=BlacklistFilter.max_size, backupCount=BlacklistFilter.max_files, - encoding=None, delay=0) + file_logger = RotatingFileHandler(f'{self.data_folder}/{BlacklistFilter.log_folder}/{BlacklistFilter.filename}', + mode='a', maxBytes=BlacklistFilter.max_size, encoding=None, delay=0, + backupCount=BlacklistFilter.max_files) file_logger.setLevel(self.log_level) @@ -67,7 +65,7 @@ class VarkenLogger(object): file_logger.setFormatter(logger_formatter) # Add the console logger - console_logger = logging.StreamHandler() + console_logger = StreamHandler() console_logger.setFormatter(logger_formatter) console_logger.setLevel(self.log_level) From fbc8488b079381953038efbb2176b978f4a9ede3 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 17 Dec 2018 20:19:41 -0600 Subject: [PATCH 65/76] bit more import scoping and added filter for using /location with domain --- varken/helpers.py | 58 +++++++++++++++++++++--------------------- varken/iniparser.py | 3 +++ varken/varkenlogger.py | 2 +- 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index aaa8ccd..4ca702d 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -1,17 +1,17 @@ -import os -import time -import tarfile -import hashlib -import urllib3 -import geoip2.database -import logging - -from json.decoder import JSONDecodeError -from os.path import abspath, join -from requests.exceptions import InvalidSchema, SSLError, ConnectionError +from time import time +from hashlib import md5 +from tarfile import open +from logging import getLogger +from geoip2.database import Reader +from urllib3 import disable_warnings +from os import stat, remove, makedirs from urllib.request import urlretrieve +from json.decoder import JSONDecodeError +from os.path import abspath, join, basename +from urllib3.exceptions import InsecureRequestWarning +from requests.exceptions import InvalidSchema, SSLError, ConnectionError -logger = logging.getLogger() +logger = getLogger() def geoip_download(data_folder): @@ -23,48 +23,48 @@ def geoip_download(data_folder): logger.info('Downloading GeoLite2 from %s', url) urlretrieve(url, tar_dbfile) - tar = tarfile.open(tar_dbfile, 'r:gz') - logging.debug('Opening GeoLite2 tar file : %s', tar_dbfile) + tar = open(tar_dbfile, 'r:gz') + logger.debug('Opening GeoLite2 tar file : %s', tar_dbfile) for files in tar.getmembers(): if 'GeoLite2-City.mmdb' in files.name: - logging.debug('"GeoLite2-City.mmdb" FOUND in tar file') - files.name = os.path.basename(files.name) + logger.debug('"GeoLite2-City.mmdb" FOUND in tar file') + files.name = basename(files.name) tar.extract(files, datafolder) - logging.debug('%s has been extracted to %s', files, datafolder) + logger.debug('%s has been extracted to %s', files, datafolder) - os.remove(tar_dbfile) + remove(tar_dbfile) def geo_lookup(ipaddress, data_folder): datafolder = data_folder - logging.debug('Reading GeoLite2 DB from %s', datafolder) + logger.debug('Reading GeoLite2 DB from %s', datafolder) dbfile = abspath(join(datafolder, 'GeoLite2-City.mmdb')) - now = time.time() + now = time() try: - dbinfo = os.stat(dbfile) + dbinfo = stat(dbfile) db_age = now - dbinfo.st_ctime if db_age > (35 * 86400): - logging.info('GeoLite2 DB is older than 35 days. Attempting to re-download...') + logger.info('GeoLite2 DB is older than 35 days. Attempting to re-download...') - os.remove(dbfile) + remove(dbfile) geoip_download(datafolder) except FileNotFoundError: - logging.error('GeoLite2 DB not found. Attempting to download...') + logger.error('GeoLite2 DB not found. Attempting to download...') geoip_download(datafolder) - reader = geoip2.database.Reader(dbfile) + reader = Reader(dbfile) return reader.city(ipaddress) def hashit(string): encoded = string.encode() - hashed = hashlib.md5(encoded).hexdigest() + hashed = md5(encoded).hexdigest() return hashed @@ -75,7 +75,7 @@ def connection_handler(session, request, verify): v = verify return_json = False - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + disable_warnings(InsecureRequestWarning) try: get = s.send(r, verify=v) @@ -106,10 +106,10 @@ def connection_handler(session, request, verify): def mkdir_p(path): - templogger = logging.getLogger('temp') + templogger = getLogger('temp') try: templogger.info('Creating folder %s ', path) - os.makedirs(path, exist_ok=True) + makedirs(path, exist_ok=True) except Exception as e: templogger.error('Could not create folder %s : %s ', path, e) diff --git a/varken/iniparser.py b/varken/iniparser.py index 24ea0b1..2e0d54d 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -44,6 +44,9 @@ class INIParser(object): filtered_strings = [section.get(k) for key, section in self.config.items() for k in section if k in BlacklistFilter.blacklisted_strings] self.filtered_strings = list(filter(None, filtered_strings)) + # Added matching for domains that use /locations. ConnectionPool ignores the location in logs + domains_only = list([ string.split('/')[0] for string in filtered_strings if '/' in string ]) + self.filtered_strings.extend(domains_only) for handler in self.logger.handlers: handler.addFilter(BlacklistFilter(set(self.filtered_strings))) diff --git a/varken/varkenlogger.py b/varken/varkenlogger.py index 6816af4..11d7f6a 100644 --- a/varken/varkenlogger.py +++ b/varken/varkenlogger.py @@ -45,7 +45,7 @@ class VarkenLogger(object): self.log_level = INFO # Make the log directory if it does not exist - mkdir_p('{}/{}'.format(self.data_folder, BlacklistFilter.log_folder)) + mkdir_p(f'{self.data_folder}/{BlacklistFilter.log_folder}') # Create the Logger self.logger = getLogger() From 0656e2f06d4b3bdf4e06641f29cea529c103d121 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 17 Dec 2018 21:16:46 -0600 Subject: [PATCH 66/76] reworked the way that the iniparser works --- Varken.py | 2 +- varken/iniparser.py | 253 ++++++++++++-------------------------------- 2 files changed, 70 insertions(+), 185 deletions(-) diff --git a/Varken.py b/Varken.py index b669c5b..136ad24 100644 --- a/Varken.py +++ b/Varken.py @@ -120,7 +120,7 @@ if __name__ == "__main__": schedule.every(server.get_missing_run_seconds).seconds.do(threaded, SICKCHILL.get_missing) if CONFIG.ciscoasa_enabled: - for firewall in CONFIG.ciscoasa_firewalls: + for firewall in CONFIG.ciscoasa_servers: ASA = CiscoAPI(firewall, DBMANAGER) schedule.every(firewall.get_bandwidth_run_seconds).seconds.do(threaded, ASA.get_bandwidth) diff --git a/varken/iniparser.py b/varken/iniparser.py index 2e0d54d..b813af0 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -14,28 +14,14 @@ class INIParser(object): self.config = ConfigParser(interpolation=None) self.data_folder = data_folder + self.services = ['sonarr', 'radarr', 'ombi', 'tautulli', 'sickchill', 'ciscoasa'] + for service in self.services: + setattr(self, f'{service}_servers', []) + self.logger = getLogger() self.influx_server = InfluxServer() - self.sonarr_enabled = False - self.sonarr_servers = [] - - self.radarr_enabled = False - self.radarr_servers = [] - - self.ombi_enabled = False - self.ombi_servers = [] - - self.tautulli_enabled = False - self.tautulli_servers = [] - - self.sickchill_enabled = False - self.sickchill_servers = [] - - self.ciscoasa_enabled = False - self.ciscoasa_firewalls = [] - self.parse_opts() self.filtered_strings = None @@ -45,7 +31,7 @@ class INIParser(object): for k in section if k in BlacklistFilter.blacklisted_strings] self.filtered_strings = list(filter(None, filtered_strings)) # Added matching for domains that use /locations. ConnectionPool ignores the location in logs - domains_only = list([ string.split('/')[0] for string in filtered_strings if '/' in string ]) + domains_only = [string.split('/')[0] for string in filtered_strings if '/' in string] self.filtered_strings.extend(domains_only) for handler in self.logger.handlers: @@ -117,204 +103,103 @@ class INIParser(object): self.influx_server = InfluxServer(url, port, username, password) - # Parse Sonarr options - self.sonarr_enabled = self.enable_check('sonarr_server_ids') + # Check for all enabled services + for service in self.services: + setattr(self, f'{service}_enabled', self.enable_check(f'{service}_server_ids')) + service_enabled = getattr(self, f'{service}_enabled') - if self.sonarr_enabled: - for server_id in self.sonarr_enabled: - section = 'sonarr-' + str(server_id) - try: - url = self.url_check(self.config.get(section, 'url')) + if service_enabled: + for server_id in service_enabled: + server = None + section = f"{service}-{server_id}" + try: + url = self.url_check(self.config.get(section, 'url')) - apikey = self.config.get(section, 'apikey') + apikey = None + if section != 'ciscoasa': + apikey = self.config.get(section, 'apikey') - scheme = 'https://' if self.config.getboolean(section, 'ssl') else 'http://' + scheme = 'https://' if self.config.getboolean(section, 'ssl') else 'http://' - verify_ssl = self.config.getboolean(section, 'verify_ssl') + verify_ssl = self.config.getboolean(section, 'verify_ssl') - if scheme != 'https://': - verify_ssl = False + if scheme != 'https://': + verify_ssl = False - queue = self.config.getboolean(section, 'queue') + if service == 'sonarr': + queue = self.config.getboolean(section, 'queue') - missing_days = self.config.getint(section, 'missing_days') + missing_days = self.config.getint(section, 'missing_days') - future_days = self.config.getint(section, 'future_days') + future_days = self.config.getint(section, 'future_days') - missing_days_run_seconds = self.config.getint(section, 'missing_days_run_seconds') + missing_days_run_seconds = self.config.getint(section, 'missing_days_run_seconds') - future_days_run_seconds = self.config.getint(section, 'future_days_run_seconds') + future_days_run_seconds = self.config.getint(section, 'future_days_run_seconds') - queue_run_seconds = self.config.getint(section, 'queue_run_seconds') + queue_run_seconds = self.config.getint(section, 'queue_run_seconds') - server = SonarrServer(server_id, scheme + url, apikey, verify_ssl, missing_days, - missing_days_run_seconds, future_days, future_days_run_seconds, - queue, queue_run_seconds) + server = SonarrServer(server_id, scheme + url, apikey, verify_ssl, missing_days, + missing_days_run_seconds, future_days, future_days_run_seconds, + queue, queue_run_seconds) - self.sonarr_servers.append(server) - except NoOptionError as e: - self.sonarr_enabled = False - self.logger.error( - '%s disabled. Error: %s', section, e) + if service == 'radarr': + queue = self.config.getboolean(section, 'queue') - # Parse Radarr options - self.radarr_enabled = self.enable_check('radarr_server_ids') + queue_run_seconds = self.config.getint(section, 'queue_run_seconds') - if self.radarr_enabled: - for server_id in self.radarr_enabled: - section = 'radarr-' + str(server_id) - try: - url = self.url_check(self.config.get(section, 'url')) + get_missing = self.config.getboolean(section, 'get_missing') - apikey = self.config.get(section, 'apikey') + get_missing_run_seconds = self.config.getint(section, 'get_missing_run_seconds') - scheme = 'https://' if self.config.getboolean(section, 'ssl') else 'http://' + server = RadarrServer(server_id, scheme + url, apikey, verify_ssl, queue, queue_run_seconds, + get_missing, get_missing_run_seconds) - verify_ssl = self.config.getboolean(section, 'verify_ssl') + if service == 'tautulli': + fallback_ip = self.config.get(section, 'fallback_ip') - if scheme != 'https://': - verify_ssl = False + get_activity = self.config.getboolean(section, 'get_activity') - queue = self.config.getboolean(section, 'queue') + get_activity_run_seconds = self.config.getint(section, 'get_activity_run_seconds') - queue_run_seconds = self.config.getint(section, 'queue_run_seconds') + server = TautulliServer(server_id, scheme + url, fallback_ip, apikey, verify_ssl, + get_activity, get_activity_run_seconds) - get_missing = self.config.getboolean(section, 'get_missing') + if service == 'ombi': + request_type_counts = self.config.getboolean(section, 'get_request_type_counts') - get_missing_run_seconds = self.config.getint(section, 'get_missing_run_seconds') + request_type_run_seconds = self.config.getint(section, 'request_type_run_seconds') - server = RadarrServer(server_id, scheme + url, apikey, verify_ssl, queue, queue_run_seconds, - get_missing, get_missing_run_seconds) - self.radarr_servers.append(server) - except NoOptionError as e: - self.radarr_enabled = False - self.logger.error( - '%s disabled. Error: %s', section, e) + request_total_counts = self.config.getboolean(section, 'get_request_total_counts') - # Parse Tautulli options - self.tautulli_enabled = self.enable_check('tautulli_server_ids') + request_total_run_seconds = self.config.getint(section, 'request_total_run_seconds') - if self.tautulli_enabled: - for server_id in self.tautulli_enabled: - section = 'tautulli-' + str(server_id) - try: - url = self.url_check(self.config.get(section, 'url')) + server = OmbiServer(server_id, scheme + url, apikey, verify_ssl, request_type_counts, + request_type_run_seconds, request_total_counts, + request_total_run_seconds) - fallback_ip = self.config.get(section, 'fallback_ip') + if service == 'sickchill': + get_missing = self.config.getboolean(section, 'get_missing') - apikey = self.config.get(section, 'apikey') + get_missing_run_seconds = self.config.getint(section, 'get_missing_run_seconds') - scheme = 'https://' if self.config.getboolean(section, 'ssl') else 'http://' + server = SickChillServer(server_id, scheme + url, apikey, verify_ssl, + get_missing, get_missing_run_seconds) - verify_ssl = self.config.getboolean(section, 'verify_ssl') + if service == 'ciscoasa': + username = self.config.get(section, 'username') - if scheme != 'https://': - verify_ssl = False + password = self.config.get(section, 'password') - get_activity = self.config.getboolean(section, 'get_activity') + outside_interface = self.config.get(section, 'outside_interface') - get_activity_run_seconds = self.config.getint(section, 'get_activity_run_seconds') + get_bandwidth_run_seconds = self.config.getint(section, 'get_bandwidth_run_seconds') - server = TautulliServer(server_id, scheme + url, fallback_ip, apikey, verify_ssl, get_activity, - get_activity_run_seconds) - self.tautulli_servers.append(server) - except NoOptionError as e: - self.tautulli_enabled = False - self.logger.error( - '%s disabled. Error: %s', section, e) + server = CiscoASAFirewall(server_id, scheme + url, username, password, outside_interface, + verify_ssl, get_bandwidth_run_seconds) - # Parse Ombi options - self.ombi_enabled = self.enable_check('ombi_server_ids') + getattr(self, f'{service}_servers').append(server) - if self.ombi_enabled: - for server_id in self.ombi_enabled: - section = 'ombi-' + str(server_id) - try: - url = self.url_check(self.config.get(section, 'url')) - - apikey = self.config.get(section, 'apikey') - - scheme = 'https://' if self.config.getboolean(section, 'ssl') else 'http://' - - verify_ssl = self.config.getboolean(section, 'verify_ssl') - - if scheme != 'https://': - verify_ssl = False - - request_type_counts = self.config.getboolean(section, 'get_request_type_counts') - - request_type_run_seconds = self.config.getint(section, 'request_type_run_seconds') - - request_total_counts = self.config.getboolean(section, 'get_request_total_counts') - - request_total_run_seconds = self.config.getint(section, 'request_total_run_seconds') - - server = OmbiServer(server_id, scheme + url, apikey, verify_ssl, request_type_counts, - request_type_run_seconds, request_total_counts, request_total_run_seconds) - self.ombi_servers.append(server) - except NoOptionError as e: - self.ombi_enabled = False - self.logger.error( - '%s disabled. Error: %s', section, e) - - # Parse SickChill options - self.sickchill_enabled = self.enable_check('sickchill_server_ids') - - if self.sickchill_enabled: - for server_id in self.sickchill_enabled: - section = 'sickchill-' + str(server_id) - try: - url = self.url_check(self.config.get(section, 'url')) - - apikey = self.config.get(section, 'apikey') - - scheme = 'https://' if self.config.getboolean(section, 'ssl') else 'http://' - - verify_ssl = self.config.getboolean(section, 'verify_ssl') - - if scheme != 'https://': - verify_ssl = False - - get_missing = self.config.getboolean(section, 'get_missing') - - get_missing_run_seconds = self.config.getint(section, 'get_missing_run_seconds') - - server = SickChillServer(server_id, scheme + url, apikey, verify_ssl, - get_missing, get_missing_run_seconds) - self.sickchill_servers.append(server) - except NoOptionError as e: - self.sickchill_enabled = False - self.logger.error( - '%s disabled. Error: %s', section, e) - - # Parse ASA opts - self.ciscoasa_enabled = self.enable_check('ciscoasa_firewall_ids') - - if self.ciscoasa_enabled: - for firewall_id in self.ciscoasa_enabled: - section = 'ciscoasa-' + str(firewall_id) - try: - url = self.url_check(self.config.get(section, 'url')) - - username = self.config.get(section, 'username') - - password = self.config.get(section, 'password') - - scheme = 'https://' if self.config.getboolean(section, 'ssl') else 'http://' - - verify_ssl = self.config.getboolean(section, 'verify_ssl') - - if scheme != 'https://': - verify_ssl = False - - outside_interface = self.config.get(section, 'outside_interface') - - get_bandwidth_run_seconds = self.config.getint(section, 'get_bandwidth_run_seconds') - - firewall = CiscoASAFirewall(firewall_id, scheme + url, username, password, outside_interface, - verify_ssl, get_bandwidth_run_seconds) - self.ciscoasa_firewalls.append(firewall) - except NoOptionError as e: - self.ciscoasa_enabled = False - self.logger.error( - '%s disabled. Error: %s', section, e) + except NoOptionError as e: + setattr(self, f'{service}_enabled', False) + self.logger.error('%s disabled. Error: %s', section, e) From 5cfbbe7a995b54f119356fd210018969c0223f6c Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 17 Dec 2018 21:23:37 -0600 Subject: [PATCH 67/76] added version logging for supportability --- Varken.py | 3 +++ varken/__init__.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Varken.py b/Varken.py index 136ad24..cfc6f05 100644 --- a/Varken.py +++ b/Varken.py @@ -12,6 +12,7 @@ from logging import getLogger, StreamHandler, Formatter, DEBUG from varken.ombi import OmbiAPI from varken.cisco import CiscoAPI +from varken import VERSION, BRANCH from varken.sonarr import SonarrAPI from varken.radarr import RadarrAPI from varken.iniparser import INIParser @@ -78,6 +79,8 @@ if __name__ == "__main__": vl.logger.info(u"Python %s", version) + vl.logger.info("Varken v%s-%s", VERSION, BRANCH) + CONFIG = INIParser(DATA_FOLDER) DBMANAGER = DBManager(CONFIG.influx_server) diff --git a/varken/__init__.py b/varken/__init__.py index b5a808d..314c9c8 100644 --- a/varken/__init__.py +++ b/varken/__init__.py @@ -1,2 +1,2 @@ -VERSION = 1.2 -BRANCH = 'nightly' +VERSION = 1.3 +BRANCH = 'pre-nightly' From 7f04419d4c891ec2f43b46955eccdd294c8ca1f7 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 17 Dec 2018 22:08:33 -0600 Subject: [PATCH 68/76] check for data directory existence for the sake of logging --- varken/helpers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index 4ca702d..eec1ec8 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -7,7 +7,7 @@ from urllib3 import disable_warnings from os import stat, remove, makedirs from urllib.request import urlretrieve from json.decoder import JSONDecodeError -from os.path import abspath, join, basename +from os.path import abspath, join, basename, isdir from urllib3.exceptions import InsecureRequestWarning from requests.exceptions import InvalidSchema, SSLError, ConnectionError @@ -108,8 +108,9 @@ def connection_handler(session, request, verify): def mkdir_p(path): templogger = getLogger('temp') try: - templogger.info('Creating folder %s ', path) - makedirs(path, exist_ok=True) + if not isdir(path): + templogger.info('Creating folder %s ', path) + makedirs(path, exist_ok=True) except Exception as e: templogger.error('Could not create folder %s : %s ', path, e) From 556109e8c5d683eade727ddf1315cd6a22c3cb14 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 17 Dec 2018 23:47:29 -0600 Subject: [PATCH 69/76] v --- Varken.py | 6 +++- varken/__init__.py | 2 +- varken/helpers.py | 88 +++++++++++++++++++++++++-------------------- varken/iniparser.py | 2 -- varken/tautulli.py | 12 +++---- 5 files changed, 62 insertions(+), 48 deletions(-) diff --git a/Varken.py b/Varken.py index cfc6f05..bb7e03e 100644 --- a/Varken.py +++ b/Varken.py @@ -17,10 +17,12 @@ from varken.sonarr import SonarrAPI from varken.radarr import RadarrAPI from varken.iniparser import INIParser from varken.dbmanager import DBManager +from varken.helpers import GeoIPHandler from varken.tautulli import TautulliAPI from varken.sickchill import SickChillAPI from varken.varkenlogger import VarkenLogger + PLATFORM_LINUX_DISTRO = ' '.join(x for x in linux_distribution() if x) @@ -95,8 +97,10 @@ if __name__ == "__main__": schedule.every(server.future_days_run_seconds).seconds.do(threaded, SONARR.get_future) if CONFIG.tautulli_enabled: + GEOIPHANDLER = GeoIPHandler(DATA_FOLDER) + schedule.every(12).to(24).hours.do(threaded, GEOIPHANDLER.update) for server in CONFIG.tautulli_servers: - TAUTULLI = TautulliAPI(server, DBMANAGER, DATA_FOLDER) + TAUTULLI = TautulliAPI(server, DBMANAGER, GEOIPHANDLER) if server.get_activity: schedule.every(server.get_activity_run_seconds).seconds.do(threaded, TAUTULLI.get_activity) diff --git a/varken/__init__.py b/varken/__init__.py index 314c9c8..ba78dc4 100644 --- a/varken/__init__.py +++ b/varken/__init__.py @@ -1,2 +1,2 @@ VERSION = 1.3 -BRANCH = 'pre-nightly' +BRANCH = 'nightly' diff --git a/varken/helpers.py b/varken/helpers.py index eec1ec8..88426b3 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -1,7 +1,8 @@ -from time import time from hashlib import md5 -from tarfile import open +from tarfile import open as taropen +from datetime import date from logging import getLogger +from calendar import monthcalendar from geoip2.database import Reader from urllib3 import disable_warnings from os import stat, remove, makedirs @@ -14,52 +15,63 @@ from requests.exceptions import InvalidSchema, SSLError, ConnectionError logger = getLogger() -def geoip_download(data_folder): - datafolder = data_folder +class GeoIPHandler(object): + def __init__(self, data_folder): + self.data_folder = data_folder + self.dbfile = abspath(join(self.data_folder, 'GeoLite2-City.mmdb')) + self.logger = getLogger() + self.update() - tar_dbfile = abspath(join(datafolder, 'GeoLite2-City.tar.gz')) + self.logger.info('Opening persistent connection to GeoLite2 DB...') + self.reader = Reader(self.dbfile) - url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz' - logger.info('Downloading GeoLite2 from %s', url) - urlretrieve(url, tar_dbfile) + def lookup(self, ipaddress): + ip = ipaddress + self.logger.debug('Getting lat/long for Tautulli stream') + return self.reader.city(ip) - tar = open(tar_dbfile, 'r:gz') - logger.debug('Opening GeoLite2 tar file : %s', tar_dbfile) + def update(self): + today = date.today() + dbdate = None + try: + dbdate = date.fromtimestamp(stat(self.dbfile).st_ctime) + except FileNotFoundError: + self.logger.error("Could not find GeoLite2 DB as: %s", self.dbfile) + self.download() + first_wednesday_day = [week[2:3][0] for week in monthcalendar(today.year, today.month) if week[2:3][0] != 0][0] + first_wednesday_date = date(today.year, today.month, first_wednesday_day) - for files in tar.getmembers(): - if 'GeoLite2-City.mmdb' in files.name: - logger.debug('"GeoLite2-City.mmdb" FOUND in tar file') - files.name = basename(files.name) - - tar.extract(files, datafolder) - logger.debug('%s has been extracted to %s', files, datafolder) - - remove(tar_dbfile) + if dbdate < first_wednesday_date < today: + self.logger.info("Newer GeoLite2 DB available, Updating...") + remove(self.dbfile) + self.download() + else: + td = first_wednesday_date - today + if td.days < 0: + self.logger.debug('Geolite2 DB is only %s days old. Keeping current copy', abs(td.days)) + else: + self.logger.debug('Geolite2 DB will update in %s days', abs(td.days)) -def geo_lookup(ipaddress, data_folder): - datafolder = data_folder - logger.debug('Reading GeoLite2 DB from %s', datafolder) + def download(self): + tar_dbfile = abspath(join(self.data_folder, 'GeoLite2-City.tar.gz')) + url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz' - dbfile = abspath(join(datafolder, 'GeoLite2-City.mmdb')) - now = time() + self.logger.info('Downloading GeoLite2 from %s', url) + urlretrieve(url, tar_dbfile) - try: - dbinfo = stat(dbfile) - db_age = now - dbinfo.st_ctime - if db_age > (35 * 86400): - logger.info('GeoLite2 DB is older than 35 days. Attempting to re-download...') + self.logger.debug('Opening GeoLite2 tar file : %s', tar_dbfile) - remove(dbfile) + tar = taropen(tar_dbfile, 'r:gz') - geoip_download(datafolder) - except FileNotFoundError: - logger.error('GeoLite2 DB not found. Attempting to download...') - geoip_download(datafolder) - - reader = Reader(dbfile) - - return reader.city(ipaddress) + for files in tar.getmembers(): + if 'GeoLite2-City.mmdb' in files.name: + self.logger.debug('"GeoLite2-City.mmdb" FOUND in tar file') + files.name = basename(files.name) + tar.extract(files, self.data_folder) + self.logger.debug('%s has been extracted to %s', files, self.data_folder) + tar.close() + remove(tar_dbfile) def hashit(string): diff --git a/varken/iniparser.py b/varken/iniparser.py index b813af0..29144fc 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -75,8 +75,6 @@ class INIParser(object): regex = compile('{}'.format(search), IGNORECASE) - print(match(regex, url_check)) - valid = match(regex, url_check) is not None if not valid: if inc_port: diff --git a/varken/tautulli.py b/varken/tautulli.py index f71ce81..896278f 100644 --- a/varken/tautulli.py +++ b/varken/tautulli.py @@ -4,18 +4,18 @@ from datetime import datetime, timezone from geoip2.errors import AddressNotFoundError from varken.structures import TautulliStream -from varken.helpers import geo_lookup, hashit, connection_handler +from varken.helpers import hashit, connection_handler class TautulliAPI(object): - def __init__(self, server, dbmanager, data_folder): + def __init__(self, server, dbmanager, geoiphandler): self.dbmanager = dbmanager self.server = server + self.geoiphandler = geoiphandler self.session = Session() self.session.params = {'apikey': self.server.api_key, 'cmd': 'get_activity'} self.endpoint = '/api/v2' self.logger = getLogger() - self.data_folder = data_folder def __repr__(self): return f"" @@ -47,13 +47,13 @@ class TautulliAPI(object): exit(1) try: - geodata = geo_lookup(session.ip_address_public, self.data_folder) + geodata = self.geoiphandler.lookup(session.ip_address_public) except (ValueError, AddressNotFoundError): if self.server.fallback_ip: - geodata = geo_lookup(self.server.fallback_ip, self.data_folder) + geodata = self.geoiphandler.lookup(self.server.fallback_ip) else: my_ip = self.session.get('http://ip.42.pl/raw').text - geodata = geo_lookup(my_ip, self.data_folder) + geodata = self.geoiphandler.lookup(my_ip) if not all([geodata.location.latitude, geodata.location.longitude]): latitude = 37.234332396 From 831498f89b2c480eba82a976916aeb5cb17d6e16 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Mon, 17 Dec 2018 23:52:14 -0600 Subject: [PATCH 70/76] changelog update and version bump --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff3fafd..01c35f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Change Log +## [v1.3-nightly](https://github.com/Boerderij/Varken/tree/v1.3-nightly) (2018-12-17) +[Full Changelog](https://github.com/Boerderij/Varken/compare/v1.2-nightly...v1.3-nightly) + +**Implemented enhancements:** + +- Create randomized 12-24 hour checks to update GeoLite DB after the first wednesday of the month [\#60](https://github.com/Boerderij/Varken/issues/60) + +**Fixed bugs:** + +- \[BUG\] Add Catchall to ombi requests [\#59](https://github.com/Boerderij/Varken/issues/59) + +**Closed issues:** + +- Unify naming and cleanup duplication in iniparser [\#61](https://github.com/Boerderij/Varken/issues/61) + ## [v1.2-nightly](https://github.com/Boerderij/Varken/tree/v1.2-nightly) (2018-12-16) [Full Changelog](https://github.com/Boerderij/Varken/compare/v1.1...v1.2-nightly) From 67b98d57ea2e2fc5eb4f371fe237b26f9c14461f Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 18 Dec 2018 20:30:04 -0600 Subject: [PATCH 71/76] added stats pull from tautulli -,- --- Varken.py | 2 ++ data/varken.example.ini | 2 ++ varken/iniparser.py | 7 ++++++- varken/structures.py | 2 ++ varken/tautulli.py | 39 +++++++++++++++++++++++++++++++++++++-- 5 files changed, 49 insertions(+), 3 deletions(-) diff --git a/Varken.py b/Varken.py index bb7e03e..20aea2b 100644 --- a/Varken.py +++ b/Varken.py @@ -103,6 +103,8 @@ if __name__ == "__main__": TAUTULLI = TautulliAPI(server, DBMANAGER, GEOIPHANDLER) if server.get_activity: schedule.every(server.get_activity_run_seconds).seconds.do(threaded, TAUTULLI.get_activity) + if server.get_stats: + schedule.every(server.get_stats_run_seconds).seconds.do(threaded, TAUTULLI.get_stats) if CONFIG.radarr_enabled: for server in CONFIG.radarr_servers: diff --git a/data/varken.example.ini b/data/varken.example.ini index 51675ad..b061984 100644 --- a/data/varken.example.ini +++ b/data/varken.example.ini @@ -27,6 +27,8 @@ ssl = false verify_ssl = false get_activity = true get_activity_run_seconds = 30 +get_stats = true +get_stats_run_seconds = 3600 [sonarr-1] url = sonarr1.domain.tld:8989 diff --git a/varken/iniparser.py b/varken/iniparser.py index 29144fc..30f3971 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -160,8 +160,13 @@ class INIParser(object): get_activity_run_seconds = self.config.getint(section, 'get_activity_run_seconds') + get_stats = self.config.getboolean(section, 'get_stats') + + get_stats_run_seconds = self.config.getint(section, 'get_stats_run_seconds') + server = TautulliServer(server_id, scheme + url, fallback_ip, apikey, verify_ssl, - get_activity, get_activity_run_seconds) + get_activity, get_activity_run_seconds, get_stats, + get_stats_run_seconds) if service == 'ombi': request_type_counts = self.config.getboolean(section, 'get_request_type_counts') diff --git a/varken/structures.py b/varken/structures.py index 1c9b8a7..accd4fe 100644 --- a/varken/structures.py +++ b/varken/structures.py @@ -71,6 +71,8 @@ class TautulliServer(NamedTuple): verify_ssl: bool = None get_activity: bool = False get_activity_run_seconds: int = 30 + get_stats: bool = False + get_stats_run_seconds: int = 30 class InfluxServer(NamedTuple): diff --git a/varken/tautulli.py b/varken/tautulli.py index 896278f..5bde35a 100644 --- a/varken/tautulli.py +++ b/varken/tautulli.py @@ -13,7 +13,7 @@ class TautulliAPI(object): self.server = server self.geoiphandler = geoiphandler self.session = Session() - self.session.params = {'apikey': self.server.api_key, 'cmd': 'get_activity'} + self.session.params = {'apikey': self.server.api_key} self.endpoint = '/api/v2' self.logger = getLogger() @@ -23,8 +23,9 @@ class TautulliAPI(object): def get_activity(self): now = datetime.now(timezone.utc).astimezone().isoformat() influx_payload = [] + params = {'cmd': 'get_activity'} - req = self.session.prepare_request(Request('GET', self.server.url + self.endpoint)) + req = self.session.prepare_request(Request('GET', self.server.url + self.endpoint, params=params)) g = connection_handler(self.session, req, self.server.verify_ssl) if not g: @@ -150,3 +151,37 @@ class TautulliAPI(object): ) self.dbmanager.write_points(influx_payload) + + def get_stats(self): + now = datetime.now(timezone.utc).astimezone().isoformat() + influx_payload = [] + params = {'cmd': 'get_libraries'} + + req = self.session.prepare_request(Request('GET', self.server.url + self.endpoint, params=params)) + g = connection_handler(self.session, req, self.server.verify_ssl) + + if not g: + return + + get = g['response']['data'] + + for library in get: + data = { + "measurement": "Tautulli", + "tags": { + "type": "library_stats", + "server": self.server.id, + "section_name": library['section_name'], + "section_type": library['section_type'] + }, + "time": now, + "fields": { + "total": int(library['count']) + } + } + if library['section_type'] == 'show': + data['fields']['seasons'] = int(library['parent_count']) + data['fields']['episodes'] = int(library['child_count']) + influx_payload.append(data) + + self.dbmanager.write_points(influx_payload) From 6665891b65819b4e60fff359539b002d00136346 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 18 Dec 2018 20:35:58 -0600 Subject: [PATCH 72/76] fixed cisco asa bug with checking for api key --- varken/iniparser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varken/iniparser.py b/varken/iniparser.py index 30f3971..dcd2e44 100644 --- a/varken/iniparser.py +++ b/varken/iniparser.py @@ -114,7 +114,7 @@ class INIParser(object): url = self.url_check(self.config.get(section, 'url')) apikey = None - if section != 'ciscoasa': + if service != 'ciscoasa': apikey = self.config.get(section, 'apikey') scheme = 'https://' if self.config.getboolean(section, 'ssl') else 'http://' From 73ce0057b5904f3aaf41ca634517ada9e9313778 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 18 Dec 2018 21:15:41 -0600 Subject: [PATCH 73/76] added pending to ombi --- varken/ombi.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/varken/ombi.py b/varken/ombi.py index e0a13bb..84ceea7 100644 --- a/varken/ombi.py +++ b/varken/ombi.py @@ -65,13 +65,15 @@ class OmbiAPI(object): for movie in movie_requests: hash_id = hashit(f'{movie.id}{movie.theMovieDbId}{movie.title}') status = None - # Denied = 0, Approved = 1, Completed = 2 + # Denied = 0, Approved = 1, Completed = 2, Pending = 3 if movie.denied: status = 0 elif movie.approved and movie.available: status = 2 elif movie.approved: status = 1 + else: + status = 3 influx_payload.append( { From d6cef8b06b56d90743e563d463635046b1d729d9 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 18 Dec 2018 21:24:02 -0600 Subject: [PATCH 74/76] fixed logic for ombi requests --- varken/ombi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/varken/ombi.py b/varken/ombi.py index 84ceea7..f9a4830 100644 --- a/varken/ombi.py +++ b/varken/ombi.py @@ -28,7 +28,8 @@ class OmbiAPI(object): get_tv = connection_handler(self.session, tv_req, self.server.verify_ssl) get_movie = connection_handler(self.session, movie_req, self.server.verify_ssl) - if not all([get_tv, get_movie]): + if not any([get_tv, get_movie]): + self.logger.error('No json replies. Discarding job') return movie_request_count = len(get_movie) From 2cc7ef7be06dc5ba865c005b587139b9efada985 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 18 Dec 2018 21:28:27 -0600 Subject: [PATCH 75/76] update picture of readme to only include modules obtainable from varken data --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e4ccef7..6707974 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Requirements:

Example Dashboard - +

Supported Modules: From 8978bb380fa917db42c55080549aef96e25f530d Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 18 Dec 2018 21:59:24 -0600 Subject: [PATCH 76/76] changelog and version bump for master release v1.4 --- CHANGELOG.md | 18 +++++++++++++++++- README.md | 2 +- varken/__init__.py | 4 ++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01c35f8..eda655d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,22 @@ # Change Log -## [v1.3-nightly](https://github.com/Boerderij/Varken/tree/v1.3-nightly) (2018-12-17) +## [v1.4](https://github.com/Boerderij/Varken/tree/v1.4) (2018-12-18) +[Full Changelog](https://github.com/Boerderij/Varken/compare/v1.3-nightly...v1.4) + +**Implemented enhancements:** + +- \[Feature Request\] Add tautulli request for library stats [\#64](https://github.com/Boerderij/Varken/issues/64) + +**Fixed bugs:** + +- \[BUG\] Ombi all requests missing half of "pending" option [\#63](https://github.com/Boerderij/Varken/issues/63) +- \[BUG\] asa bug with checking for apikey [\#62](https://github.com/Boerderij/Varken/issues/62) + +**Merged pull requests:** + +- v1.4 Merge [\#65](https://github.com/Boerderij/Varken/pull/65) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) + +## [v1.3-nightly](https://github.com/Boerderij/Varken/tree/v1.3-nightly) (2018-12-18) [Full Changelog](https://github.com/Boerderij/Varken/compare/v1.2-nightly...v1.3-nightly) **Implemented enhancements:** diff --git a/README.md b/README.md index 6707974..fe68aa7 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Requirements:

Example Dashboard - +

Supported Modules: diff --git a/varken/__init__.py b/varken/__init__.py index ba78dc4..e19b0bd 100644 --- a/varken/__init__.py +++ b/varken/__init__.py @@ -1,2 +1,2 @@ -VERSION = 1.3 -BRANCH = 'nightly' +VERSION = 1.4 +BRANCH = 'master'