From f217da5e62c067a66fffaa43e50b33cc641c2f70 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 5 Feb 2019 20:46:19 -0600 Subject: [PATCH 01/20] switch from travis-ci to jenkins --- .travis.yml | 18 --------- Jenkinsfile | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++ deploy.sh | 79 ------------------------------------- prebuild.sh | 3 -- 4 files changed, 111 insertions(+), 100 deletions(-) delete mode 100644 .travis.yml create mode 100644 Jenkinsfile delete mode 100644 deploy.sh delete mode 100644 prebuild.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0dd0518..0000000 --- a/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -sudo: true -dist: xenial -language: minimal -services: - - docker -addons: - apt: - packages: - - docker-ce -deploy: - - provider: script - script: bash deploy.sh - on: - branch: master - - provider: script - script: bash deploy.sh - on: - branch: develop \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..85ad7f0 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,111 @@ +pipeline { + agent none + environment { + REPOSITORY = "boerderij/varken" + } + stages { + stage('Flake8') { + agent { label 'amd64'} + steps { + sh ''' + python3 -m venv venv && venv/bin/pip install flake8 && venv/bin/python -m flake8 --max-line-length 120 *.py varken/*.py + rm -rf venv/ + ''' + } + } + stage('Docker Builds') { + parallel { + stage('amd64') { + when { + anyOf { + branch 'master' + branch 'develop' + } + } + agent { label 'amd64'} + steps { + script { + if (BRANCH_NAME == 'master') { + def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() + def image = docker.build("${REPOSITORY}:${tag}-amd64") + image.push() + + } else if (BRANCH_NAME == 'develop') { + def image = docker.build("${REPOSITORY}:develop-amd64") + image.push() + } + } + } + } + stage('ARMv6') { + when { + anyOf { + branch 'master' + branch 'develop' + } + } + agent { label 'arm64'} + steps { + script { + if (BRANCH_NAME == 'master') { + def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() + def image = docker.build("${REPOSITORY}:${tag}-arm", "-f Dockerfile.arm .") + image.push() + } else if (BRANCH_NAME == 'develop') { + def image = docker.build("${REPOSITORY}:develop-arm", "-f Dockerfile.arm .") + image.push() + } + } + } + } + stage('ARM64v8') { + when { + anyOf { + branch 'master' + branch 'develop' + } + } + agent { label 'arm64'} + steps { + script { + if (BRANCH_NAME == 'master') { + def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() + def image = docker.build("${REPOSITORY}:${tag}-arm64", "-f Dockerfile.arm64 .") + image.push() + } else if (BRANCH_NAME == 'develop') { + def image = docker.build("${REPOSITORY}:develop-arm64", "-f Dockerfile.arm64 .") + image.push() + } + } + } + } + } + } + stage('Docker Manifest Build') { + when { + anyOf { + branch 'master' + branch 'develop' + } + } + agent { label 'amd64'} + steps { + script { + if (BRANCH_NAME == 'master') { + def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() + sh(script: "docker manifest create ${REPOSITORY}:${tag} ${REPOSITORY}:${tag}-amd64 ${REPOSITORY}:${tag}-arm64 ${REPOSITORY}:${tag}-arm") + sh(script: "docker manifest inspect ${REPOSITORY}:${tag}") + sh(script: "docker manifest push -p ${REPOSITORY}:${tag}") + sh(script: "docker manifest create ${REPOSITORY}:latest ${REPOSITORY}:${tag}-amd64 ${REPOSITORY}:${tag}-arm64 ${REPOSITORY}:${tag}-arm") + sh(script: "docker manifest inspect ${REPOSITORY}:latest") + sh(script: "docker manifest push -p ${REPOSITORY}:latest") + } else if (BRANCH_NAME == 'develop') { + sh(script: "docker manifest create ${REPOSITORY}:develop ${REPOSITORY}:develop-amd64 ${REPOSITORY}:develop-arm64 ${REPOSITORY}:develop-arm") + sh(script: "docker manifest inspect ${REPOSITORY}:develop") + sh(script: "docker manifest push -p ${REPOSITORY}:develop") + } + } + } + } + } +} diff --git a/deploy.sh b/deploy.sh deleted file mode 100644 index d2feb82..0000000 --- a/deploy.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -# Travis-ci convenience environment vars used: -# TRAVIS_BRANCH | branch name -# $TRAVIS_REPO_SLUG | organization/project (GitHub Capitalization) -# Travis-ci manual environment vars used: -# GITHUB_USER | github username -# GITHUB_TOKEN | $GITHUB_USER's token -# DOCKER_USER | docker username -# DOCKER_PASSWORD | $DOCKER_USER's password - -VERSION="$(grep -i version varken/__init__.py | cut -d' ' -f3 | tr -d \")" - -# Set branch to latest if master, else keep the same -if [[ "$TRAVIS_BRANCH" == "master" ]]; then - BRANCH="latest" -else - BRANCH="$TRAVIS_BRANCH" -fi - -# get the docker lowercase variant of the repo_name -REPOSITORY="$(echo $TRAVIS_REPO_SLUG | tr '[:upper:]' '[:lower:]')" - -# Docker experimental config -echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json -[[ -d ~/.docker ]] || mkdir ~/.docker -[[ -f ~/.docker/config.json ]] || touch ~/.docker/config.json -echo '{"experimental":"enabled"}' | sudo tee ~/.docker/config.json -sudo service docker restart - -# Auth -echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USER" --password-stdin - -# Prepare QEMU for ARM builds -docker run --rm --privileged multiarch/qemu-user-static:register --reset -bash prebuild.sh -chmod +x qemu-aarch64-static qemu-arm-static - -# Set tag based off of branch -if [[ "$BRANCH" == "latest" ]]; then - TAG="$VERSION" -else - TAG="$BRANCH" -fi - -# AMDx64 -docker build -t "${REPOSITORY}:${TAG}-amd64" . && \ -docker push "${REPOSITORY}:${TAG}-amd64" - -# Create Initial Manifests -docker manifest create "${REPOSITORY}:${TAG}" "${REPOSITORY}:${TAG}-amd64" -if [[ "$BRANCH" == "latest" ]]; then - docker manifest create "${REPOSITORY}:${BRANCH}" "${REPOSITORY}:${TAG}-amd64" -fi - -# ARM variants -for i in $(ls *arm*); do - ARCH="$(echo ${i} | cut -d. -f2)" - docker build -f "Dockerfile.${ARCH}" -t "${REPOSITORY}:${TAG}-${ARCH}" . && \ - docker push "${REPOSITORY}:${TAG}-${ARCH}" - # Add variant to manifest - docker manifest create -a "${REPOSITORY}:${TAG}" "${REPOSITORY}:${TAG}-${ARCH}" - if [[ "$BRANCH" == "latest" ]]; then - docker manifest create -a "${REPOSITORY}:${BRANCH}" "${REPOSITORY}:${TAG}-${ARCH}" - fi -done - -docker manifest inspect "${REPOSITORY}:${TAG}" && \ -docker manifest push "${REPOSITORY}:${TAG}" -if [[ "$BRANCH" == "latest" ]]; then - docker manifest inspect "${REPOSITORY}:${BRANCH}" && \ - docker manifest push "${REPOSITORY}:${BRANCH}" -fi - -# Git tags -if [[ "$BRANCH" == "latest" ]]; then - git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${REPOSITORY}.git" && \ - git tag "${VERSION}" && \ - git push --tags -fi \ No newline at end of file diff --git a/prebuild.sh b/prebuild.sh deleted file mode 100644 index 8d12fd7..0000000 --- a/prebuild.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -wget -q "https://github.com/multiarch/qemu-user-static/releases/download/v3.1.0-2/qemu-aarch64-static" -wget -q "https://github.com/multiarch/qemu-user-static/releases/download/v3.1.0-2/qemu-arm-static" \ No newline at end of file From cdb50319d0b275cdcaaa6b7e3b2542fa5248e377 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 5 Feb 2019 20:52:41 -0600 Subject: [PATCH 02/20] flake8 --- Varken.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Varken.py b/Varken.py index cd83c34..a47feff 100644 --- a/Varken.py +++ b/Varken.py @@ -10,7 +10,8 @@ from os.path import isdir, abspath, dirname, join from argparse import ArgumentParser, RawTextHelpFormatter from logging import getLogger, StreamHandler, Formatter, DEBUG -from varken import structures # Needed to check version of python +# Needed to check version of python +from varken import structures # noqa from varken.ombi import OmbiAPI from varken.unifi import UniFiAPI from varken import VERSION, BRANCH From 4270fc5a68d0277bbc69afffdccd93843cb1d72f Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 5 Feb 2019 20:54:37 -0600 Subject: [PATCH 03/20] remove dependancies --- Dockerfile.arm | 2 -- Dockerfile.arm64 | 2 -- 2 files changed, 4 deletions(-) diff --git a/Dockerfile.arm b/Dockerfile.arm index 3bcad32..8f12617 100644 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -6,8 +6,6 @@ ENV DEBUG="False" WORKDIR /app -COPY /qemu-arm-static /usr/bin/qemu-arm-static - COPY /requirements.txt /Varken.py /app/ COPY /varken /app/varken diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index d18a154..98e5ff9 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -6,8 +6,6 @@ ENV DEBUG="False" WORKDIR /app -COPY /qemu-aarch64-static /usr/bin/qemu-aarch64-static - COPY /requirements.txt /Varken.py /app/ COPY /varken /app/varken From bffac4d6a2be09d2a7d703a564f32dc5110287d4 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 5 Feb 2019 23:47:56 -0600 Subject: [PATCH 04/20] better jenkinsfile --- Jenkinsfile | 61 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 85ad7f0..d751aef 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,16 +1,24 @@ pipeline { agent none environment { - REPOSITORY = "boerderij/varken" + DOCKER_REPO = "boerderij/varken" + GIT_REPO = 'Boerderij/Varken' + VERSION_FILE = "varken/__init__.py" + FLAKE_FILES = "Varken.py varken/*.py" + TAG = "" + GIT_TOKEN = credentials('github-jenkins-token') } stages { stage('Flake8') { agent { label 'amd64'} steps { - sh ''' - python3 -m venv venv && venv/bin/pip install flake8 && venv/bin/python -m flake8 --max-line-length 120 *.py varken/*.py + sh """ + python3 -m venv venv && venv/bin/pip install flake8 && venv/bin/python -m flake8 --max-line-length 120 ${FLAKE_FILES} rm -rf venv/ - ''' + """ + script { + TAG = sh(returnStdout: true, script: 'grep -i version ${VERSION_FILE} | cut -d" " -f3 | tr -d \\"').trim() + } } } stage('Docker Builds') { @@ -26,12 +34,11 @@ pipeline { steps { script { if (BRANCH_NAME == 'master') { - def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() - def image = docker.build("${REPOSITORY}:${tag}-amd64") + def image = docker.build("${DOCKER_REPO}:${TAG}-amd64") image.push() } else if (BRANCH_NAME == 'develop') { - def image = docker.build("${REPOSITORY}:develop-amd64") + def image = docker.build("${DOCKER_REPO}:develop-amd64") image.push() } } @@ -48,11 +55,10 @@ pipeline { steps { script { if (BRANCH_NAME == 'master') { - def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() - def image = docker.build("${REPOSITORY}:${tag}-arm", "-f Dockerfile.arm .") + def image = docker.build("${DOCKER_REPO}:${TAG}-arm", "-f Dockerfile.arm .") image.push() } else if (BRANCH_NAME == 'develop') { - def image = docker.build("${REPOSITORY}:develop-arm", "-f Dockerfile.arm .") + def image = docker.build("${DOCKER_REPO}:develop-arm", "-f Dockerfile.arm .") image.push() } } @@ -69,11 +75,10 @@ pipeline { steps { script { if (BRANCH_NAME == 'master') { - def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() - def image = docker.build("${REPOSITORY}:${tag}-arm64", "-f Dockerfile.arm64 .") + def image = docker.build("${DOCKER_REPO}:${TAG}-arm64", "-f Dockerfile.arm64 .") image.push() } else if (BRANCH_NAME == 'develop') { - def image = docker.build("${REPOSITORY}:develop-arm64", "-f Dockerfile.arm64 .") + def image = docker.build("${DOCKER_REPO}:develop-arm64", "-f Dockerfile.arm64 .") image.push() } } @@ -92,20 +97,30 @@ pipeline { steps { script { if (BRANCH_NAME == 'master') { - def tag = sh(returnStdout: true, script: 'grep -i version varken/__init__.py | cut -d" " -f3 | tr -d \\"').trim() - sh(script: "docker manifest create ${REPOSITORY}:${tag} ${REPOSITORY}:${tag}-amd64 ${REPOSITORY}:${tag}-arm64 ${REPOSITORY}:${tag}-arm") - sh(script: "docker manifest inspect ${REPOSITORY}:${tag}") - sh(script: "docker manifest push -p ${REPOSITORY}:${tag}") - sh(script: "docker manifest create ${REPOSITORY}:latest ${REPOSITORY}:${tag}-amd64 ${REPOSITORY}:${tag}-arm64 ${REPOSITORY}:${tag}-arm") - sh(script: "docker manifest inspect ${REPOSITORY}:latest") - sh(script: "docker manifest push -p ${REPOSITORY}:latest") + sh(script: "docker manifest create ${DOCKER_REPO}:${TAG} ${DOCKER_REPO}:${TAG}-amd64 ${DOCKER_REPO}:${TAG}-arm64 ${DOCKER_REPO}:${TAG}-arm") + sh(script: "docker manifest inspect ${DOCKER_REPO}:${TAG}") + sh(script: "docker manifest push -p ${DOCKER_REPO}:${TAG}") + sh(script: "docker manifest create ${DOCKER_REPO}:latest ${DOCKER_REPO}:${TAG}-amd64 ${DOCKER_REPO}:${TAG}-arm64 ${DOCKER_REPO}:${TAG}-arm") + sh(script: "docker manifest inspect ${DOCKER_REPO}:latest") + sh(script: "docker manifest push -p ${DOCKER_REPO}:latest") } else if (BRANCH_NAME == 'develop') { - sh(script: "docker manifest create ${REPOSITORY}:develop ${REPOSITORY}:develop-amd64 ${REPOSITORY}:develop-arm64 ${REPOSITORY}:develop-arm") - sh(script: "docker manifest inspect ${REPOSITORY}:develop") - sh(script: "docker manifest push -p ${REPOSITORY}:develop") + sh(script: "docker manifest create ${DOCKER_REPO}:develop ${DOCKER_REPO}:develop-amd64 ${DOCKER_REPO}:develop-arm64 ${DOCKER_REPO}:develop-arm") + sh(script: "docker manifest inspect ${DOCKER_REPO}:develop") + sh(script: "docker manifest push -p ${DOCKER_REPO}:develop") } } } } + stage('GitHub Release') { + when { branch 'master' } + agent { label 'amd64'} + steps { + sh """ + git remote set-url origin "https://${GIT_TOKEN_USR}:${GIT_TOKEN_PSW}@github.com/${GIT_REPO}.git" + git tag ${TAG} + git push --tags + """ + } + } } } From 4eda93a4bf8f36b8b366b581fc23679603192834 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 5 Feb 2019 23:51:02 -0600 Subject: [PATCH 05/20] update badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 105a180..bb61fe5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@

-[![Build Status](https://travis-ci.org/Boerderij/Varken.svg?branch=master)](https://travis-ci.org/Boerderij/Varken) +[![Build Status](https://jenkins.cajun.pro/buildStatus/icon?job=Varken/master)](https://jenkins.cajun.pro/job/Varken/develop) [![Discord](https://img.shields.io/discord/518970285773422592.svg?colorB=7289DA&label=Discord&logo=Discord&logoColor=7289DA&style=flat-square)](https://discord.gg/VjZ6qSM) [![BuyMeACoffee](https://img.shields.io/badge/BuyMeACoffee-Donate-ff813f.svg?logo=CoffeeScript&style=flat-square)](https://www.buymeacoffee.com/varken) [![Docker-Layers](https://images.microbadger.com/badges/image/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken) From 1fd6eef44b0fca71e7886baf7d800a7fa400e368 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 12 Feb 2019 12:44:32 -0600 Subject: [PATCH 06/20] open/close file for windows issues --- varken/helpers.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/varken/helpers.py b/varken/helpers.py index 93817be..8657d43 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -24,7 +24,14 @@ class GeoIPHandler(object): self.update() self.logger.info('Opening persistent connection to GeoLite2 DB...') - self.reader = Reader(self.dbfile) + self.reader = None + self.reader_manager(action='open') + + def reader_manager(self, action=None): + if action == 'open': + self.reader = Reader(self.dbfile) + else: + self.reader.close() def lookup(self, ipaddress): ip = ipaddress @@ -47,8 +54,10 @@ class GeoIPHandler(object): if dbdate < first_wednesday_date < today: self.logger.info("Newer GeoLite2 DB available, Updating...") + self.reader_manager(action='close') remove(self.dbfile) self.download() + self.reader_manager(action='open') else: td = first_wednesday_date - today if td.days < 0: From 1cef8d2e60a31263d7269079c34fe59e9edacfc8 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 12 Feb 2019 12:56:40 -0600 Subject: [PATCH 07/20] change order --- varken/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index 8657d43..77fb542 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -21,11 +21,11 @@ class GeoIPHandler(object): self.data_folder = data_folder self.dbfile = abspath(join(self.data_folder, 'GeoLite2-City.mmdb')) self.logger = getLogger() + self.reader = None + self.reader_manager(action='open') self.update() self.logger.info('Opening persistent connection to GeoLite2 DB...') - self.reader = None - self.reader_manager(action='open') def reader_manager(self, action=None): if action == 'open': From b7e8d6e3fc8e6efb3b65e44abe544403ae2b6031 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Tue, 12 Feb 2019 13:02:00 -0600 Subject: [PATCH 08/20] keep retrying geolite database download. Fixes #103 --- varken/helpers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index 77fb542..34ab940 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -2,6 +2,7 @@ from hashlib import md5 from datetime import date from logging import getLogger from ipaddress import IPv4Address +from urllib.error import HTTPError from calendar import monthcalendar from geoip2.database import Reader from tarfile import open as taropen @@ -68,9 +69,15 @@ class GeoIPHandler(object): def download(self): tar_dbfile = abspath(join(self.data_folder, 'GeoLite2-City.tar.gz')) url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz' + downloaded = False - self.logger.info('Downloading GeoLite2 from %s', url) - urlretrieve(url, tar_dbfile) + while not downloaded: + self.logger.info('Downloading GeoLite2 from %s', url) + try: + urlretrieve(url, tar_dbfile) + downloaded = True + except HTTPError as e: + self.logger.error('Problem downloading new MaxMind DB... Trying again. Error: %s', e) self.logger.debug('Opening GeoLite2 tar file : %s', tar_dbfile) From b0bbe431d5b0beee4001ca8dbfea4764b60c74eb Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Wed, 6 Mar 2019 08:53:36 -0800 Subject: [PATCH 09/20] Add new Tautulli structures #111 --- varken/structures.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/varken/structures.py b/varken/structures.py index b210180..cf3e984 100644 --- a/varken/structures.py +++ b/varken/structures.py @@ -455,3 +455,5 @@ class TautulliStream(NamedTuple): width: str = None writers: list = None year: str = None + secure: str = None + relayed: str = None From c704bd97ed2fabc0a36d9ebbcb6eb96c2dbe68e5 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 12:47:13 -0400 Subject: [PATCH 10/20] Remove print in Varken.py --- Varken.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Varken.py b/Varken.py index a47feff..1c0c7f2 100644 --- a/Varken.py +++ b/Varken.py @@ -32,7 +32,6 @@ def thread(): while schedule.jobs: job = QUEUE.get() a = job() - print(a) if a is not None: schedule.clear(a) QUEUE.task_done() From 3796b62d1888f13ff34e4be914f1cdeec0cf7cdd Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 14:08:10 -0400 Subject: [PATCH 11/20] Rework DB downloader fixes #113 --- varken/helpers.py | 54 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index 34ab940..ac39da4 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -1,5 +1,6 @@ from hashlib import md5 -from datetime import date +from datetime import date, timedelta +from time import sleep from logging import getLogger from ipaddress import IPv4Address from urllib.error import HTTPError @@ -24,13 +25,21 @@ class GeoIPHandler(object): self.logger = getLogger() self.reader = None self.reader_manager(action='open') - self.update() self.logger.info('Opening persistent connection to GeoLite2 DB...') def reader_manager(self, action=None): if action == 'open': - self.reader = Reader(self.dbfile) + try: + self.reader = Reader(self.dbfile) + except FileNotFoundError: + self.logger.error("Could not find GeoLite2 DB! Downloading!") + result_status = self.download() + if result_status: + self.logger.error("Could not download GeoLite2 DB!!!, You may need to manually install it.") + exit(1) + else: + self.reader = Reader(self.dbfile) else: self.reader.close() @@ -45,41 +54,62 @@ class GeoIPHandler(object): try: dbdate = date.fromtimestamp(stat(self.dbfile).st_mtime) + db_next_update = date.fromtimestamp(stat(self.dbfile).st_mtime) + timedelta(days=60) + except FileNotFoundError: self.logger.error("Could not find GeoLite2 DB as: %s", self.dbfile) self.download() dbdate = date.fromtimestamp(stat(self.dbfile).st_mtime) + db_next_update = date.fromtimestamp(stat(self.dbfile).st_mtime) + timedelta(days=60) - first_wednesday_day = [week[2:3][0] for week in monthcalendar(today.year, today.month) if week[2:3][0] != 0][0] - first_wednesday_date = date(today.year, today.month, first_wednesday_day) - if dbdate < first_wednesday_date < today: + if db_next_update < today: self.logger.info("Newer GeoLite2 DB available, Updating...") + self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", + dbdate, db_next_update, today) self.reader_manager(action='close') - remove(self.dbfile) self.download() self.reader_manager(action='open') else: - td = first_wednesday_date - today + td = dbdate - today if td.days < 0: - self.logger.debug('Geolite2 DB is only %s days old. Keeping current copy', abs(td.days)) + self.logger.debug("Geolite2 DB is only %s days old. Keeping current copy. Next update after %s", + abs(td.days), db_next_update) + self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", + dbdate, db_next_update, today) else: - self.logger.debug('Geolite2 DB will update in %s days', abs(td.days)) + self.logger.debug("Geolite2 DB will update in %s days", abs(td.days)) + self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", + dbdate, db_next_update, today) + def download(self): tar_dbfile = abspath(join(self.data_folder, 'GeoLite2-City.tar.gz')) url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz' downloaded = False + retry_counter = 0 + while not downloaded: self.logger.info('Downloading GeoLite2 from %s', url) try: urlretrieve(url, tar_dbfile) downloaded = True except HTTPError as e: - self.logger.error('Problem downloading new MaxMind DB... Trying again. Error: %s', e) + self.logger.error("Problem downloading new GeoLite2 DB... Trying again. Error: %s", e) + sleep(2) + retry_counter = (retry_counter + 1) - self.logger.debug('Opening GeoLite2 tar file : %s', tar_dbfile) + if retry_counter >= 3: + self.logger.error("Retried downloading the new GeoLite2 DB 3 times and failed... Aborting!") + result_status = 1 + return result_status + try: + remove(self.dbfile) + except FileNotFoundError: + self.logger.warn("Cannot remove GeoLite2 DB as it does not exsist!") + + self.logger.debug("Opening GeoLite2 tar file : %s", tar_dbfile) tar = taropen(tar_dbfile, 'r:gz') From 2ac3ca584c7a69860873278f371367ff36ab4173 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 14:16:34 -0400 Subject: [PATCH 12/20] Blank lines removed --- varken/helpers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index ac39da4..3ad5324 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -62,7 +62,6 @@ class GeoIPHandler(object): dbdate = date.fromtimestamp(stat(self.dbfile).st_mtime) db_next_update = date.fromtimestamp(stat(self.dbfile).st_mtime) + timedelta(days=60) - if db_next_update < today: self.logger.info("Newer GeoLite2 DB available, Updating...") self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", @@ -82,7 +81,6 @@ class GeoIPHandler(object): self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", dbdate, db_next_update, today) - def download(self): tar_dbfile = abspath(join(self.data_folder, 'GeoLite2-City.tar.gz')) url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz' From 52e852b93010ac4b359b954392a6a888ca2b00be Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 14:18:10 -0400 Subject: [PATCH 13/20] Unused import removed --- varken/helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/varken/helpers.py b/varken/helpers.py index 3ad5324..05f6978 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -4,7 +4,6 @@ from time import sleep from logging import getLogger from ipaddress import IPv4Address from urllib.error import HTTPError -from calendar import monthcalendar from geoip2.database import Reader from tarfile import open as taropen from urllib3 import disable_warnings From c72a6b2c14202f2fa60ea5a39258f826b78151fa Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 16:36:44 -0400 Subject: [PATCH 14/20] Clean up logging logic for DB check --- varken/helpers.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/varken/helpers.py b/varken/helpers.py index 05f6978..459b51b 100644 --- a/varken/helpers.py +++ b/varken/helpers.py @@ -69,16 +69,10 @@ class GeoIPHandler(object): self.download() self.reader_manager(action='open') else: - td = dbdate - today - if td.days < 0: - self.logger.debug("Geolite2 DB is only %s days old. Keeping current copy. Next update after %s", - abs(td.days), db_next_update) - self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", - dbdate, db_next_update, today) - else: - self.logger.debug("Geolite2 DB will update in %s days", abs(td.days)) - self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", - dbdate, db_next_update, today) + db_days_update = db_next_update - today + self.logger.debug("Geolite2 DB will update in %s days", abs(db_days_update.days)) + self.logger.debug("GeoLite2 DB date %s, DB updates after: %s, Today: %s", + dbdate, db_next_update, today) def download(self): tar_dbfile = abspath(join(self.data_folder, 'GeoLite2-City.tar.gz')) @@ -117,7 +111,11 @@ class GeoIPHandler(object): tar.extract(files, self.data_folder) self.logger.debug('%s has been extracted to %s', files, self.data_folder) tar.close() - remove(tar_dbfile) + try: + remove(tar_dbfile) + self.logger.debug('Removed the GeoLite2 DB TAR file.') + except FileNotFoundError: + self.logger.warn("Cannot remove GeoLite2 DB TAR file as it does not exsist!") def hashit(string): From a5705b764bc8ffd73f59236922bd6ce382c2972d Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 18:36:33 -0400 Subject: [PATCH 15/20] Version bump --- varken/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varken/__init__.py b/varken/__init__.py index 1422b2e..7bdf047 100644 --- a/varken/__init__.py +++ b/varken/__init__.py @@ -1,2 +1,2 @@ -VERSION = "1.6.4" +VERSION = "1.6.5" BRANCH = 'master' From 6e292cc767cb7c3167d57365a58ba1b8fc8f30f4 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 18:39:11 -0400 Subject: [PATCH 16/20] Relayed is an integer --- varken/structures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varken/structures.py b/varken/structures.py index cf3e984..0957e44 100644 --- a/varken/structures.py +++ b/varken/structures.py @@ -456,4 +456,4 @@ class TautulliStream(NamedTuple): writers: list = None year: str = None secure: str = None - relayed: str = None + relayed: int = None From 07e4adbdb851b6c46b084b92dc342723d49f4922 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 18:46:50 -0400 Subject: [PATCH 17/20] Add new "relayed" and "secure" to Tautulli data pushed to influx #114 --- varken/tautulli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/varken/tautulli.py b/varken/tautulli.py index 579cd8f..7dea8a4 100644 --- a/varken/tautulli.py +++ b/varken/tautulli.py @@ -140,6 +140,8 @@ class TautulliAPI(object): "longitude": longitude, "player_state": player_state, "device_type": session.platform, + "relayed": session.relayed, + "secure": session.secure, "server": self.server.id }, "time": now, From 47479a93119523867216f91be9c0e314406d3da8 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 18:54:03 -0400 Subject: [PATCH 18/20] Update README for master buiild status --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bb61fe5..8f7e31f 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@

- -[![Build Status](https://jenkins.cajun.pro/buildStatus/icon?job=Varken/master)](https://jenkins.cajun.pro/job/Varken/develop) + +[![Build Status](https://jenkins.cajun.pro/buildStatus/icon?job=Varken/master)](https://jenkins.cajun.pro/job/Varken/job/master/) [![Discord](https://img.shields.io/discord/518970285773422592.svg?colorB=7289DA&label=Discord&logo=Discord&logoColor=7289DA&style=flat-square)](https://discord.gg/VjZ6qSM) [![BuyMeACoffee](https://img.shields.io/badge/BuyMeACoffee-Donate-ff813f.svg?logo=CoffeeScript&style=flat-square)](https://www.buymeacoffee.com/varken) [![Docker-Layers](https://images.microbadger.com/badges/image/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken) From 9062d3c2d4a4be8229940f9922e14802e34897f5 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 19:35:44 -0400 Subject: [PATCH 19/20] Update CHANGELOG --- CHANGELOG.md | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0756b42..30b0503 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,29 @@ # Change Log -## [v1.6.4](https://github.com/Boerderij/Varken/tree/v1.6.4) (2019-02-03) -[Full Changelog](https://github.com/Boerderij/Varken/compare/1.6.3...v1.6.4) +## [v1.6.5](https://github.com/Boerderij/Varken/tree/v1.6.5) (2019-03-11) +[Full Changelog](https://github.com/Boerderij/Varken/compare/v1.6.4...v1.6.5) + +**Implemented enhancements:** + +- \[Feature Request\] Add new "relayed" and "secure" to Tautulli data pushed to influx [\#114](https://github.com/Boerderij/Varken/issues/114) +- \[BUG\] Changes to Tautulli breaks Varken `TypeError` `Secure` `relayed` [\#111](https://github.com/Boerderij/Varken/issues/111) + +**Fixed bugs:** + +- \[BUG\] Handle GeoIP Downloads better [\#113](https://github.com/Boerderij/Varken/issues/113) +- \[BUG\] - "None" outputted to stdout many times with no benefit? [\#105](https://github.com/Boerderij/Varken/issues/105) +- \[BUG\] windows file open error [\#104](https://github.com/Boerderij/Varken/issues/104) +- \[BUG\] Not catching DB url resolve [\#103](https://github.com/Boerderij/Varken/issues/103) + +**Merged pull requests:** + +- v1.6.5 Merge [\#115](https://github.com/Boerderij/Varken/pull/115) ([samwiseg0](https://github.com/samwiseg0)) + +## [v1.6.4](https://github.com/Boerderij/Varken/tree/v1.6.4) (2019-02-04) +[Full Changelog](https://github.com/Boerderij/Varken/compare/1.6.4...v1.6.4) + +## [1.6.4](https://github.com/Boerderij/Varken/tree/1.6.4) (2019-02-04) +[Full Changelog](https://github.com/Boerderij/Varken/compare/1.6.3...1.6.4) **Fixed bugs:** From e9a0b72be3b81151600db0f9f29ef6a2aa570365 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Mon, 11 Mar 2019 19:37:12 -0400 Subject: [PATCH 20/20] Remove dupe --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30b0503..93351f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,9 +19,6 @@ - v1.6.5 Merge [\#115](https://github.com/Boerderij/Varken/pull/115) ([samwiseg0](https://github.com/samwiseg0)) -## [v1.6.4](https://github.com/Boerderij/Varken/tree/v1.6.4) (2019-02-04) -[Full Changelog](https://github.com/Boerderij/Varken/compare/1.6.4...v1.6.4) - ## [1.6.4](https://github.com/Boerderij/Varken/tree/1.6.4) (2019-02-04) [Full Changelog](https://github.com/Boerderij/Varken/compare/1.6.3...1.6.4)