From 2100769a1bababb7b049540f0a302168679b28ce Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Wed, 16 Jan 2019 12:40:33 -0600 Subject: [PATCH 01/20] final tweak --- Dockerfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5b0194b..04e8806 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,11 +6,7 @@ ENV DEBUG="False" COPY / /app -RUN \ - python3 -m pip install -r /app/requirements.txt && \ - chown -R abc:abc \ - /config \ - /app +RUN python3 -m pip install -r /app/requirements.txt WORKDIR /app From 75fef0888b30d311fb84830b69683db8e3cebb94 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Wed, 16 Jan 2019 12:42:37 -0600 Subject: [PATCH 02/20] final tweak2 --- Dockerfile.arm64 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index 28f1c42..5922d9a 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -6,7 +6,7 @@ ENV DEBUG="False" COPY / /app -COPY /tmp/qemu-arm-static /usr/bin/qemu-aarch64-static +COPY /tmp/qemu-aarch64-static /usr/bin/qemu-aarch64-static RUN python3 -m pip install -r /app/requirements.txt From 46c656219f72917447347a899e9b054ea2a9b831 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Wed, 16 Jan 2019 12:44:18 -0600 Subject: [PATCH 03/20] typos for days --- manifest_generator.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 manifest_generator.py diff --git a/manifest_generator.py b/manifest_generator.py new file mode 100644 index 0000000..db200db --- /dev/null +++ b/manifest_generator.py @@ -0,0 +1,37 @@ +import yaml + +from varken import VERSION + +org = 'boerderij' +project = 'varken' +namespace = f"{org}/{project}" + +yaml_arr = [] +tags = ['latest', VERSION] + +# Docker image, arch, variant, os +arch_list = [('arm', 'arm', 'v6', 'linux'), + ('armhf', 'arm', 'v7', 'linux'), + ('arm64', 'arm64', 'v8', 'linux'), + ('amd64', 'amd64', None, 'linux')] + +for tag in tags: + yaml_doc = { + 'image': f'{namespace}:{tag}', + 'manifests': [] + } + for arch in arch_list: + info = { + 'image': f"{namespace}:{tag}-{arch[0]}", + 'platform': { + 'architecture': arch[1], + 'os': arch[3] + } + } + if arch[2]: + info['platform']['variant'] = arch[2] + yaml_doc['manifests'].append(info) + yaml_arr.append(yaml_doc) + +with open(f".manifest.yaml", 'w') as file: + yaml.dump_all(yaml_arr, file, default_flow_style=False) From 106e0534b173200ac43d4189809b95d94a09ceaa Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Wed, 16 Jan 2019 13:12:57 -0600 Subject: [PATCH 04/20] cleanup for next version --- deploy.sh | 10 ++++++---- manifest_generator.py | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/deploy.sh b/deploy.sh index 6944546..c5ae409 100644 --- a/deploy.sh +++ b/deploy.sh @@ -19,13 +19,15 @@ echo "$DOCKER_PASSWORD" | docker login -u="$DOCKER_USER" --password-stdin # Latest x64 docker build -t "${NAMESPACE}:latest" . && \ -docker push "${NAMESPACE}:latest" && \ -# Versioned x64 -docker tag "${NAMESPACE}:latest" "${NAMESPACE}:${VERSION}" && \ -docker push "${NAMESPACE}:${VERSION}" && \ +docker push "${NAMESPACE}:latest" # x64 Arch docker tag "${NAMESPACE}:latest" "${NAMESPACE}:latest-amd64" && \ docker push "${NAMESPACE}:latest-amd64" +# Versioned x64 +docker tag "${NAMESPACE}:latest" "${NAMESPACE}:${VERSION}" && \ +docker push "${NAMESPACE}:${VERSION}" +docker tag "${NAMESPACE}:latest" "${NAMESPACE}:${VERSION}-amd64" && \ +docker push "${NAMESPACE}:${VERSION}-amd64" # Prepare qemu for ARM builds docker run --rm --privileged multiarch/qemu-user-static:register --reset diff --git a/manifest_generator.py b/manifest_generator.py index db200db..d429f50 100644 --- a/manifest_generator.py +++ b/manifest_generator.py @@ -4,7 +4,7 @@ from varken import VERSION org = 'boerderij' project = 'varken' -namespace = f"{org}/{project}" +namespace = "{}/{}".format(org, project) yaml_arr = [] tags = ['latest', VERSION] @@ -17,12 +17,12 @@ arch_list = [('arm', 'arm', 'v6', 'linux'), for tag in tags: yaml_doc = { - 'image': f'{namespace}:{tag}', + 'image': '{}:{}'.format(namespace, tag), 'manifests': [] } for arch in arch_list: info = { - 'image': f"{namespace}:{tag}-{arch[0]}", + 'image': "{}:{}-{}".format(namespace, tag, arch[0]), 'platform': { 'architecture': arch[1], 'os': arch[3] From 088015653c188117eef31335de744f0b095f5c9e Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Wed, 16 Jan 2019 13:13:16 -0600 Subject: [PATCH 05/20] version bump --- varken/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/varken/__init__.py b/varken/__init__.py index 196ac44..4847e96 100644 --- a/varken/__init__.py +++ b/varken/__init__.py @@ -1,2 +1,2 @@ -VERSION = "1.6.3" -BRANCH = 'master' +VERSION = "1.6.4" +BRANCH = 'nightly' From ad2f124b61b5d63f5745d2fc85049c3218727211 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Wed, 16 Jan 2019 23:44:48 -0600 Subject: [PATCH 06/20] fixed deploy script logic --- .travis.yml | 4 ++ deploy.sh | 108 +++++++++++++++++++++++++++++++++++----------------- 2 files changed, 78 insertions(+), 34 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0c1436a..b557acc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,3 +15,7 @@ deploy: script: bash deploy.sh on: branch: master + - provider: script + script: bash deploy.sh + on: + branch: develop \ No newline at end of file diff --git a/deploy.sh b/deploy.sh index c5ae409..b924a54 100644 --- a/deploy.sh +++ b/deploy.sh @@ -1,11 +1,24 @@ #!/usr/bin/env bash +# Travis-ci convenience environment vars used: +# TRAVIS_BRANCH | branch name +# $TRAVIS_REPO_SLUG | organization/project (GitHub Capitalization) +# Travis-ci manual environment vars used: +# GITHUB_USER | github username +# GITHUB_TOKEN | $GITHUB_USER's token +# DOCKER_USER | docker username +# DOCKER_PASSWORD | $DOCKER_USER's password + VERSION="$(grep -i version varken/__init__.py | cut -d' ' -f3 | tr -d \")" -# Docker -GITHUB_USER='dirtycajunrice' -DOCKER_USER='dirtycajunrice' -PROJECT='varken' -NAMESPACE="boerderij/${PROJECT}" +# Set branch to latest if master, else keep the same +if [[ "$TRAVIS_BRANCH" == "master" ]]; then + BRANCH="latest" +else + BRANCH="$TRAVIS_BRANCH" +fi + +# get the docker lowercase variant of the repo_name +REPOSITORY="$(echo $TRAVIS_REPO_SLUG | tr '[:upper:]' '[:lower:]')" # Docker experimental config echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json @@ -15,42 +28,69 @@ echo '{"experimental":"enabled"}' | sudo tee ~/.docker/config.json sudo service docker restart # Auth -echo "$DOCKER_PASSWORD" | docker login -u="$DOCKER_USER" --password-stdin +echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USER" --password-stdin -# Latest x64 -docker build -t "${NAMESPACE}:latest" . && \ -docker push "${NAMESPACE}:latest" -# x64 Arch -docker tag "${NAMESPACE}:latest" "${NAMESPACE}:latest-amd64" && \ -docker push "${NAMESPACE}:latest-amd64" -# Versioned x64 -docker tag "${NAMESPACE}:latest" "${NAMESPACE}:${VERSION}" && \ -docker push "${NAMESPACE}:${VERSION}" -docker tag "${NAMESPACE}:latest" "${NAMESPACE}:${VERSION}-amd64" && \ -docker push "${NAMESPACE}:${VERSION}-amd64" - -# Prepare qemu for ARM builds +# Prepare QEMU for ARM builds docker run --rm --privileged multiarch/qemu-user-static:register --reset wget -P tmp/ "https://github.com/multiarch/qemu-user-static/releases/download/v3.1.0-2/qemu-aarch64-static" wget -P tmp/ "https://github.com/multiarch/qemu-user-static/releases/download/v3.1.0-2/qemu-arm-static" chmod +x tmp/qemu-aarch64-static tmp/qemu-arm-static -# ARM images +# Set tag based off of branch +if [[ "$BRANCH" == "latest" ]]; then + TAG="$VERSION" +else + TAG="$BRANCH" +fi + +# AMDx64 +docker build -t "${REPOSITORY}:${TAG}-amd64" . && \ +docker push "${REPOSITORY}:${TAG}-amd64" + +# Create Initial Manifests +docker manifest create "${REPOSITORY}:${TAG}" "${REPOSITORY}:${TAG}-amd64" +if [[ "$BRANCH" == "latest" ]]; then + docker manifest create "${REPOSITORY}:${BRANCH}" "${REPOSITORY}:${TAG}-amd64" +fi + +# ARM variants for i in $(ls *arm*); do - arch="$(echo ${i} | cut -d. -f2)" - # Latest - docker build -f "./Dockerfile.${arch}" -t "${NAMESPACE}:latest-${arch}" . && \ - docker push "${NAMESPACE}:latest-${arch}" && \ - # Versioned - docker tag "${NAMESPACE}:latest-${arch}" "${NAMESPACE}:${VERSION}-${arch}" && \ - docker push "${NAMESPACE}:${VERSION}-${arch}" + ARCH="$(echo ${i} | cut -d. -f2)" + docker build -t "${REPOSITORY}:${TAG}-${ARCH}" . && \ + docker push "${REPOSITORY}:${TAG}-${ARCH}" + # Add variant to manifest + docker manifest create -a "${REPOSITORY}:${TAG}" "${REPOSITORY}:${TAG}-${ARCH}" + if [[ "$BRANCH" == "latest" ]]; then + docker manifest create -a "${REPOSITORY}:${BRANCH}" "${REPOSITORY}:${TAG}-${ARCH}" + fi + if [[ "$ARCH" == "arm64" ]]; then + docker manifest annotate "${REPOSITORY}:${TAG}" "${REPOSITORY}:${TAG}-${ARCH}" --variant v8 --arch arm64 + if [[ "$BRANCH" == "latest" ]]; then + docker manifest annotate "${REPOSITORY}:${BRANCH}" "${REPOSITORY}:${TAG}-${ARCH}" --variant v8 --arch arm64 + fi + elif [[ "$ARCH" == "armhf" ]]; then + docker manifest annotate "${REPOSITORY}:${TAG}" "${REPOSITORY}:${TAG}-${ARCH}" --variant v7 --arch arm + if [[ "$BRANCH" == "latest" ]]; then + docker manifest annotate "${REPOSITORY}:${BRANCH}" "${REPOSITORY}:${TAG}-${ARCH}" --variant v7 --arch arm + fi + elif [[ "$ARCH" == "arm" ]]; then + docker manifest annotate "${REPOSITORY}:${TAG}" "${REPOSITORY}:${TAG}-${ARCH}" --variant v6 --arch arm + if [[ "$BRANCH" == "latest" ]]; then + docker manifest annotate "${REPOSITORY}:${BRANCH}" "${REPOSITORY}:${TAG}-${ARCH}" --variant v6 --arch arm + fi + fi done -wget -O manifest-tool https://github.com/estesp/manifest-tool/releases/download/v0.9.0/manifest-tool-linux-amd64 && \ -chmod +x manifest-tool && \ -python3 manifest_generator.py && \ -./manifest-tool --username "$DOCKER_USER" --password "$DOCKER_PASSWORD" push from-spec ".manifest.yaml" +docker manifest inspect "${REPOSITORY}:${TAG}" && \ +docker manifest push "${REPOSITORY}:${TAG}" +if [[ "$BRANCH" == "latest" ]]; then + docker manifest inspect "${REPOSITORY}:${BRANCH}" && \ + docker manifest push "${REPOSITORY}:${BRANCH}" +fi + # Git tags -git remote set-url origin "https://${GITHUB_USER}:${GITHUB_API_KEY}@github.com/${NAMESPACE}.git" && \ -git tag "${VERSION}" && \ -git push --tags \ No newline at end of file +if [[ "$BRANCH" == "latest" ]]; then + git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${REPOSITORY}.git" && \ + git tag "${VERSION}" && \ + git push --tags +fi \ No newline at end of file From 502fd819a39ecfd2a3032873754ecd34a212641f Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Wed, 16 Jan 2019 23:52:22 -0600 Subject: [PATCH 07/20] remove restriction --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b557acc..0dd0518 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,6 @@ addons: apt: packages: - docker-ce -branches: - only: - - master deploy: - provider: script script: bash deploy.sh From 98794b3e7bd0fec32bcad5de88b0b16c83d0be87 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Wed, 16 Jan 2019 23:58:12 -0600 Subject: [PATCH 08/20] add file specification so it doesnt repeatedly build the same one --- deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy.sh b/deploy.sh index b924a54..830096c 100644 --- a/deploy.sh +++ b/deploy.sh @@ -56,7 +56,7 @@ fi # ARM variants for i in $(ls *arm*); do ARCH="$(echo ${i} | cut -d. -f2)" - docker build -t "${REPOSITORY}:${TAG}-${ARCH}" . && \ + docker build -f "Dockerfile.${ARCH}" -t "${REPOSITORY}:${TAG}-${ARCH}" . && \ docker push "${REPOSITORY}:${TAG}-${ARCH}" # Add variant to manifest docker manifest create -a "${REPOSITORY}:${TAG}" "${REPOSITORY}:${TAG}-${ARCH}" From 0efdf9595ffb59b84005ca9d2057aab1cf7e210f Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Fri, 18 Jan 2019 11:51:26 -0500 Subject: [PATCH 09/20] Add requestedByAlias to Ombi structures #97 --- varken/structures.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/varken/structures.py b/varken/structures.py index 9c081da..b210180 100644 --- a/varken/structures.py +++ b/varken/structures.py @@ -138,6 +138,7 @@ class OmbiTVRequest(NamedTuple): title: str = None totalSeasons: int = None tvDbId: int = None + requestedByAlias: str = None class OmbiMovieRequest(NamedTuple): @@ -173,6 +174,7 @@ class OmbiMovieRequest(NamedTuple): title: str = None langCode: str = None languageCode: str = None + requestedByAlias: str = None # Sonarr From 3896f5242bd2d6cf06c9208b62fe1d0e9c3e40ec Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Fri, 25 Jan 2019 12:34:29 -0600 Subject: [PATCH 10/20] refactor docker deployment --- Dockerfile | 10 +++++++--- Dockerfile.arm | 12 ++++++++---- Dockerfile.arm64 | 12 ++++++++---- Dockerfile.armhf | 17 ----------------- deploy.sh | 21 ++------------------- manifest_generator.py | 37 ------------------------------------- prebuild.sh | 3 +++ varken/__init__.py | 2 +- 8 files changed, 29 insertions(+), 85 deletions(-) delete mode 100644 Dockerfile.armhf delete mode 100644 manifest_generator.py create mode 100644 prebuild.sh diff --git a/Dockerfile b/Dockerfile index 04e8806..8c84f9c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,12 +4,16 @@ LABEL maintainers="dirtycajunrice,samwiseg0" ENV DEBUG="False" -COPY / /app +WORKDIR /app + +COPY /requirements.txt /Varken.py /app/ + +COPY /varken /app/varken + +COPY /data /app/data RUN python3 -m pip install -r /app/requirements.txt -WORKDIR /app - CMD cp /app/data/varken.example.ini /config/varken.example.ini && python3 /app/Varken.py --data-folder /config VOLUME /config diff --git a/Dockerfile.arm b/Dockerfile.arm index 5d5624e..3bcad32 100644 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -4,14 +4,18 @@ LABEL maintainers="dirtycajunrice,samwiseg0" ENV DEBUG="False" -COPY / /app +WORKDIR /app -COPY /tmp/qemu-arm-static /usr/bin/qemu-arm-static +COPY /qemu-arm-static /usr/bin/qemu-arm-static + +COPY /requirements.txt /Varken.py /app/ + +COPY /varken /app/varken + +COPY /data /app/data RUN python3 -m pip install -r /app/requirements.txt -WORKDIR /app - CMD cp /app/data/varken.example.ini /config/varken.example.ini && python3 /app/Varken.py --data-folder /config VOLUME /config \ No newline at end of file diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index 5922d9a..d18a154 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -4,14 +4,18 @@ LABEL maintainers="dirtycajunrice,samwiseg0" ENV DEBUG="False" -COPY / /app +WORKDIR /app -COPY /tmp/qemu-aarch64-static /usr/bin/qemu-aarch64-static +COPY /qemu-aarch64-static /usr/bin/qemu-aarch64-static + +COPY /requirements.txt /Varken.py /app/ + +COPY /varken /app/varken + +COPY /data /app/data RUN python3 -m pip install -r /app/requirements.txt -WORKDIR /app - CMD cp /app/data/varken.example.ini /config/varken.example.ini && python3 /app/Varken.py --data-folder /config VOLUME /config \ No newline at end of file diff --git a/Dockerfile.armhf b/Dockerfile.armhf deleted file mode 100644 index 4ed529a..0000000 --- a/Dockerfile.armhf +++ /dev/null @@ -1,17 +0,0 @@ -FROM arm32v7/python:3.7.2-slim - -LABEL maintainers="dirtycajunrice,samwiseg0" - -ENV DEBUG="False" - -COPY / /app - -COPY /tmp/qemu-arm-static /usr/bin/qemu-arm-static - -RUN python3 -m pip install -r /app/requirements.txt - -WORKDIR /app - -CMD cp /app/data/varken.example.ini /config/varken.example.ini && python3 /app/Varken.py --data-folder /config - -VOLUME /config \ No newline at end of file diff --git a/deploy.sh b/deploy.sh index 830096c..d2feb82 100644 --- a/deploy.sh +++ b/deploy.sh @@ -32,9 +32,8 @@ echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USER" --password-stdin # Prepare QEMU for ARM builds docker run --rm --privileged multiarch/qemu-user-static:register --reset -wget -P tmp/ "https://github.com/multiarch/qemu-user-static/releases/download/v3.1.0-2/qemu-aarch64-static" -wget -P tmp/ "https://github.com/multiarch/qemu-user-static/releases/download/v3.1.0-2/qemu-arm-static" -chmod +x tmp/qemu-aarch64-static tmp/qemu-arm-static +bash prebuild.sh +chmod +x qemu-aarch64-static qemu-arm-static # Set tag based off of branch if [[ "$BRANCH" == "latest" ]]; then @@ -63,22 +62,6 @@ for i in $(ls *arm*); do if [[ "$BRANCH" == "latest" ]]; then docker manifest create -a "${REPOSITORY}:${BRANCH}" "${REPOSITORY}:${TAG}-${ARCH}" fi - if [[ "$ARCH" == "arm64" ]]; then - docker manifest annotate "${REPOSITORY}:${TAG}" "${REPOSITORY}:${TAG}-${ARCH}" --variant v8 --arch arm64 - if [[ "$BRANCH" == "latest" ]]; then - docker manifest annotate "${REPOSITORY}:${BRANCH}" "${REPOSITORY}:${TAG}-${ARCH}" --variant v8 --arch arm64 - fi - elif [[ "$ARCH" == "armhf" ]]; then - docker manifest annotate "${REPOSITORY}:${TAG}" "${REPOSITORY}:${TAG}-${ARCH}" --variant v7 --arch arm - if [[ "$BRANCH" == "latest" ]]; then - docker manifest annotate "${REPOSITORY}:${BRANCH}" "${REPOSITORY}:${TAG}-${ARCH}" --variant v7 --arch arm - fi - elif [[ "$ARCH" == "arm" ]]; then - docker manifest annotate "${REPOSITORY}:${TAG}" "${REPOSITORY}:${TAG}-${ARCH}" --variant v6 --arch arm - if [[ "$BRANCH" == "latest" ]]; then - docker manifest annotate "${REPOSITORY}:${BRANCH}" "${REPOSITORY}:${TAG}-${ARCH}" --variant v6 --arch arm - fi - fi done docker manifest inspect "${REPOSITORY}:${TAG}" && \ diff --git a/manifest_generator.py b/manifest_generator.py deleted file mode 100644 index d429f50..0000000 --- a/manifest_generator.py +++ /dev/null @@ -1,37 +0,0 @@ -import yaml - -from varken import VERSION - -org = 'boerderij' -project = 'varken' -namespace = "{}/{}".format(org, project) - -yaml_arr = [] -tags = ['latest', VERSION] - -# Docker image, arch, variant, os -arch_list = [('arm', 'arm', 'v6', 'linux'), - ('armhf', 'arm', 'v7', 'linux'), - ('arm64', 'arm64', 'v8', 'linux'), - ('amd64', 'amd64', None, 'linux')] - -for tag in tags: - yaml_doc = { - 'image': '{}:{}'.format(namespace, tag), - 'manifests': [] - } - for arch in arch_list: - info = { - 'image': "{}:{}-{}".format(namespace, tag, arch[0]), - 'platform': { - 'architecture': arch[1], - 'os': arch[3] - } - } - if arch[2]: - info['platform']['variant'] = arch[2] - yaml_doc['manifests'].append(info) - yaml_arr.append(yaml_doc) - -with open(f".manifest.yaml", 'w') as file: - yaml.dump_all(yaml_arr, file, default_flow_style=False) diff --git a/prebuild.sh b/prebuild.sh new file mode 100644 index 0000000..8d12fd7 --- /dev/null +++ b/prebuild.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +wget -q "https://github.com/multiarch/qemu-user-static/releases/download/v3.1.0-2/qemu-aarch64-static" +wget -q "https://github.com/multiarch/qemu-user-static/releases/download/v3.1.0-2/qemu-arm-static" \ No newline at end of file diff --git a/varken/__init__.py b/varken/__init__.py index 4847e96..9ec9472 100644 --- a/varken/__init__.py +++ b/varken/__init__.py @@ -1,2 +1,2 @@ VERSION = "1.6.4" -BRANCH = 'nightly' +BRANCH = 'develop' From cb2b3ef2f6e289703085c589fc788234c7e92d48 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Fri, 25 Jan 2019 12:39:54 -0600 Subject: [PATCH 11/20] add logo to readme --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6bc4cbf..61ce25c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ -# Varken +

+ +

+ [![Build Status](https://travis-ci.org/Boerderij/Varken.svg?branch=master)](https://travis-ci.org/Boerderij/Varken) [![Discord](https://img.shields.io/badge/Discord-Varken-7289DA.svg?logo=discord&style=flat-square)](https://discord.gg/VjZ6qSM) [![BuyMeACoffee](https://img.shields.io/badge/BuyMeACoffee-Donate-ff813f.svg?logo=CoffeeScript&style=flat-square)](https://www.buymeacoffee.com/varken) From 5bbf9d0f0104930c517db8572fb4a9cf1ddb6dc2 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Fri, 25 Jan 2019 12:46:39 -0600 Subject: [PATCH 12/20] fix badges --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 61ce25c..1adc5e9 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,10 @@

[![Build Status](https://travis-ci.org/Boerderij/Varken.svg?branch=master)](https://travis-ci.org/Boerderij/Varken) -[![Discord](https://img.shields.io/badge/Discord-Varken-7289DA.svg?logo=discord&style=flat-square)](https://discord.gg/VjZ6qSM) +[![Discord](https://img.shields.io/discord/518970285773422592.svg?colorB=7289DA&label=Discord&logo=Discord&logoColor=7289DA&style=flat-square)](https://discord.gg/VjZ6qSM) [![BuyMeACoffee](https://img.shields.io/badge/BuyMeACoffee-Donate-ff813f.svg?logo=CoffeeScript&style=flat-square)](https://www.buymeacoffee.com/varken) [![Docker-Layers](https://images.microbadger.com/badges/image/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken) -[![Docker-Version](https://images.microbadger.com/badges/version/boerderij/varken.svg)](https://microbadger.com/images/boerderij/varken) +[![Release](https://img.shields.io/github/release/boerderij/varken.svg?style=flat-square)](https://microbadger.com/images/boerderij/varken) [![Docker Pulls](https://img.shields.io/docker/pulls/boerderij/varken.svg)](https://hub.docker.com/r/boerderij/varken/) Dutch for PIG. PIG is an Acronym for Plex/InfluxDB/Grafana From c4c1ace54fdb3834083a472546ce114c1e9bbf25 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Fri, 25 Jan 2019 13:53:44 -0600 Subject: [PATCH 13/20] check payload for unifi --- varken.xml | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ varken/unifi.py | 54 ++++++++++++++++++++++++++----------------------- 2 files changed, 83 insertions(+), 25 deletions(-) create mode 100644 varken.xml diff --git a/varken.xml b/varken.xml new file mode 100644 index 0000000..b7fbf3a --- /dev/null +++ b/varken.xml @@ -0,0 +1,54 @@ + + + Varken + boerderij/varken + https://hub.docker.com/r/boerderij/varken/~/dockerfile/ + bridge + + sh + false + https://discord.gg/VjZ6qSM + + + Varken is a standalone command-line utility to aggregate data from the Plex ecosystem into InfluxDB. Examples use Grafana for a frontend + + + + + Pig.png + + + + + + + Varken is a standalone command-line utility to aggregate data from the Plex ecosystem into InfluxDB. Examples use Grafana for a frontend + + + bridge + + + + + /mnt/user/appdata/varken + /config + rw + + + + + 99 + PGID + + + + 100 + PUID + + + + + 99 + 100 + /mnt/user/appdata/varken + \ No newline at end of file diff --git a/varken/unifi.py b/varken/unifi.py index 1cf3333..01f68ac 100644 --- a/varken/unifi.py +++ b/varken/unifi.py @@ -47,29 +47,33 @@ class UniFiAPI(object): self.logger.error("Could not find a USG named %s from your UniFi Controller", self.server.usg_name) return - influx_payload = [ - { - "measurement": "UniFi", - "tags": { - "model": device['model'], - "name": device['name'] - }, - "time": now, - "fields": { - "bytes_current": device['wan1']['bytes-r'], - "rx_bytes_total": device['wan1']['rx_bytes'], - "rx_bytes_current": device['wan1']['rx_bytes-r'], - "tx_bytes_total": device['wan1']['tx_bytes'], - "tx_bytes_current": device['wan1']['tx_bytes-r'], - "speedtest_latency": device['speedtest-status']['latency'], - "speedtest_download": device['speedtest-status']['xput_download'], - "speedtest_upload": device['speedtest-status']['xput_upload'], - "cpu_loadavg_1": device['sys_stats']['loadavg_1'], - "cpu_loadavg_5": device['sys_stats']['loadavg_5'], - "cpu_loadavg_15": device['sys_stats']['loadavg_15'], - "cpu_util": device['system-stats']['cpu'], - "mem_util": device['system-stats']['mem'], + try: + influx_payload = [ + { + "measurement": "UniFi", + "tags": { + "model": device['model'], + "name": device['name'] + }, + "time": now, + "fields": { + "bytes_current": device['wan1']['bytes-r'], + "rx_bytes_total": device['wan1']['rx_bytes'], + "rx_bytes_current": device['wan1']['rx_bytes-r'], + "tx_bytes_total": device['wan1']['tx_bytes'], + "tx_bytes_current": device['wan1']['tx_bytes-r'], + # Commenting speedtest out until Unifi gets their shit together + # "speedtest_latency": device['speedtest-status']['latency'], + # "speedtest_download": device['speedtest-status']['xput_download'], + # "speedtest_upload": device['speedtest-status']['xput_upload'], + "cpu_loadavg_1": device['sys_stats']['loadavg_1'], + "cpu_loadavg_5": device['sys_stats']['loadavg_5'], + "cpu_loadavg_15": device['sys_stats']['loadavg_15'], + "cpu_util": device['system-stats']['cpu'], + "mem_util": device['system-stats']['mem'], + } } - } - ] - self.dbmanager.write_points(influx_payload) + ] + self.dbmanager.write_points(influx_payload) + except KeyError as e: + self.logger.error('Error building payload for unifi. Discarding. Error: %s', e) From 371edb9f0234a9487b664e8c21bbbbd8f2fb984a Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Fri, 25 Jan 2019 13:57:27 -0600 Subject: [PATCH 14/20] check payload for sickchill --- varken/sickchill.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/varken/sickchill.py b/varken/sickchill.py index 7be6acf..39f1934 100644 --- a/varken/sickchill.py +++ b/varken/sickchill.py @@ -42,23 +42,27 @@ class SickChillAPI(object): sxe = f'S{show.season:0>2}E{show.episode:0>2}' hash_id = hashit(f'{self.server.id}{show.show_name}{sxe}') missing_types = [(0, 'future'), (1, 'later'), (2, 'soon'), (3, 'today'), (4, 'missed')] - influx_payload.append( - { - "measurement": "SickChill", - "tags": { - "type": [item[0] for item in missing_types if key in item][0], - "indexerid": show.indexerid, - "server": self.server.id, - "name": show.show_name, - "epname": show.ep_name, - "sxe": sxe, - "airdate": show.airdate, - }, - "time": now, - "fields": { - "hash": hash_id + try: + influx_payload.append( + { + "measurement": "SickChill", + "tags": { + "type": [item[0] for item in missing_types if key in item][0], + "indexerid": show.indexerid, + "server": self.server.id, + "name": show.show_name, + "epname": show.ep_name, + "sxe": sxe, + "airdate": show.airdate, + }, + "time": now, + "fields": { + "hash": hash_id + } } - } - ) + ) + except IndexError as e: + self.logger.error('Error building payload for sickchill. Discarding. Error: %s', e) - self.dbmanager.write_points(influx_payload) + if influx_payload: + self.dbmanager.write_points(influx_payload) From ec6f0ab35508c97db2d6fdd9d49b9ce2fb36ae6f Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Fri, 25 Jan 2019 14:11:51 -0600 Subject: [PATCH 15/20] add version output to dbmanager for influx --- varken.xml | 1 + varken/dbmanager.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/varken.xml b/varken.xml index b7fbf3a..b846c57 100644 --- a/varken.xml +++ b/varken.xml @@ -1,3 +1,4 @@ + Varken diff --git a/varken/dbmanager.py b/varken/dbmanager.py index e3c5322..2f076df 100644 --- a/varken/dbmanager.py +++ b/varken/dbmanager.py @@ -10,8 +10,10 @@ class DBManager(object): self.influx = InfluxDBClient(host=self.server.url, port=self.server.port, username=self.server.username, password=self.server.password, ssl=self.server.ssl, database='varken', verify_ssl=self.server.verify_ssl) + version = self.influx.request('ping', expected_response_code=204).headers['X-Influxdb-Version'] databases = [db['name'] for db in self.influx.get_list_database()] self.logger = getLogger() + self.logger.info('Influxdb version: %s', version) if 'varken' not in databases: self.logger.info("Creating varken database") From b29da07310557133fb7e32cb9e3dfe3848f172cc Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Fri, 25 Jan 2019 14:15:03 -0600 Subject: [PATCH 16/20] fstring to .format() in Varken.py for version check --- Varken.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Varken.py b/Varken.py index bd06a89..cd83c34 100644 --- a/Varken.py +++ b/Varken.py @@ -98,13 +98,13 @@ if __name__ == "__main__": SONARR = SonarrAPI(server, DBMANAGER) if server.queue: at_time = schedule.every(server.queue_run_seconds).seconds - at_time.do(QUEUE.put, SONARR.get_queue).tag(f"sonarr-{server.id}-get_queue") + at_time.do(QUEUE.put, SONARR.get_queue).tag("sonarr-{}-get_queue".format(server.id)) if server.missing_days > 0: at_time = schedule.every(server.missing_days_run_seconds).seconds - at_time.do(QUEUE.put, SONARR.get_missing).tag(f"sonarr-{server.id}-get_missing") + at_time.do(QUEUE.put, SONARR.get_missing).tag("sonarr-{}-get_missing".format(server.id)) if server.future_days > 0: at_time = schedule.every(server.future_days_run_seconds).seconds - at_time.do(QUEUE.put, SONARR.get_future).tag(f"sonarr-{server.id}-get_future") + at_time.do(QUEUE.put, SONARR.get_future).tag("sonarr-{}-get_future".format(server.id)) if CONFIG.tautulli_enabled: GEOIPHANDLER = GeoIPHandler(DATA_FOLDER) @@ -113,46 +113,46 @@ if __name__ == "__main__": TAUTULLI = TautulliAPI(server, DBMANAGER, GEOIPHANDLER) if server.get_activity: at_time = schedule.every(server.get_activity_run_seconds).seconds - at_time.do(QUEUE.put, TAUTULLI.get_activity).tag(f"tautulli-{server.id}-get_activity") + at_time.do(QUEUE.put, TAUTULLI.get_activity).tag("tautulli-{}-get_activity".format(server.id)) if server.get_stats: at_time = schedule.every(server.get_stats_run_seconds).seconds - at_time.do(QUEUE.put, TAUTULLI.get_stats).tag(f"tautulli-{server.id}-get_stats") + at_time.do(QUEUE.put, TAUTULLI.get_stats).tag("tautulli-{}-get_stats".format(server.id)) if CONFIG.radarr_enabled: for server in CONFIG.radarr_servers: RADARR = RadarrAPI(server, DBMANAGER) if server.get_missing: at_time = schedule.every(server.get_missing_run_seconds).seconds - at_time.do(QUEUE.put, RADARR.get_missing).tag(f"radarr-{server.id}-get_missing") + at_time.do(QUEUE.put, RADARR.get_missing).tag("radarr-{}-get_missing".format(server.id)) if server.queue: at_time = schedule.every(server.queue_run_seconds).seconds - at_time.do(QUEUE.put, RADARR.get_queue).tag(f"radarr-{server.id}-get_queue") + at_time.do(QUEUE.put, RADARR.get_queue).tag("radarr-{}-get_queue".format(server.id)) if CONFIG.ombi_enabled: for server in CONFIG.ombi_servers: OMBI = OmbiAPI(server, DBMANAGER) if server.request_type_counts: at_time = schedule.every(server.request_type_run_seconds).seconds - at_time.do(QUEUE.put, OMBI.get_request_counts).tag(f"ombi-{server.id}-get_request_counts") + at_time.do(QUEUE.put, OMBI.get_request_counts).tag("ombi-{}-get_request_counts".format(server.id)) if server.request_total_counts: at_time = schedule.every(server.request_total_run_seconds).seconds - at_time.do(QUEUE.put, OMBI.get_all_requests).tag(f"ombi-{server.id}-get_all_requests") + at_time.do(QUEUE.put, OMBI.get_all_requests).tag("ombi-{}-get_all_requests".format(server.id)) if server.issue_status_counts: at_time = schedule.every(server.issue_status_run_seconds).seconds - at_time.do(QUEUE.put, OMBI.get_issue_counts).tag(f"ombi-{server.id}-get_issue_counts") + at_time.do(QUEUE.put, OMBI.get_issue_counts).tag("ombi-{}-get_issue_counts".format(server.id)) if CONFIG.sickchill_enabled: for server in CONFIG.sickchill_servers: SICKCHILL = SickChillAPI(server, DBMANAGER) if server.get_missing: at_time = schedule.every(server.get_missing_run_seconds).seconds - at_time.do(QUEUE.put, SICKCHILL.get_missing).tag(f"sickchill-{server.id}-get_missing") + at_time.do(QUEUE.put, SICKCHILL.get_missing).tag("sickchill-{}-get_missing".format(server.id)) if CONFIG.unifi_enabled: for server in CONFIG.unifi_servers: UNIFI = UniFiAPI(server, DBMANAGER) at_time = schedule.every(server.get_usg_stats_run_seconds).seconds - at_time.do(QUEUE.put, UNIFI.get_usg_stats).tag(f"unifi-{server.id}-get_usg_stats") + at_time.do(QUEUE.put, UNIFI.get_usg_stats).tag("unifi-{}-get_usg_stats".format(server.id)) # Run all on startup SERVICES_ENABLED = [CONFIG.ombi_enabled, CONFIG.radarr_enabled, CONFIG.tautulli_enabled, CONFIG.unifi_enabled, From 8018684071d0ac674d883069700e99e6673d2c70 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Fri, 25 Jan 2019 16:07:03 -0600 Subject: [PATCH 17/20] give pip link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1adc5e9..105a180 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ frontend Requirements: * [Python 3.6.7+](https://www.python.org/downloads/release/python-367/) -* Python3-pip +* [Python3-pip](https://pip.pypa.io/en/stable/installing/) * [InfluxDB](https://www.influxdata.com/)

From 376e4b1530105633b82d68bd816c5abaa5bfd294 Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Sun, 3 Feb 2019 21:28:51 -0600 Subject: [PATCH 18/20] float cast --- varken/unifi.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/varken/unifi.py b/varken/unifi.py index 01f68ac..dcb2c50 100644 --- a/varken/unifi.py +++ b/varken/unifi.py @@ -66,11 +66,11 @@ class UniFiAPI(object): # "speedtest_latency": device['speedtest-status']['latency'], # "speedtest_download": device['speedtest-status']['xput_download'], # "speedtest_upload": device['speedtest-status']['xput_upload'], - "cpu_loadavg_1": device['sys_stats']['loadavg_1'], - "cpu_loadavg_5": device['sys_stats']['loadavg_5'], - "cpu_loadavg_15": device['sys_stats']['loadavg_15'], - "cpu_util": device['system-stats']['cpu'], - "mem_util": device['system-stats']['mem'], + "cpu_loadavg_1": float(device['sys_stats']['loadavg_1']), + "cpu_loadavg_5": float(device['sys_stats']['loadavg_5']), + "cpu_loadavg_15": float(device['sys_stats']['loadavg_15']), + "cpu_util": float(device['system-stats']['cpu']), + "mem_util": float(device['system-stats']['mem']), } } ] From 2b19255833591bbe3217eefc6413333c5d81f3fc Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Sun, 3 Feb 2019 23:45:44 -0600 Subject: [PATCH 19/20] master v1.6.4 bump + changelog --- CHANGELOG.md | 29 ++++++++++++++++++----------- varken/__init__.py | 2 +- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae09124..0756b42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,20 @@ # Change Log -## [v1.6.3](https://github.com/Boerderij/Varken/tree/v1.6.3) (2019-01-16) -[Full Changelog](https://github.com/Boerderij/Varken/compare/v1.6.2...v1.6.3) +## [v1.6.4](https://github.com/Boerderij/Varken/tree/v1.6.4) (2019-02-03) +[Full Changelog](https://github.com/Boerderij/Varken/compare/1.6.3...v1.6.4) + +**Fixed bugs:** + +- \[BUG\] fstring in Varken.py Doesnt allow py version check [\#102](https://github.com/Boerderij/Varken/issues/102) +- \[BUG\] Unifi loadavg is str instead of float [\#101](https://github.com/Boerderij/Varken/issues/101) +- \[BUG\] requestedByAlias to added to Ombi structures [\#97](https://github.com/Boerderij/Varken/issues/97) + +**Merged pull requests:** + +- v1.6.4 Merge [\#100](https://github.com/Boerderij/Varken/pull/100) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) + +## [1.6.3](https://github.com/Boerderij/Varken/tree/1.6.3) (2019-01-16) +[Full Changelog](https://github.com/Boerderij/Varken/compare/v1.6.2...1.6.3) **Implemented enhancements:** @@ -13,7 +26,9 @@ **Merged pull requests:** -- v1.6.3 Merge [\#94](https://github.com/Boerderij/Varken/pull/92) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) +- double typo [\#96](https://github.com/Boerderij/Varken/pull/96) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) +- tweaks [\#95](https://github.com/Boerderij/Varken/pull/95) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) +- v1.6.3 Merge [\#94](https://github.com/Boerderij/Varken/pull/94) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) ## [v1.6.2](https://github.com/Boerderij/Varken/tree/v1.6.2) (2019-01-12) [Full Changelog](https://github.com/Boerderij/Varken/compare/v1.6.1...v1.6.2) @@ -68,7 +83,6 @@ **Implemented enhancements:** - \[Feature Request\] Add issues from Ombi [\#70](https://github.com/Boerderij/Varken/issues/70) -- \[Feature Request\] Allow DNS Hostnames [\#66](https://github.com/Boerderij/Varken/issues/66) - Replace static grafana configs with a Public Example [\#32](https://github.com/Boerderij/Varken/issues/32) **Fixed bugs:** @@ -123,7 +137,6 @@ **Closed issues:** - Initial startup requires admin access to InfluxDB [\#53](https://github.com/Boerderij/Varken/issues/53) -- Ability to add custom tautulli port [\#49](https://github.com/Boerderij/Varken/issues/49) **Merged pull requests:** @@ -151,12 +164,7 @@ - use a config.ini instead of command-line flags [\#33](https://github.com/Boerderij/Varken/issues/33) - Migrate crontab to python schedule package [\#31](https://github.com/Boerderij/Varken/issues/31) - Consolidate missing and missing\_days in sonarr.py [\#30](https://github.com/Boerderij/Varken/issues/30) -- Database Withou any scripts [\#29](https://github.com/Boerderij/Varken/issues/29) -- Grafana dashboard json doesn't match format of readme screenshot? [\#28](https://github.com/Boerderij/Varken/issues/28) - Ombi something new \[Request\] [\#26](https://github.com/Boerderij/Varken/issues/26) -- Users Online not populating [\#24](https://github.com/Boerderij/Varken/issues/24) -- Missing dashboard [\#23](https://github.com/Boerderij/Varken/issues/23) -- Is there a Docker Image available for these scripts? [\#22](https://github.com/Boerderij/Varken/issues/22) - Support for Linux without ASA [\#21](https://github.com/Boerderij/Varken/issues/21) **Merged pull requests:** @@ -173,7 +181,6 @@ **Closed issues:** -- Tautulli.py not working. [\#18](https://github.com/Boerderij/Varken/issues/18) - Issues with scripts [\#12](https://github.com/Boerderij/Varken/issues/12) - issue with new tautulli.py [\#10](https://github.com/Boerderij/Varken/issues/10) - ombi.py fails when attempting to update influxdb [\#9](https://github.com/Boerderij/Varken/issues/9) diff --git a/varken/__init__.py b/varken/__init__.py index 9ec9472..1422b2e 100644 --- a/varken/__init__.py +++ b/varken/__init__.py @@ -1,2 +1,2 @@ VERSION = "1.6.4" -BRANCH = 'develop' +BRANCH = 'master' From f5be97beb9d9fd261de8460ef13ab35411c4205d Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Sun, 3 Feb 2019 23:50:02 -0600 Subject: [PATCH 20/20] remove manifest-generator --- manifest_generator.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 manifest_generator.py diff --git a/manifest_generator.py b/manifest_generator.py deleted file mode 100644 index db200db..0000000 --- a/manifest_generator.py +++ /dev/null @@ -1,37 +0,0 @@ -import yaml - -from varken import VERSION - -org = 'boerderij' -project = 'varken' -namespace = f"{org}/{project}" - -yaml_arr = [] -tags = ['latest', VERSION] - -# Docker image, arch, variant, os -arch_list = [('arm', 'arm', 'v6', 'linux'), - ('armhf', 'arm', 'v7', 'linux'), - ('arm64', 'arm64', 'v8', 'linux'), - ('amd64', 'amd64', None, 'linux')] - -for tag in tags: - yaml_doc = { - 'image': f'{namespace}:{tag}', - 'manifests': [] - } - for arch in arch_list: - info = { - 'image': f"{namespace}:{tag}-{arch[0]}", - 'platform': { - 'architecture': arch[1], - 'os': arch[3] - } - } - if arch[2]: - info['platform']['variant'] = arch[2] - yaml_doc['manifests'].append(info) - yaml_arr.append(yaml_doc) - -with open(f".manifest.yaml", 'w') as file: - yaml.dump_all(yaml_arr, file, default_flow_style=False)