From 13227258b3408b81baf38997199efd09c399b41b Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Fri, 31 Aug 2018 15:10:57 -0400 Subject: [PATCH 1/5] Update ROW map to reflect script changes --- dashboard/panel_row_worldmap.json | 62 +++++++++++++++++++------------ 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/dashboard/panel_row_worldmap.json b/dashboard/panel_row_worldmap.json index 73a7fb5..63b381d 100644 --- a/dashboard/panel_row_worldmap.json +++ b/dashboard/panel_row_worldmap.json @@ -1,30 +1,33 @@ { - "circleMaxSize": "5", - "circleMinSize": "1", + "circleMaxSize": "2", + "circleMinSize": "2", "colors": [ - "#cca300", - "#c15c17", + "#e67817", + "#6d3c97", "#890f02" ], "datasource": "plex", "decimals": 0, - "esLocationName": "", - "esMetric": "$tag_counter", + "esGeoPoint": "geohash", + "esLocationName": "location", + "esMetric": "metric", "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 0 + "h": 10, + "w": 10, + "x": 10, + "y": 6 }, "hideEmpty": false, + "hideTimeOverride": true, "hideZero": false, "id": 4, "initialZoom": "4", + "interval": "", "links": [], "locationData": "table", - "mapCenter": "(0°, 0°)", - "mapCenterLatitude": 0, - "mapCenterLongitude": 0, + "mapCenter": "custom", + "mapCenterLatitude": "37.9", + "mapCenterLongitude": "-94.9", "maxDataPoints": 1, "minSpan": 8, "mouseWheelZoom": false, @@ -32,7 +35,7 @@ "stickyLabels": false, "tableQueryOptions": { "geohashField": "geohash", - "labelField": "location", + "labelField": "full_location", "latitudeField": "latitude", "longitudeField": "longitude", "metricField": "metric", @@ -43,12 +46,6 @@ "alias": "$tag_region_code", "dsType": "influxdb", "groupBy": [ - { - "params": [ - "location" - ], - "type": "tag" - }, { "params": [ "latitude" @@ -60,6 +57,18 @@ "longitude" ], "type": "tag" + }, + { + "params": [ + "full_location" + ], + "type": "tag" + }, + { + "params": [ + "name" + ], + "type": "tag" } ], "measurement": "Tautulli", @@ -71,10 +80,14 @@ [ { "params": [ - "location" + "session_key" ], "type": "field" }, + { + "params": [], + "type": "distinct" + }, { "params": [], "type": "count" @@ -96,12 +109,13 @@ ] } ], - "thresholds": "5,10", + "thresholds": "2,3", "timeFrom": "1m", + "timeShift": null, "title": "", "type": "grafana-worldmap-panel", - "unitPlural": "", + "unitPlural": "Streams", "unitSingle": "", - "unitSingular": "", + "unitSingular": "Stream", "valueName": "current" } From b523f1346e32580bac190f19cea898222b24c7b8 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Fri, 31 Aug 2018 15:11:38 -0400 Subject: [PATCH 2/5] Add online users beta table --- dashboard/beta_online_users_table.json | 316 +++++++++++++++++++++++++ 1 file changed, 316 insertions(+) create mode 100644 dashboard/beta_online_users_table.json diff --git a/dashboard/beta_online_users_table.json b/dashboard/beta_online_users_table.json new file mode 100644 index 0000000..09bc070 --- /dev/null +++ b/dashboard/beta_online_users_table.json @@ -0,0 +1,316 @@ +{ + "columns": [], + "datasource": "plex", + "fontSize": "100%", + "gridPos": { + "h": 13, + "w": 16, + "x": 0, + "y": 16 + }, + "hideTimeOverride": true, + "id": 9, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Tautulli", + "type": "absolute", + "url": "https://tautulli.domain.tld" + } + ], + "minSpan": 12, + "pageSize": 14, + "scroll": true, + "showHeader": true, + "sort": { + "col": 9, + "desc": true + }, + "styles": [ + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "MM/DD/YY h:mm:ss a", + "decimals": 2, + "link": false, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "User", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "name", + "preserveFormat": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Media", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "title", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Decision", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "video_decision", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Quality", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "quality", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Limits", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "quality_profile", + "preserveFormat": true, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Version", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "product_version", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Device", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "platform", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "distinct", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Location", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "location", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Player State", + "colorMode": "row", + "colors": [ + "rgba(50, 172, 45, 0.3)", + "rgba(14, 221, 229, 0.56)", + "rgba(214, 103, 28, 0.8)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "", + "linkUrl": "", + "mappingType": 1, + "pattern": "player_state", + "thresholds": [ + "1", + "3" + ], + "type": "string", + "unit": "none", + "valueMaps": [ + { + "text": "Playing", + "value": "0" + }, + { + "text": "Paused", + "value": "1" + }, + { + "text": "Buffering", + "value": "3" + } + ] + } + ], + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "name" + ], + "type": "tag" + }, + { + "params": [ + "title" + ], + "type": "tag" + }, + { + "params": [ + "quality" + ], + "type": "tag" + }, + { + "params": [ + "video_decision" + ], + "type": "tag" + }, + { + "params": [ + "quality_profile" + ], + "type": "tag" + }, + { + "params": [ + "platform" + ], + "type": "tag" + }, + { + "params": [ + "product_version" + ], + "type": "tag" + }, + { + "params": [ + "location" + ], + "type": "tag" + } + ], + "hide": false, + "limit": "", + "measurement": "Tautulli", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT distinct(\"session_key\") FROM \"Tautulli\" WHERE (\"type\" = 'Session') AND $timeFilter GROUP BY \"name\", \"title\", \"quality\", \"video_decision\", \"quality_profile\", \"platform\", \"product_version\", \"location\", \"player_state\"", + "rawQuery": true, + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "session_key" + ], + "type": "field" + }, + { + "params": [], + "type": "distinct" + } + ] + ], + "tags": [ + { + "key": "type", + "operator": "=", + "value": "Session" + } + ] + } + ], + "timeFrom": "1m", + "title": "Users Online", + "transform": "table", + "type": "table" +} From d7f8bf83a0df45e96fab5f481f4686ffc16367b4 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Fri, 31 Aug 2018 15:13:19 -0400 Subject: [PATCH 3/5] Major change in data structure that should resolve the double data issue --- tautulli.py | 157 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 101 insertions(+), 56 deletions(-) diff --git a/tautulli.py b/tautulli.py index 41a1bd7..d2cf67e 100644 --- a/tautulli.py +++ b/tautulli.py @@ -1,134 +1,179 @@ -# Do not edit this script. Edit configuration.py import os -import shutil import tarfile -import requests import urllib.request -import geoip2.database +import time from datetime import datetime, timezone +import geoip2.database from influxdb import InfluxDBClient +import requests import configuration -current_time = datetime.now(timezone.utc).astimezone().isoformat() +CURRENT_TIME = datetime.now(timezone.utc).astimezone().isoformat() -payload = {'apikey': configuration.tautulli_api_key, 'cmd': 'get_activity'} +PAYLOAD = {'apikey': configuration.tautulli_api_key, 'cmd': 'get_activity'} -activity = requests.get('{}/api/v2'.format(configuration.tautulli_url), params=payload).json()['response']['data'] +ACTIVITY = requests.get('{}/api/v2'.format(configuration.tautulli_url), + params=PAYLOAD).json()['response']['data'] -sessions = {d['session_id']: d for d in activity['sessions']} +SESSIONS = {d['session_id']: d for d in ACTIVITY['sessions']} + +TAR_DBFILE = '{}/GeoLite2-City.tar.gz'.format(os.path.dirname(os.path.realpath(__file__))) + +DBFILE = '{}/GeoLite2-City.mmdb'.format(os.path.dirname(os.path.realpath(__file__))) + +NOW = time.time() + +DB_AGE = NOW - (86400 * 35) + +#remove the running db file if it is older than 35 days +try: + t = os.stat(DBFILE) + c = t.st_ctime + if c < DB_AGE: + os.remove(DBFILE) +except FileNotFoundError: + pass -def GeoLite2db(ipaddress): - tar_dbfile = '{}/GeoLite2-City.tar.gz'.format(os.path.dirname(os.path.realpath(__file__))) +def geo_lookup(ipaddress): + """Lookup an IP using the local GeoLite2 DB""" + if not os.path.isfile(DBFILE): + urllib.request.urlretrieve( + 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz', + TAR_DBFILE) - dbfile = '{}/GeoLite2-City.mmdb'.format(os.path.dirname(os.path.realpath(__file__))) - - if not os.path.isfile(dbfile): - urllib.request.urlretrieve('http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz', tar_dbfile) - - tar = tarfile.open(tar_dbfile, "r:gz") + tar = tarfile.open(TAR_DBFILE, "r:gz") for files in tar.getmembers(): if 'GeoLite2-City.mmdb' in files.name: files.name = os.path.basename(files.name) tar.extract(files, '{}/'.format(os.path.dirname(os.path.realpath(__file__)))) - reader = geoip2.database.Reader(dbfile) - geodata = reader.city(ipaddress) + reader = geoip2.database.Reader(DBFILE) - return geodata + return reader.city(ipaddress) -influx_payload = [ +INFLUX_PAYLOAD = [ { "measurement": "Tautulli", "tags": { "type": "stream_count" }, - "time": current_time, + "time": CURRENT_TIME, "fields": { - "current_streams": int(activity['stream_count']), - "transcode_streams": int(activity['stream_count_transcode']), - "direct_play_streams": int(activity['stream_count_direct_play']), - "direct_streams": int(activity['stream_count_direct_stream']) + "current_streams": int(ACTIVITY['stream_count']), + "transcode_streams": int(ACTIVITY['stream_count_transcode']), + "direct_play_streams": int(ACTIVITY['stream_count_direct_play']), + "direct_streams": int(ACTIVITY['stream_count_direct_stream']) } } ] -for session in sessions.keys(): +for session in SESSIONS.keys(): try: - geodata = GeoLite2db(sessions[session]['ip_address_public']) + geodata = geo_lookup(SESSIONS[session]['ip_address_public']) except ValueError: if configuration.tautulli_failback_ip: - geodata = GeoLite2db(configuration.tautulli_failback_ip) + geodata = geo_lookup(configuration.tautulli_failback_ip) else: - geodata = GeoLite2db(requests.get('http://ip.42.pl/raw').text) + geodata = geo_lookup(requests.get('http://ip.42.pl/raw').text) latitude = geodata.location.latitude - # Get the latitude of each session. If we cant find it then... if not geodata.location.latitude: latitude = 37.234332396 else: latitude = geodata.location.latitude - # Get the longitude of each session. If we cant find it then... if not geodata.location.longitude: longitude = -115.80666344 else: longitude = geodata.location.longitude - decision = sessions[session]['transcode_decision'] + decision = SESSIONS[session]['transcode_decision'] if decision == 'copy': decision = 'direct stream' - video_decision = sessions[session]['stream_video_decision'] + video_decision = SESSIONS[session]['stream_video_decision'] if video_decision == 'copy': video_decision = 'direct stream' - quality = sessions[session]['stream_video_resolution'] + elif video_decision == '': + video_decision = 'Music' + + quality = SESSIONS[session]['stream_video_resolution'] + # If the video resolution is empty. Asssume it's an audio stream + # and use the container for music if not quality: - quality = sessions[session]['container'].upper() + quality = SESSIONS[session]['container'].upper() elif quality in ('SD', 'sd'): - quality = sessions[session]['stream_video_resolution'].upper() + quality = SESSIONS[session]['stream_video_resolution'].upper() elif quality in '4k': - quality = sessions[session]['stream_video_resolution'].upper() + quality = SESSIONS[session]['stream_video_resolution'].upper() else: - quality = '{}p'.format(sessions[session]['stream_video_resolution']) + quality = '{}p'.format(SESSIONS[session]['stream_video_resolution']) - influx_payload.append( + + # Translate player_state to integers so we can colorize the table + player_state = SESSIONS[session]['state'].lower() + + if player_state == 'playing': + player_state = 0 + + elif player_state == 'paused': + player_state = 1 + + elif player_state == 'buffering': + player_state = 3 + + + INFLUX_PAYLOAD.append( { "measurement": "Tautulli", "tags": { "type": "Session", - "region_code": geodata.subdivisions.most_specific.iso_code, - "latitude": latitude, - "longitude": longitude, - "location": '{} - {}'.format(geodata.subdivisions.most_specific.name, geodata.city.name), - "session_key": sessions[session]['session_key'] - }, - "time": current_time, - "fields": { - "name": sessions[session]['friendly_name'], - "title": sessions[session]['full_title'], + "session_id": SESSIONS[session]['session_id'], + "name": SESSIONS[session]['friendly_name'], + "title": SESSIONS[session]['full_title'], + "platform": SESSIONS[session]['platform'], + "product_version": SESSIONS[session]['product_version'], "quality": quality, "video_decision": video_decision.title(), "transcode_decision": decision.title(), - "platform": sessions[session]['platform'], - "product_version": sessions[session]['product_version'], - "quality_profile": sessions[session]['quality_profile'], - "progress_percent": sessions[session]['progress_percent'], + "media_type": SESSIONS[session]['media_type'].title(), + "audio_codec": SESSIONS[session]['audio_codec'].upper(), + "audio_profile": SESSIONS[session]['audio_profile'].upper(), + "stream_audio_codec": SESSIONS[session]['stream_audio_codec'].upper(), + "quality_profile": SESSIONS[session]['quality_profile'], + "progress_percent": SESSIONS[session]['progress_percent'], + "region_code": geodata.subdivisions.most_specific.iso_code, "location": geodata.city.name, + "full_location": '{} - {}'.format(geodata.subdivisions.most_specific.name, + geodata.city.name), + "latitude": latitude, + "longitude": longitude, + "player_state": player_state, + "device_type": SESSIONS[session]['platform'] + }, + "time": CURRENT_TIME, + "fields": { + "session_id": SESSIONS[session]['session_id'], + "session_key": SESSIONS[session]['session_key'] } } ) -influx = InfluxDBClient(configuration.influxdb_url, configuration.influxdb_port, configuration.influxdb_username, - configuration.influxdb_password, configuration.tautulli_influxdb_db_name) -influx.write_points(influx_payload) +INFLUX_SENDER = InfluxDBClient(configuration.influxdb_url, + configuration.influxdb_port, + configuration.influxdb_username, + configuration.influxdb_password, + configuration.tautulli_influxdb_db_name) + +INFLUX_SENDER.write_points(INFLUX_PAYLOAD) From 122f198857c3bf605ff00ff1e0eb8d919a1decb8 Mon Sep 17 00:00:00 2001 From: samwiseg0 Date: Wed, 26 Sep 2018 08:49:34 -0700 Subject: [PATCH 4/5] Handle invalid IPs by using the failback IP if it is not found in the DB --- tautulli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tautulli.py b/tautulli.py index d2cf67e..96f61ed 100644 --- a/tautulli.py +++ b/tautulli.py @@ -72,7 +72,7 @@ INFLUX_PAYLOAD = [ for session in SESSIONS.keys(): try: geodata = geo_lookup(SESSIONS[session]['ip_address_public']) - except ValueError: + except (ValueError, geoip2.errors.AddressNotFoundError): if configuration.tautulli_failback_ip: geodata = geo_lookup(configuration.tautulli_failback_ip) else: From 96b81a5511fb668eb61ddc623f585c31078714e1 Mon Sep 17 00:00:00 2001 From: roxedus Date: Sat, 20 Oct 2018 18:09:50 +0200 Subject: [PATCH 5/5] Added selfplug --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 4944605..ec2098e 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,29 @@ Requirements /w install links: [Grafana](http://docs.grafana.org/installation/), 1. Install `grafana-cli plugins install grafana-worldmap-panel` 1. Click the + on your menu and click import. Using the .json provided in this repo, paste it in and customize as you like. + + +### Docker + +Repo is included in [si0972/grafana-scripts](https://github.com/si0972/grafana-scripts-docker) + +
Example +

+ +``` +docker create \ + --name=grafana-scripts \ + -v :/Scripts \ + -e plex=true \ + -e PGID= -e PUID= \ + si0972/grafana-scripts:latest +``` +

+
+ + + + ## Scripts ### `sonarr.py` Gathers data from Sonarr and pushes it to influxdb.