Merge pull request #20 from DirtyCajunRice/nightly

v0.1

README.md

@@ -17,6 +17,29 @@ Requirements /w install links: [Grafana](http://docs.grafana.org/installation/),
 1. Install `grafana-cli plugins install grafana-worldmap-panel`
 1. Click the + on your menu and click import. Using the .json provided in this repo, paste it in and customize as you like.
 
+
+
+### Docker
+
+Repo is included in [si0972/grafana-scripts](https://github.com/si0972/grafana-scripts-docker)
+
+<details><summary>Example</summary>
+<p>
+
+```
+docker create \
+  --name=grafana-scripts \
+  -v <path to data>:/Scripts \
+  -e plex=true \
+  -e PGID=<gid> -e PUID=<uid>  \
+  si0972/grafana-scripts:latest
+```
+</p>
+</details>
+
+
+
+
 ## Scripts
 ### `sonarr.py`
 Gathers data from Sonarr and pushes it to influxdb.

dashboard/beta_online_users_table.json

@@ -0,0 +1,316 @@
+{
+  "columns": [],
+  "datasource": "plex",
+  "fontSize": "100%",
+  "gridPos": {
+    "h": 13,
+    "w": 16,
+    "x": 0,
+    "y": 16
+  },
+  "hideTimeOverride": true,
+  "id": 9,
+  "interval": "",
+  "links": [
+    {
+      "targetBlank": true,
+      "title": "Tautulli",
+      "type": "absolute",
+      "url": "https://tautulli.domain.tld"
+    }
+  ],
+  "minSpan": 12,
+  "pageSize": 14,
+  "scroll": true,
+  "showHeader": true,
+  "sort": {
+    "col": 9,
+    "desc": true
+  },
+  "styles": [
+    {
+      "alias": "",
+      "colorMode": null,
+      "colors": [
+        "rgba(245, 54, 54, 0.9)",
+        "rgba(237, 129, 40, 0.89)",
+        "rgba(50, 172, 45, 0.97)"
+      ],
+      "dateFormat": "MM/DD/YY h:mm:ss a",
+      "decimals": 2,
+      "link": false,
+      "pattern": "Time",
+      "thresholds": [],
+      "type": "hidden",
+      "unit": "short"
+    },
+    {
+      "alias": "User",
+      "colorMode": null,
+      "colors": [
+        "rgba(245, 54, 54, 0.9)",
+        "rgba(237, 129, 40, 0.89)",
+        "rgba(50, 172, 45, 0.97)"
+      ],
+      "dateFormat": "YYYY-MM-DD HH:mm:ss",
+      "decimals": 2,
+      "pattern": "name",
+      "preserveFormat": false,
+      "thresholds": [],
+      "type": "string",
+      "unit": "short"
+    },
+    {
+      "alias": "Media",
+      "colorMode": null,
+      "colors": [
+        "rgba(245, 54, 54, 0.9)",
+        "rgba(237, 129, 40, 0.89)",
+        "rgba(50, 172, 45, 0.97)"
+      ],
+      "dateFormat": "YYYY-MM-DD HH:mm:ss",
+      "decimals": 2,
+      "pattern": "title",
+      "thresholds": [],
+      "type": "string",
+      "unit": "short"
+    },
+    {
+      "alias": "Decision",
+      "colorMode": null,
+      "colors": [
+        "rgba(245, 54, 54, 0.9)",
+        "rgba(237, 129, 40, 0.89)",
+        "rgba(50, 172, 45, 0.97)"
+      ],
+      "dateFormat": "YYYY-MM-DD HH:mm:ss",
+      "decimals": 2,
+      "pattern": "video_decision",
+      "preserveFormat": false,
+      "sanitize": false,
+      "thresholds": [],
+      "type": "string",
+      "unit": "short"
+    },
+    {
+      "alias": "Quality",
+      "colorMode": null,
+      "colors": [
+        "rgba(245, 54, 54, 0.9)",
+        "rgba(237, 129, 40, 0.89)",
+        "rgba(50, 172, 45, 0.97)"
+      ],
+      "dateFormat": "YYYY-MM-DD HH:mm:ss",
+      "decimals": 2,
+      "pattern": "quality",
+      "thresholds": [],
+      "type": "string",
+      "unit": "short"
+    },
+    {
+      "alias": "Limits",
+      "colorMode": null,
+      "colors": [
+        "rgba(245, 54, 54, 0.9)",
+        "rgba(237, 129, 40, 0.89)",
+        "rgba(50, 172, 45, 0.97)"
+      ],
+      "dateFormat": "YYYY-MM-DD HH:mm:ss",
+      "decimals": 2,
+      "pattern": "quality_profile",
+      "preserveFormat": true,
+      "sanitize": false,
+      "thresholds": [],
+      "type": "string",
+      "unit": "short"
+    },
+    {
+      "alias": "Version",
+      "colorMode": null,
+      "colors": [
+        "rgba(245, 54, 54, 0.9)",
+        "rgba(237, 129, 40, 0.89)",
+        "rgba(50, 172, 45, 0.97)"
+      ],
+      "dateFormat": "YYYY-MM-DD HH:mm:ss",
+      "decimals": 0,
+      "pattern": "product_version",
+      "thresholds": [],
+      "type": "string",
+      "unit": "short"
+    },
+    {
+      "alias": "Device",
+      "colorMode": null,
+      "colors": [
+        "rgba(245, 54, 54, 0.9)",
+        "rgba(237, 129, 40, 0.89)",
+        "rgba(50, 172, 45, 0.97)"
+      ],
+      "dateFormat": "YYYY-MM-DD HH:mm:ss",
+      "decimals": 2,
+      "pattern": "platform",
+      "thresholds": [],
+      "type": "string",
+      "unit": "short"
+    },
+    {
+      "alias": "",
+      "colorMode": null,
+      "colors": [
+        "rgba(245, 54, 54, 0.9)",
+        "rgba(237, 129, 40, 0.89)",
+        "rgba(50, 172, 45, 0.97)"
+      ],
+      "dateFormat": "YYYY-MM-DD HH:mm:ss",
+      "decimals": 2,
+      "mappingType": 1,
+      "pattern": "distinct",
+      "thresholds": [],
+      "type": "hidden",
+      "unit": "short"
+    },
+    {
+      "alias": "Location",
+      "colorMode": null,
+      "colors": [
+        "rgba(245, 54, 54, 0.9)",
+        "rgba(237, 129, 40, 0.89)",
+        "rgba(50, 172, 45, 0.97)"
+      ],
+      "dateFormat": "YYYY-MM-DD HH:mm:ss",
+      "decimals": 2,
+      "mappingType": 1,
+      "pattern": "location",
+      "thresholds": [],
+      "type": "string",
+      "unit": "short"
+    },
+    {
+      "alias": "Player State",
+      "colorMode": "row",
+      "colors": [
+        "rgba(50, 172, 45, 0.3)",
+        "rgba(14, 221, 229, 0.56)",
+        "rgba(214, 103, 28, 0.8)"
+      ],
+      "dateFormat": "YYYY-MM-DD HH:mm:ss",
+      "decimals": 0,
+      "link": false,
+      "linkTargetBlank": false,
+      "linkTooltip": "",
+      "linkUrl": "",
+      "mappingType": 1,
+      "pattern": "player_state",
+      "thresholds": [
+        "1",
+        "3"
+      ],
+      "type": "string",
+      "unit": "none",
+      "valueMaps": [
+        {
+          "text": "Playing",
+          "value": "0"
+        },
+        {
+          "text": "Paused",
+          "value": "1"
+        },
+        {
+          "text": "Buffering",
+          "value": "3"
+        }
+      ]
+    }
+  ],
+  "targets": [
+    {
+      "dsType": "influxdb",
+      "groupBy": [
+        {
+          "params": [
+            "name"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "title"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "quality"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "video_decision"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "quality_profile"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "platform"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "product_version"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "location"
+          ],
+          "type": "tag"
+        }
+      ],
+      "hide": false,
+      "limit": "",
+      "measurement": "Tautulli",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT distinct(\"session_key\") FROM \"Tautulli\" WHERE (\"type\" = 'Session') AND $timeFilter GROUP BY \"name\",  \"title\", \"quality\", \"video_decision\", \"quality_profile\", \"platform\", \"product_version\", \"location\", \"player_state\"",
+      "rawQuery": true,
+      "refId": "A",
+      "resultFormat": "table",
+      "select": [
+        [
+          {
+            "params": [
+              "session_key"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "distinct"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "type",
+          "operator": "=",
+          "value": "Session"
+        }
+      ]
+    }
+  ],
+  "timeFrom": "1m",
+  "title": "Users Online",
+  "transform": "table",
+  "type": "table"
+}

dashboard/panel_row_worldmap.json

@@ -1,30 +1,33 @@
 
{
-  "circleMaxSize": "5",
+  "circleMaxSize": "2",
-  "circleMinSize": "1",
+  "circleMinSize": "2",
   "colors": [
-    "#cca300",
+    "#e67817",
-    "#c15c17",
+    "#6d3c97",
     "#890f02"
   ],
   "datasource": "plex",
   "decimals": 0,
-  "esLocationName": "",
+  "esGeoPoint": "geohash",
-  "esMetric": "$tag_counter",
+  "esLocationName": "location",
+  "esMetric": "metric",
   "gridPos": {
-    "h": 8,
+    "h": 10,
-    "w": 8,
+    "w": 10,
-    "x": 16,
+    "x": 10,
-    "y": 0
+    "y": 6
   },
   "hideEmpty": false,
+  "hideTimeOverride": true,
   "hideZero": false,
   "id": 4,
   "initialZoom": "4",
+  "interval": "",
   "links": [],
   "locationData": "table",
-  "mapCenter": "(0°, 0°)",
+  "mapCenter": "custom",
-  "mapCenterLatitude": 0,
+  "mapCenterLatitude": "37.9",
-  "mapCenterLongitude": 0,
+  "mapCenterLongitude": "-94.9",
   "maxDataPoints": 1,
   "minSpan": 8,
   "mouseWheelZoom": false,
@@ -32,7 +35,7 @@
   "stickyLabels": false,
   "tableQueryOptions": {
     "geohashField": "geohash",
-    "labelField": "location",
+    "labelField": "full_location",
     "latitudeField": "latitude",
     "longitudeField": "longitude",
     "metricField": "metric",
@@ -43,12 +46,6 @@
       "alias": "$tag_region_code",
       "dsType": "influxdb",
       "groupBy": [
-        {
-          "params": [
-            "location"
-          ],
-          "type": "tag"
-        },
         {
           "params": [
             "latitude"
@@ -60,6 +57,18 @@
             "longitude"
           ],
           "type": "tag"
+        },
+        {
+          "params": [
+            "full_location"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "name"
+          ],
+          "type": "tag"
         }
       ],
       "measurement": "Tautulli",
@@ -71,10 +80,14 @@
         [
           {
             "params": [
-              "location"
+              "session_key"
             ],
             "type": "field"
           },
+          {
+            "params": [],
+            "type": "distinct"
+          },
           {
             "params": [],
             "type": "count"
@@ -96,12 +109,13 @@
       ]
     }
   ],
-  "thresholds": "5,10",
+  "thresholds": "2,3",
   "timeFrom": "1m",
+  "timeShift": null,
   "title": "",
   "type": "grafana-worldmap-panel",
-  "unitPlural": "",
+  "unitPlural": "Streams",
   "unitSingle": "",
-  "unitSingular": "",
+  "unitSingular": "Stream",
   "valueName": "current"
 }

tautulli.py

@@ -1,134 +1,179 @@
-# Do not edit this script. Edit configuration.py
 import os
-import shutil
 import tarfile
-import requests
 import urllib.request
-import geoip2.database
+import time
 from datetime import datetime, timezone
+import geoip2.database
 from influxdb import InfluxDBClient
+import requests
 import configuration
 
-current_time = datetime.now(timezone.utc).astimezone().isoformat()
+CURRENT_TIME = datetime.now(timezone.utc).astimezone().isoformat()
 
-payload = {'apikey': configuration.tautulli_api_key, 'cmd': 'get_activity'}
+PAYLOAD = {'apikey': configuration.tautulli_api_key, 'cmd': 'get_activity'}
 
-activity = requests.get('{}/api/v2'.format(configuration.tautulli_url), params=payload).json()['response']['data']
+ACTIVITY = requests.get('{}/api/v2'.format(configuration.tautulli_url),
+                        params=PAYLOAD).json()['response']['data']
 
-sessions = {d['session_id']: d for d in activity['sessions']}
+SESSIONS = {d['session_id']: d for d in ACTIVITY['sessions']}
+
+TAR_DBFILE = '{}/GeoLite2-City.tar.gz'.format(os.path.dirname(os.path.realpath(__file__)))
+
+DBFILE = '{}/GeoLite2-City.mmdb'.format(os.path.dirname(os.path.realpath(__file__)))
+
+NOW = time.time()
+
+DB_AGE = NOW - (86400 * 35)
+
+#remove the running db file if it is older than 35 days
+try:
+    t = os.stat(DBFILE)
+    c = t.st_ctime
+    if c < DB_AGE:
+        os.remove(DBFILE)
+except FileNotFoundError:
+    pass
 
 
-def GeoLite2db(ipaddress):
+def geo_lookup(ipaddress):
-    tar_dbfile = '{}/GeoLite2-City.tar.gz'.format(os.path.dirname(os.path.realpath(__file__)))
+    """Lookup an IP using the local GeoLite2 DB"""
+    if not os.path.isfile(DBFILE):
+        urllib.request.urlretrieve(
+            'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz',
+            TAR_DBFILE)
 
-    dbfile = '{}/GeoLite2-City.mmdb'.format(os.path.dirname(os.path.realpath(__file__)))
+        tar = tarfile.open(TAR_DBFILE, "r:gz")
-
-    if not os.path.isfile(dbfile):
-        urllib.request.urlretrieve('http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz', tar_dbfile)
-
-        tar = tarfile.open(tar_dbfile, "r:gz")
         for files in tar.getmembers():
             if 'GeoLite2-City.mmdb' in files.name:
                 files.name = os.path.basename(files.name)
                 tar.extract(files, '{}/'.format(os.path.dirname(os.path.realpath(__file__))))
 
-    reader = geoip2.database.Reader(dbfile)
+    reader = geoip2.database.Reader(DBFILE)
-    geodata = reader.city(ipaddress)
 
-    return geodata
+    return reader.city(ipaddress)
 
 
-influx_payload = [
+INFLUX_PAYLOAD = [
     {
         "measurement": "Tautulli",
         "tags": {
             "type": "stream_count"
         },
-        "time": current_time,
+        "time": CURRENT_TIME,
         "fields": {
-            "current_streams": int(activity['stream_count']),
+            "current_streams": int(ACTIVITY['stream_count']),
-            "transcode_streams": int(activity['stream_count_transcode']),
+            "transcode_streams": int(ACTIVITY['stream_count_transcode']),
-            "direct_play_streams": int(activity['stream_count_direct_play']),
+            "direct_play_streams": int(ACTIVITY['stream_count_direct_play']),
-            "direct_streams": int(activity['stream_count_direct_stream'])
+            "direct_streams": int(ACTIVITY['stream_count_direct_stream'])
         }
     }
 ]
 
-for session in sessions.keys():
+for session in SESSIONS.keys():
     try:
-        geodata = GeoLite2db(sessions[session]['ip_address_public'])
+        geodata = geo_lookup(SESSIONS[session]['ip_address_public'])
-    except ValueError:
+    except (ValueError, geoip2.errors.AddressNotFoundError):
         if configuration.tautulli_failback_ip:
-            geodata = GeoLite2db(configuration.tautulli_failback_ip)
+            geodata = geo_lookup(configuration.tautulli_failback_ip)
         else:
-            geodata = GeoLite2db(requests.get('http://ip.42.pl/raw').text)
+            geodata = geo_lookup(requests.get('http://ip.42.pl/raw').text)
 
     latitude = geodata.location.latitude
 
-    # Get the latitude of each session. If we cant find it then...
     if not geodata.location.latitude:
         latitude = 37.234332396
     else:
         latitude = geodata.location.latitude
 
-    # Get the longitude of each session. If we cant find it then...
     if not geodata.location.longitude:
         longitude = -115.80666344
     else:
         longitude = geodata.location.longitude
 
-    decision = sessions[session]['transcode_decision']
+    decision = SESSIONS[session]['transcode_decision']
 
     if decision == 'copy':
         decision = 'direct stream'
 
-    video_decision = sessions[session]['stream_video_decision']
+    video_decision = SESSIONS[session]['stream_video_decision']
 
     if video_decision == 'copy':
         video_decision = 'direct stream'
 
-    quality = sessions[session]['stream_video_resolution']
+    elif video_decision == '':
+        video_decision = 'Music'
+
+    quality = SESSIONS[session]['stream_video_resolution']
+
 
     # If the video resolution is empty. Asssume it's an audio stream
+    # and use the container for music
     if not quality:
-        quality = sessions[session]['container'].upper()
+        quality = SESSIONS[session]['container'].upper()
 
     elif quality in ('SD', 'sd'):
-        quality = sessions[session]['stream_video_resolution'].upper()
+        quality = SESSIONS[session]['stream_video_resolution'].upper()
 
     elif quality in '4k':
-        quality = sessions[session]['stream_video_resolution'].upper()
+        quality = SESSIONS[session]['stream_video_resolution'].upper()
 
     else:
-        quality = '{}p'.format(sessions[session]['stream_video_resolution'])
+        quality = '{}p'.format(SESSIONS[session]['stream_video_resolution'])
 
-    influx_payload.append(
+
+    # Translate player_state to integers so we can colorize the table
+    player_state = SESSIONS[session]['state'].lower()
+
+    if player_state == 'playing':
+        player_state = 0
+
+    elif player_state == 'paused':
+        player_state = 1
+
+    elif player_state == 'buffering':
+        player_state = 3
+
+
+    INFLUX_PAYLOAD.append(
         {
             "measurement": "Tautulli",
             "tags": {
                 "type": "Session",
-                "region_code": geodata.subdivisions.most_specific.iso_code,
+                "session_id": SESSIONS[session]['session_id'],
-                "latitude": latitude,
+                "name": SESSIONS[session]['friendly_name'],
-                "longitude": longitude,
+                "title": SESSIONS[session]['full_title'],
-                "location": '{} - {}'.format(geodata.subdivisions.most_specific.name, geodata.city.name),
+                "platform": SESSIONS[session]['platform'],
-                "session_key": sessions[session]['session_key']
+                "product_version": SESSIONS[session]['product_version'],
-            },
-            "time": current_time,
-            "fields": {
-                "name": sessions[session]['friendly_name'],
-                "title": sessions[session]['full_title'],
                 "quality": quality,
                 "video_decision": video_decision.title(),
                 "transcode_decision": decision.title(),
-                "platform": sessions[session]['platform'],
+                "media_type": SESSIONS[session]['media_type'].title(),
-                "product_version": sessions[session]['product_version'],
+                "audio_codec": SESSIONS[session]['audio_codec'].upper(),
-                "quality_profile": sessions[session]['quality_profile'],
+                "audio_profile": SESSIONS[session]['audio_profile'].upper(),
-                "progress_percent": sessions[session]['progress_percent'],
+                "stream_audio_codec": SESSIONS[session]['stream_audio_codec'].upper(),
+                "quality_profile": SESSIONS[session]['quality_profile'],
+                "progress_percent": SESSIONS[session]['progress_percent'],
+                "region_code": geodata.subdivisions.most_specific.iso_code,
                 "location": geodata.city.name,
+                "full_location": '{} - {}'.format(geodata.subdivisions.most_specific.name,
+                                                  geodata.city.name),
+                "latitude": latitude,
+                "longitude": longitude,
+                "player_state": player_state,
+                "device_type": SESSIONS[session]['platform']
+            },
+            "time": CURRENT_TIME,
+            "fields": {
+                "session_id": SESSIONS[session]['session_id'],
+                "session_key": SESSIONS[session]['session_key']
             }
         }
     )
 
-influx = InfluxDBClient(configuration.influxdb_url, configuration.influxdb_port, configuration.influxdb_username,
+INFLUX_SENDER = InfluxDBClient(configuration.influxdb_url,
-                        configuration.influxdb_password, configuration.tautulli_influxdb_db_name)
+                               configuration.influxdb_port,
-influx.write_points(influx_payload)
+                               configuration.influxdb_username,
+                               configuration.influxdb_password,
+                               configuration.tautulli_influxdb_db_name)
+
+INFLUX_SENDER.write_points(INFLUX_PAYLOAD)