allow historical import of tautulli

utilities/historical_tautulli_import.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+from argparse import ArgumentParser
+from os import access, R_OK
+from os.path import isdir, abspath, dirname, join
+from logging import getLogger, StreamHandler, Formatter, DEBUG
+
+from varken.iniparser import INIParser
+from varken.dbmanager import DBManager
+from varken.helpers import GeoIPHandler
+from varken.tautulli import TautulliAPI
+
+if __name__ == "__main__":
+    parser = ArgumentParser(prog='varken',
+                            description='Tautulli historical import tool')
+    parser.add_argument("-d", "--data-folder", help='Define an alternate data folder location')
+    parser.add_argument("-D", "--days", default=30, type=int, help='Specify length of historical import')
+    opts = parser.parse_args()
+
+    DATA_FOLDER = abspath(join(dirname(__file__), '..', 'data'))
+
+    templogger = getLogger('temp')
+    templogger.setLevel(DEBUG)
+    tempch = StreamHandler()
+    tempformatter = Formatter('%(asctime)s : %(levelname)s : %(module)s : %(message)s', '%Y-%m-%d %H:%M:%S')
+    tempch.setFormatter(tempformatter)
+    templogger.addHandler(tempch)
+
+    if opts.data_folder:
+        ARG_FOLDER = opts.data_folder
+
+        if isdir(ARG_FOLDER):
+            DATA_FOLDER = ARG_FOLDER
+            if not access(DATA_FOLDER, R_OK):
+                templogger.error("Read permission error for %s", DATA_FOLDER)
+                exit(1)
+        else:
+            templogger.error("%s does not exist", ARG_FOLDER)
+            exit(1)
+
+    CONFIG = INIParser(DATA_FOLDER)
+    DBMANAGER = DBManager(CONFIG.influx_server)
+
+    if CONFIG.tautulli_enabled:
+        GEOIPHANDLER = GeoIPHandler(DATA_FOLDER)
+        for server in CONFIG.tautulli_servers:
+            TAUTULLI = TautulliAPI(server, DBMANAGER, GEOIPHANDLER)
+            TAUTULLI.get_historical(days=opts.days)

varken/structures.py

@@ -273,8 +273,8 @@ class TautulliStream(NamedTuple):
     audience_rating_image: str = None
     audio_bitrate: str = None
     audio_bitrate_mode: str = None
-    audio_channels: str = None
     audio_channel_layout: str = None
+    audio_channels: str = None
     audio_codec: str = None
     audio_decision: str = None
     audio_language: str = None
@@ -292,6 +292,8 @@ class TautulliStream(NamedTuple):
     collections: list = None
     container: str = None
     content_rating: str = None
+    current_session: str = None
+    date: str = None
     deleted_user: int = None
     device: str = None
     directors: list = None
@@ -307,6 +309,8 @@ class TautulliStream(NamedTuple):
     grandparent_rating_key: str = None
     grandparent_thumb: str = None
     grandparent_title: str = None
+    group_count: int = None
+    group_ids: str = None
     guid: str = None
     height: str = None
     id: str = None
@@ -331,16 +335,19 @@ class TautulliStream(NamedTuple):
     optimized_version: int = None
     optimized_version_profile: str = None
     optimized_version_title: str = None
-    originally_available_at: str = None
     original_title: str = None
+    originally_available_at: str = None
     parent_media_index: str = None
     parent_rating_key: str = None
     parent_thumb: str = None
     parent_title: str = None
+    paused_counter: int = None
+    percent_complete: int = None
     platform: str = None
     platform_name: str = None
     platform_version: str = None
     player: str = None
+    pre_tautulli: str = None
     product: str = None
     product_version: str = None
     profile: str = None
@@ -349,20 +356,25 @@ class TautulliStream(NamedTuple):
     rating: str = None
     rating_image: str = None
     rating_key: str = None
+    reference_id: int = None
     relay: int = None
+    relayed: int = None
     section_id: str = None
+    secure: str = None
     selected: int = None
     session_id: str = None
     session_key: str = None
     shared_libraries: list = None
     sort_title: str = None
+    started: int = None
     state: str = None
+    stopped: int = None
     stream_aspect_ratio: str = None
     stream_audio_bitrate: str = None
     stream_audio_bitrate_mode: str = None
-    stream_audio_channels: str = None
     stream_audio_channel_layout: str = None
     stream_audio_channel_layout_: str = None
+    stream_audio_channels: str = None
     stream_audio_codec: str = None
     stream_audio_decision: str = None
     stream_audio_language: str = None
@@ -380,8 +392,8 @@ class TautulliStream(NamedTuple):
     stream_subtitle_language: str = None
     stream_subtitle_language_code: str = None
     stream_subtitle_location: str = None
-    stream_video_bitrate: str = None
     stream_video_bit_depth: str = None
+    stream_video_bitrate: str = None
     stream_video_codec: str = None
     stream_video_codec_level: str = None
     stream_video_decision: str = None
@@ -393,7 +405,7 @@ class TautulliStream(NamedTuple):
     stream_video_resolution: str = None
     stream_video_width: str = None
     studio: str = None
-    subtitles: int = None
+    sub_type: str = None
     subtitle_codec: str = None
     subtitle_container: str = None
     subtitle_decision: str = None
@@ -402,7 +414,7 @@ class TautulliStream(NamedTuple):
     subtitle_language: str = None
     subtitle_language_code: str = None
     subtitle_location: str = None
-    sub_type: str = None
+    subtitles: int = None
     summary: str = None
     synced_version: int = None
     synced_version_profile: str = None
@@ -433,17 +445,17 @@ class TautulliStream(NamedTuple):
     type: str = None
     updated_at: str = None
     user: str = None
-    username: str = None
     user_id: int = None
     user_rating: str = None
     user_thumb: str = None
-    video_bitrate: str = None
+    username: str = None
     video_bit_depth: str = None
+    video_bitrate: str = None
     video_codec: str = None
     video_codec_level: str = None
     video_decision: str = None
-    video_framerate: str = None
     video_frame_rate: str = None
+    video_framerate: str = None
     video_height: str = None
     video_language: str = None
     video_language_code: str = None
@@ -452,11 +464,10 @@ class TautulliStream(NamedTuple):
     video_resolution: str = None
     video_width: str = None
     view_offset: str = None
+    watched_status: int = None
     width: str = None
     writers: list = None
     year: str = None
-    secure: str = None
-    relayed: int = None
 
 
 # Lidarr

varken/tautulli.py

@@ -1,7 +1,8 @@
 from logging import getLogger
 from requests import Session, Request
-from datetime import datetime, timezone
 from geoip2.errors import AddressNotFoundError
+from datetime import datetime, timezone, date, timedelta
+from influxdb.exceptions import InfluxDBClientError
 
 from varken.structures import TautulliStream
 from varken.helpers import hashit, connection_handler
@@ -60,7 +61,7 @@ class TautulliAPI(object):
                 if not self.my_ip:
                     # Try the fallback ip in the config file
                     try:
-                        self.logger.debug('Atempting to use the failback IP...')
+                        self.logger.debug('Attempting to use the fallback IP...')
                         geodata = self.geoiphandler.lookup(self.server.fallback_ip)
                     except AddressNotFoundError as e:
                         self.logger.error('%s', e)
@@ -215,3 +216,135 @@ class TautulliAPI(object):
             influx_payload.append(data)
 
         self.dbmanager.write_points(influx_payload)
+
+    def get_historical(self, days=30):
+        influx_payload = []
+        start_date = date.today() - timedelta(days=days)
+        params = {'cmd': 'get_history', 'grouping': 1, 'length': 1000000}
+        req = self.session.prepare_request(Request('GET', self.server.url + self.endpoint, params=params))
+        g = connection_handler(self.session, req, self.server.verify_ssl)
+
+        if not g:
+            return
+
+        get = g['response']['data']['data']
+
+        params = {'cmd': 'get_stream_data', 'row_id': 0}
+        sessions = []
+        for history_item in get:
+            if not history_item['id']:
+                self.logger.debug('Skipping entry with no ID. (%s)', history_item['full_title'])
+                continue
+            if date.fromtimestamp(history_item['started'] < start_date):
+                continue
+            params['row_id'] = history_item['id']
+            req = self.session.prepare_request(Request('GET', self.server.url + self.endpoint, params=params))
+            g = connection_handler(self.session, req, self.server.verify_ssl)
+            if not g:
+                self.logger.debug('Could not get historical stream data for %s. Skipping.', history_item['full_title'])
+            try:
+                self.logger.debug('Adding %s to history', history_item['full_title'])
+                history_item.update(g['response']['data'])
+                sessions.append(TautulliStream(**history_item))
+            except TypeError as e:
+                self.logger.error('TypeError has occurred : %s while creating TautulliStream structure', e)
+                continue
+
+        for session in sessions:
+            try:
+                geodata = self.geoiphandler.lookup(session.ip_address)
+            except (ValueError, AddressNotFoundError):
+                self.logger.debug('Public IP missing for Tautulli session...')
+                if not self.my_ip:
+                    # Try the fallback ip in the config file
+                    try:
+                        self.logger.debug('Attempting to use the fallback IP...')
+                        geodata = self.geoiphandler.lookup(self.server.fallback_ip)
+                    except AddressNotFoundError as e:
+                        self.logger.error('%s', e)
+
+                        self.my_ip = self.session.get('http://ip.42.pl/raw').text
+                        self.logger.debug('Looked the public IP and set it to %s', self.my_ip)
+
+                        geodata = self.geoiphandler.lookup(self.my_ip)
+
+                else:
+                    geodata = self.geoiphandler.lookup(self.my_ip)
+
+            if not all([geodata.location.latitude, geodata.location.longitude]):
+                latitude = 37.234332396
+                longitude = -115.80666344
+            else:
+                latitude = geodata.location.latitude
+                longitude = geodata.location.longitude
+
+            if not geodata.city.name:
+                location = '👽'
+            else:
+                location = geodata.city.name
+
+            decision = session.transcode_decision
+            if decision == 'copy':
+                decision = 'direct stream'
+
+            video_decision = session.stream_video_decision
+            if video_decision == 'copy':
+                video_decision = 'direct stream'
+            elif video_decision == '':
+                video_decision = 'Music'
+
+            quality = session.stream_video_resolution
+            if not quality:
+                quality = session.container.upper()
+            elif quality in ('SD', 'sd', '4k'):
+                quality = session.stream_video_resolution.upper()
+            else:
+                quality = session.stream_video_resolution + 'p'
+
+            player_state = 100
+
+            hash_id = hashit(f'{session.id}{session.session_key}{session.user}{session.full_title}')
+            influx_payload.append(
+                {
+                    "measurement": "Tautulli",
+                    "tags": {
+                        "type": "Session",
+                        "session_id": session.session_id,
+                        "friendly_name": session.friendly_name,
+                        "username": session.user,
+                        "title": session.full_title,
+                        "platform": session.platform,
+                        "quality": quality,
+                        "video_decision": video_decision.title(),
+                        "transcode_decision": decision.title(),
+                        "transcode_hw_decoding": session.transcode_hw_decoding,
+                        "transcode_hw_encoding": session.transcode_hw_encoding,
+                        "media_type": session.media_type.title(),
+                        "audio_codec": session.audio_codec.upper(),
+                        "stream_audio_codec": session.stream_audio_codec.upper(),
+                        "quality_profile": session.quality_profile,
+                        "progress_percent": session.progress_percent,
+                        "region_code": geodata.subdivisions.most_specific.iso_code,
+                        "location": location,
+                        "full_location": f'{geodata.subdivisions.most_specific.name} - {geodata.city.name}',
+                        "latitude": latitude,
+                        "longitude": longitude,
+                        "player_state": player_state,
+                        "device_type": session.platform,
+                        "relayed": session.relayed,
+                        "secure": session.secure,
+                        "server": self.server.id
+                    },
+                    "time": datetime.fromtimestamp(session.stopped).astimezone().isoformat(),
+                    "fields": {
+                        "hash": hash_id
+                    }
+                }
+            )
+            try:
+                self.dbmanager.write_points(influx_payload)
+            except InfluxDBClientError as e:
+                if "beyond retention policy" in str(e):
+                    self.logger.debug('Only imported 30 days of data per retention policy')
+                else:
+                    self.logger.error('Something went wrong... post this output in discord: %s', e)