diff --git a/utilities/historical_tautulli_import.py b/utilities/historical_tautulli_import.py new file mode 100644 index 0000000..62bd0f8 --- /dev/null +++ b/utilities/historical_tautulli_import.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +from argparse import ArgumentParser +from os import access, R_OK +from os.path import isdir, abspath, dirname, join +from logging import getLogger, StreamHandler, Formatter, DEBUG + +from varken.iniparser import INIParser +from varken.dbmanager import DBManager +from varken.helpers import GeoIPHandler +from varken.tautulli import TautulliAPI + +if __name__ == "__main__": + parser = ArgumentParser(prog='varken', + description='Tautulli historical import tool') + parser.add_argument("-d", "--data-folder", help='Define an alternate data folder location') + parser.add_argument("-D", "--days", default=30, type=int, help='Specify length of historical import') + opts = parser.parse_args() + + DATA_FOLDER = abspath(join(dirname(__file__), '..', 'data')) + + templogger = getLogger('temp') + templogger.setLevel(DEBUG) + tempch = StreamHandler() + tempformatter = Formatter('%(asctime)s : %(levelname)s : %(module)s : %(message)s', '%Y-%m-%d %H:%M:%S') + tempch.setFormatter(tempformatter) + templogger.addHandler(tempch) + + if opts.data_folder: + ARG_FOLDER = opts.data_folder + + if isdir(ARG_FOLDER): + DATA_FOLDER = ARG_FOLDER + if not access(DATA_FOLDER, R_OK): + templogger.error("Read permission error for %s", DATA_FOLDER) + exit(1) + else: + templogger.error("%s does not exist", ARG_FOLDER) + exit(1) + + CONFIG = INIParser(DATA_FOLDER) + DBMANAGER = DBManager(CONFIG.influx_server) + + if CONFIG.tautulli_enabled: + GEOIPHANDLER = GeoIPHandler(DATA_FOLDER) + for server in CONFIG.tautulli_servers: + TAUTULLI = TautulliAPI(server, DBMANAGER, GEOIPHANDLER) + TAUTULLI.get_historical(days=opts.days) diff --git a/varken/structures.py b/varken/structures.py index 94e2478..9781067 100644 --- a/varken/structures.py +++ b/varken/structures.py @@ -273,8 +273,8 @@ class TautulliStream(NamedTuple): audience_rating_image: str = None audio_bitrate: str = None audio_bitrate_mode: str = None - audio_channels: str = None audio_channel_layout: str = None + audio_channels: str = None audio_codec: str = None audio_decision: str = None audio_language: str = None @@ -292,6 +292,8 @@ class TautulliStream(NamedTuple): collections: list = None container: str = None content_rating: str = None + current_session: str = None + date: str = None deleted_user: int = None device: str = None directors: list = None @@ -307,6 +309,8 @@ class TautulliStream(NamedTuple): grandparent_rating_key: str = None grandparent_thumb: str = None grandparent_title: str = None + group_count: int = None + group_ids: str = None guid: str = None height: str = None id: str = None @@ -331,16 +335,19 @@ class TautulliStream(NamedTuple): optimized_version: int = None optimized_version_profile: str = None optimized_version_title: str = None - originally_available_at: str = None original_title: str = None + originally_available_at: str = None parent_media_index: str = None parent_rating_key: str = None parent_thumb: str = None parent_title: str = None + paused_counter: int = None + percent_complete: int = None platform: str = None platform_name: str = None platform_version: str = None player: str = None + pre_tautulli: str = None product: str = None product_version: str = None profile: str = None @@ -349,20 +356,25 @@ class TautulliStream(NamedTuple): rating: str = None rating_image: str = None rating_key: str = None + reference_id: int = None relay: int = None + relayed: int = None section_id: str = None + secure: str = None selected: int = None session_id: str = None session_key: str = None shared_libraries: list = None sort_title: str = None + started: int = None state: str = None + stopped: int = None stream_aspect_ratio: str = None stream_audio_bitrate: str = None stream_audio_bitrate_mode: str = None - stream_audio_channels: str = None stream_audio_channel_layout: str = None stream_audio_channel_layout_: str = None + stream_audio_channels: str = None stream_audio_codec: str = None stream_audio_decision: str = None stream_audio_language: str = None @@ -380,8 +392,8 @@ class TautulliStream(NamedTuple): stream_subtitle_language: str = None stream_subtitle_language_code: str = None stream_subtitle_location: str = None - stream_video_bitrate: str = None stream_video_bit_depth: str = None + stream_video_bitrate: str = None stream_video_codec: str = None stream_video_codec_level: str = None stream_video_decision: str = None @@ -393,7 +405,7 @@ class TautulliStream(NamedTuple): stream_video_resolution: str = None stream_video_width: str = None studio: str = None - subtitles: int = None + sub_type: str = None subtitle_codec: str = None subtitle_container: str = None subtitle_decision: str = None @@ -402,7 +414,7 @@ class TautulliStream(NamedTuple): subtitle_language: str = None subtitle_language_code: str = None subtitle_location: str = None - sub_type: str = None + subtitles: int = None summary: str = None synced_version: int = None synced_version_profile: str = None @@ -433,17 +445,17 @@ class TautulliStream(NamedTuple): type: str = None updated_at: str = None user: str = None - username: str = None user_id: int = None user_rating: str = None user_thumb: str = None - video_bitrate: str = None + username: str = None video_bit_depth: str = None + video_bitrate: str = None video_codec: str = None video_codec_level: str = None video_decision: str = None - video_framerate: str = None video_frame_rate: str = None + video_framerate: str = None video_height: str = None video_language: str = None video_language_code: str = None @@ -452,11 +464,10 @@ class TautulliStream(NamedTuple): video_resolution: str = None video_width: str = None view_offset: str = None + watched_status: int = None width: str = None writers: list = None year: str = None - secure: str = None - relayed: int = None # Lidarr diff --git a/varken/tautulli.py b/varken/tautulli.py index c49d95b..0792fc7 100644 --- a/varken/tautulli.py +++ b/varken/tautulli.py @@ -1,7 +1,8 @@ from logging import getLogger from requests import Session, Request -from datetime import datetime, timezone from geoip2.errors import AddressNotFoundError +from datetime import datetime, timezone, date, timedelta +from influxdb.exceptions import InfluxDBClientError from varken.structures import TautulliStream from varken.helpers import hashit, connection_handler @@ -60,7 +61,7 @@ class TautulliAPI(object): if not self.my_ip: # Try the fallback ip in the config file try: - self.logger.debug('Atempting to use the failback IP...') + self.logger.debug('Attempting to use the fallback IP...') geodata = self.geoiphandler.lookup(self.server.fallback_ip) except AddressNotFoundError as e: self.logger.error('%s', e) @@ -215,3 +216,135 @@ class TautulliAPI(object): influx_payload.append(data) self.dbmanager.write_points(influx_payload) + + def get_historical(self, days=30): + influx_payload = [] + start_date = date.today() - timedelta(days=days) + params = {'cmd': 'get_history', 'grouping': 1, 'length': 1000000} + req = self.session.prepare_request(Request('GET', self.server.url + self.endpoint, params=params)) + g = connection_handler(self.session, req, self.server.verify_ssl) + + if not g: + return + + get = g['response']['data']['data'] + + params = {'cmd': 'get_stream_data', 'row_id': 0} + sessions = [] + for history_item in get: + if not history_item['id']: + self.logger.debug('Skipping entry with no ID. (%s)', history_item['full_title']) + continue + if date.fromtimestamp(history_item['started'] < start_date): + continue + params['row_id'] = history_item['id'] + req = self.session.prepare_request(Request('GET', self.server.url + self.endpoint, params=params)) + g = connection_handler(self.session, req, self.server.verify_ssl) + if not g: + self.logger.debug('Could not get historical stream data for %s. Skipping.', history_item['full_title']) + try: + self.logger.debug('Adding %s to history', history_item['full_title']) + history_item.update(g['response']['data']) + sessions.append(TautulliStream(**history_item)) + except TypeError as e: + self.logger.error('TypeError has occurred : %s while creating TautulliStream structure', e) + continue + + for session in sessions: + try: + geodata = self.geoiphandler.lookup(session.ip_address) + except (ValueError, AddressNotFoundError): + self.logger.debug('Public IP missing for Tautulli session...') + if not self.my_ip: + # Try the fallback ip in the config file + try: + self.logger.debug('Attempting to use the fallback IP...') + geodata = self.geoiphandler.lookup(self.server.fallback_ip) + except AddressNotFoundError as e: + self.logger.error('%s', e) + + self.my_ip = self.session.get('http://ip.42.pl/raw').text + self.logger.debug('Looked the public IP and set it to %s', self.my_ip) + + geodata = self.geoiphandler.lookup(self.my_ip) + + else: + geodata = self.geoiphandler.lookup(self.my_ip) + + if not all([geodata.location.latitude, geodata.location.longitude]): + latitude = 37.234332396 + longitude = -115.80666344 + else: + latitude = geodata.location.latitude + longitude = geodata.location.longitude + + if not geodata.city.name: + location = '👽' + else: + location = geodata.city.name + + decision = session.transcode_decision + if decision == 'copy': + decision = 'direct stream' + + video_decision = session.stream_video_decision + if video_decision == 'copy': + video_decision = 'direct stream' + elif video_decision == '': + video_decision = 'Music' + + quality = session.stream_video_resolution + if not quality: + quality = session.container.upper() + elif quality in ('SD', 'sd', '4k'): + quality = session.stream_video_resolution.upper() + else: + quality = session.stream_video_resolution + 'p' + + player_state = 100 + + hash_id = hashit(f'{session.id}{session.session_key}{session.user}{session.full_title}') + influx_payload.append( + { + "measurement": "Tautulli", + "tags": { + "type": "Session", + "session_id": session.session_id, + "friendly_name": session.friendly_name, + "username": session.user, + "title": session.full_title, + "platform": session.platform, + "quality": quality, + "video_decision": video_decision.title(), + "transcode_decision": decision.title(), + "transcode_hw_decoding": session.transcode_hw_decoding, + "transcode_hw_encoding": session.transcode_hw_encoding, + "media_type": session.media_type.title(), + "audio_codec": session.audio_codec.upper(), + "stream_audio_codec": session.stream_audio_codec.upper(), + "quality_profile": session.quality_profile, + "progress_percent": session.progress_percent, + "region_code": geodata.subdivisions.most_specific.iso_code, + "location": location, + "full_location": f'{geodata.subdivisions.most_specific.name} - {geodata.city.name}', + "latitude": latitude, + "longitude": longitude, + "player_state": player_state, + "device_type": session.platform, + "relayed": session.relayed, + "secure": session.secure, + "server": self.server.id + }, + "time": datetime.fromtimestamp(session.stopped).astimezone().isoformat(), + "fields": { + "hash": hash_id + } + } + ) + try: + self.dbmanager.write_points(influx_payload) + except InfluxDBClientError as e: + if "beyond retention policy" in str(e): + self.logger.debug('Only imported 30 days of data per retention policy') + else: + self.logger.error('Something went wrong... post this output in discord: %s', e)