d_mcknight/Varken
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

allow historical import of tautulli

Browse source
This commit is contained in:
Nicholas St. Germain 2019-04-29 14:27:57 -05:00
parent 1db99a46ed
commit 640989e495
3 changed files with 204 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
utilities/historical_tautulli_import.py Normal file
View file

@ -0,0 +1,47 @@
#!/usr/bin/env python3
from argparse import ArgumentParser
from os import access, R_OK
from os.path import isdir, abspath, dirname, join
from logging import getLogger, StreamHandler, Formatter, DEBUG
from varken.iniparser import INIParser
from varken.dbmanager import DBManager
from varken.helpers import GeoIPHandler
from varken.tautulli import TautulliAPI
if __name__ == "__main__":
parser = ArgumentParser(prog='varken',
description='Tautulli historical import tool')
parser.add_argument("-d", "--data-folder", help='Define an alternate data folder location')
parser.add_argument("-D", "--days", default=30, type=int, help='Specify length of historical import')
opts = parser.parse_args()
DATA_FOLDER = abspath(join(dirname(__file__), '..', 'data'))
templogger = getLogger('temp')
templogger.setLevel(DEBUG)
tempch = StreamHandler()
tempformatter = Formatter('%(asctime)s : %(levelname)s : %(module)s : %(message)s', '%Y-%m-%d %H:%M:%S')
tempch.setFormatter(tempformatter)
templogger.addHandler(tempch)
if opts.data_folder:
ARG_FOLDER = opts.data_folder
if isdir(ARG_FOLDER):
DATA_FOLDER = ARG_FOLDER
if not access(DATA_FOLDER, R_OK):
templogger.error("Read permission error for %s", DATA_FOLDER)
exit(1)
else:
templogger.error("%s does not exist", ARG_FOLDER)
exit(1)
CONFIG = INIParser(DATA_FOLDER)
DBMANAGER = DBManager(CONFIG.influx_server)
if CONFIG.tautulli_enabled:
GEOIPHANDLER = GeoIPHandler(DATA_FOLDER)
for server in CONFIG.tautulli_servers:
TAUTULLI = TautulliAPI(server, DBMANAGER, GEOIPHANDLER)
TAUTULLI.get_historical(days=opts.days)

33
varken/structures.py
View file

@ -273,8 +273,8 @@ class TautulliStream(NamedTuple):
audience_rating_image: str = None
audio_bitrate: str = None
audio_bitrate_mode: str = None
audio_channels: str = None
audio_channel_layout: str = None
audio_channels: str = None
audio_codec: str = None
audio_decision: str = None
audio_language: str = None
@ -292,6 +292,8 @@ class TautulliStream(NamedTuple):
collections: list = None
container: str = None
content_rating: str = None
current_session: str = None
date: str = None
deleted_user: int = None
device: str = None
directors: list = None
@ -307,6 +309,8 @@ class TautulliStream(NamedTuple):
grandparent_rating_key: str = None
grandparent_thumb: str = None
grandparent_title: str = None
group_count: int = None
group_ids: str = None
guid: str = None
height: str = None
id: str = None
@ -331,16 +335,19 @@ class TautulliStream(NamedTuple):
optimized_version: int = None
optimized_version_profile: str = None
optimized_version_title: str = None
originally_available_at: str = None
original_title: str = None
originally_available_at: str = None
parent_media_index: str = None
parent_rating_key: str = None
parent_thumb: str = None
parent_title: str = None
paused_counter: int = None
percent_complete: int = None
platform: str = None
platform_name: str = None
platform_version: str = None
player: str = None
pre_tautulli: str = None
product: str = None
product_version: str = None
profile: str = None
@ -349,20 +356,25 @@ class TautulliStream(NamedTuple):
rating: str = None
rating_image: str = None
rating_key: str = None
reference_id: int = None
relay: int = None
relayed: int = None
section_id: str = None
secure: str = None
selected: int = None
session_id: str = None
session_key: str = None
shared_libraries: list = None
sort_title: str = None
started: int = None
state: str = None
stopped: int = None
stream_aspect_ratio: str = None
stream_audio_bitrate: str = None
stream_audio_bitrate_mode: str = None
stream_audio_channels: str = None
stream_audio_channel_layout: str = None
stream_audio_channel_layout_: str = None
stream_audio_channels: str = None
stream_audio_codec: str = None
stream_audio_decision: str = None
stream_audio_language: str = None
@ -380,8 +392,8 @@ class TautulliStream(NamedTuple):
stream_subtitle_language: str = None
stream_subtitle_language_code: str = None
stream_subtitle_location: str = None
stream_video_bitrate: str = None
stream_video_bit_depth: str = None
stream_video_bitrate: str = None
stream_video_codec: str = None
stream_video_codec_level: str = None
stream_video_decision: str = None
@ -393,7 +405,7 @@ class TautulliStream(NamedTuple):
stream_video_resolution: str = None
stream_video_width: str = None
studio: str = None
subtitles: int = None
sub_type: str = None
subtitle_codec: str = None
subtitle_container: str = None
subtitle_decision: str = None
@ -402,7 +414,7 @@ class TautulliStream(NamedTuple):
subtitle_language: str = None
subtitle_language_code: str = None
subtitle_location: str = None
sub_type: str = None
subtitles: int = None
summary: str = None
synced_version: int = None
synced_version_profile: str = None
@ -433,17 +445,17 @@ class TautulliStream(NamedTuple):
type: str = None
updated_at: str = None
user: str = None
username: str = None
user_id: int = None
user_rating: str = None
user_thumb: str = None
video_bitrate: str = None
username: str = None
video_bit_depth: str = None
video_bitrate: str = None
video_codec: str = None
video_codec_level: str = None
video_decision: str = None
video_framerate: str = None
video_frame_rate: str = None
video_framerate: str = None
video_height: str = None
video_language: str = None
video_language_code: str = None
@ -452,11 +464,10 @@ class TautulliStream(NamedTuple):
video_resolution: str = None
video_width: str = None
view_offset: str = None
watched_status: int = None
width: str = None
writers: list = None
year: str = None
secure: str = None
relayed: int = None
# Lidarr

137
varken/tautulli.py
View file

@ -1,7 +1,8 @@
from logging import getLogger
from requests import Session, Request
from datetime import datetime, timezone
from geoip2.errors import AddressNotFoundError
from datetime import datetime, timezone, date, timedelta
from influxdb.exceptions import InfluxDBClientError
from varken.structures import TautulliStream
from varken.helpers import hashit, connection_handler
@ -60,7 +61,7 @@ class TautulliAPI(object):
if not self.my_ip:
# Try the fallback ip in the config file
try:
self.logger.debug('Atempting to use the failback IP...')
self.logger.debug('Attempting to use the fallback IP...')
geodata = self.geoiphandler.lookup(self.server.fallback_ip)
except AddressNotFoundError as e:
self.logger.error('%s', e)
@ -215,3 +216,135 @@ class TautulliAPI(object):
influx_payload.append(data)
self.dbmanager.write_points(influx_payload)
def get_historical(self, days=30):
influx_payload = []
start_date = date.today() - timedelta(days=days)
params = {'cmd': 'get_history', 'grouping': 1, 'length': 1000000}
req = self.session.prepare_request(Request('GET', self.server.url + self.endpoint, params=params))
g = connection_handler(self.session, req, self.server.verify_ssl)
if not g:
return
get = g['response']['data']['data']
params = {'cmd': 'get_stream_data', 'row_id': 0}
sessions = []
for history_item in get:
if not history_item['id']:
self.logger.debug('Skipping entry with no ID. (%s)', history_item['full_title'])
continue
if date.fromtimestamp(history_item['started'] < start_date):
continue
params['row_id'] = history_item['id']
req = self.session.prepare_request(Request('GET', self.server.url + self.endpoint, params=params))
g = connection_handler(self.session, req, self.server.verify_ssl)
if not g:
self.logger.debug('Could not get historical stream data for %s. Skipping.', history_item['full_title'])
try:
self.logger.debug('Adding %s to history', history_item['full_title'])
history_item.update(g['response']['data'])
sessions.append(TautulliStream(**history_item))
except TypeError as e:
self.logger.error('TypeError has occurred : %s while creating TautulliStream structure', e)
continue
for session in sessions:
try:
geodata = self.geoiphandler.lookup(session.ip_address)
except (ValueError, AddressNotFoundError):
self.logger.debug('Public IP missing for Tautulli session...')
if not self.my_ip:
# Try the fallback ip in the config file
try:
self.logger.debug('Attempting to use the fallback IP...')
geodata = self.geoiphandler.lookup(self.server.fallback_ip)
except AddressNotFoundError as e:
self.logger.error('%s', e)
self.my_ip = self.session.get('http://ip.42.pl/raw').text
self.logger.debug('Looked the public IP and set it to %s', self.my_ip)
geodata = self.geoiphandler.lookup(self.my_ip)
else:
geodata = self.geoiphandler.lookup(self.my_ip)
if not all([geodata.location.latitude, geodata.location.longitude]):
latitude = 37.234332396
longitude = -115.80666344
else:
latitude = geodata.location.latitude
longitude = geodata.location.longitude
if not geodata.city.name:
location = '👽'
else:
location = geodata.city.name
decision = session.transcode_decision
if decision == 'copy':
decision = 'direct stream'
video_decision = session.stream_video_decision
if video_decision == 'copy':
video_decision = 'direct stream'
elif video_decision == '':
video_decision = 'Music'
quality = session.stream_video_resolution
if not quality:
quality = session.container.upper()
elif quality in ('SD', 'sd', '4k'):
quality = session.stream_video_resolution.upper()
else:
quality = session.stream_video_resolution + 'p'
player_state = 100
hash_id = hashit(f'{session.id}{session.session_key}{session.user}{session.full_title}')
influx_payload.append(
{
"measurement": "Tautulli",
"tags": {
"type": "Session",
"session_id": session.session_id,
"friendly_name": session.friendly_name,
"username": session.user,
"title": session.full_title,
"platform": session.platform,
"quality": quality,
"video_decision": video_decision.title(),
"transcode_decision": decision.title(),
"transcode_hw_decoding": session.transcode_hw_decoding,
"transcode_hw_encoding": session.transcode_hw_encoding,
"media_type": session.media_type.title(),
"audio_codec": session.audio_codec.upper(),
"stream_audio_codec": session.stream_audio_codec.upper(),
"quality_profile": session.quality_profile,
"progress_percent": session.progress_percent,
"region_code": geodata.subdivisions.most_specific.iso_code,
"location": location,
"full_location": f'{geodata.subdivisions.most_specific.name} - {geodata.city.name}',
"latitude": latitude,
"longitude": longitude,
"player_state": player_state,
"device_type": session.platform,
"relayed": session.relayed,
"secure": session.secure,
"server": self.server.id
},
"time": datetime.fromtimestamp(session.stopped).astimezone().isoformat(),
"fields": {
"hash": hash_id
}
}
)
try:
self.dbmanager.write_points(influx_payload)
except InfluxDBClientError as e:
if "beyond retention policy" in str(e):
self.logger.debug('Only imported 30 days of data per retention policy')
else:
self.logger.error('Something went wrong... post this output in discord: %s', e)