d_mcknight/Varken
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Ultra-threaded concurrency. For SCIENCE!

Browse source
This commit is contained in:
Nicholas St. Germain 2019-04-24 18:17:11 -05:00
parent 5827bb2c97
commit f943968087
1 changed files with 23 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
Varken.py
View file

@ -29,17 +29,13 @@ from varken.varkenlogger import VarkenLogger
PLATFORM_LINUX_DISTRO = ' '.join(x for x in linux_distribution() if x) PLATFORM_LINUX_DISTRO = ' '.join(x for x in linux_distribution() if x)
def thread(): def thread(job):
while schedule.jobs: worker = Thread(target=job)
job = QUEUE.get() if isinstance(job, tuple):
if isinstance(job, tuple): job, query = job[0], job[1]
job, query = job[0], job[1] worker = Thread(target=job, kwargs={'query': query})
a = job(query=query)
else: worker.start()
a = job()
if a is not None:
schedule.clear(a)
QUEUE.task_done()
if __name__ == "__main__": if __name__ == "__main__":
@ -109,49 +105,49 @@ if __name__ == "__main__":
SONARR = SonarrAPI(server, DBMANAGER) SONARR = SonarrAPI(server, DBMANAGER)
if server.queue: if server.queue:
at_time = schedule.every(server.queue_run_seconds).seconds at_time = schedule.every(server.queue_run_seconds).seconds
at_time.do(QUEUE.put, SONARR.get_queue).tag("sonarr-{}-get_queue".format(server.id)) at_time.do(thread, SONARR.get_queue).tag("sonarr-{}-get_queue".format(server.id))
if server.missing_days > 0: if server.missing_days > 0:
at_time = schedule.every(server.missing_days_run_seconds).seconds at_time = schedule.every(server.missing_days_run_seconds).seconds
at_time.do(QUEUE.put, SONARR.get_missing).tag("sonarr-{}-get_missing".format(server.id)) at_time.do(thread, SONARR.get_missing).tag("sonarr-{}-get_missing".format(server.id))
if server.future_days > 0: if server.future_days > 0:
at_time = schedule.every(server.future_days_run_seconds).seconds at_time = schedule.every(server.future_days_run_seconds).seconds
at_time.do(QUEUE.put, SONARR.get_future).tag("sonarr-{}-get_future".format(server.id)) at_time.do(thread, SONARR.get_future).tag("sonarr-{}-get_future".format(server.id))
if CONFIG.tautulli_enabled: if CONFIG.tautulli_enabled:
GEOIPHANDLER = GeoIPHandler(DATA_FOLDER) GEOIPHANDLER = GeoIPHandler(DATA_FOLDER)
schedule.every(12).to(24).hours.do(QUEUE.put, GEOIPHANDLER.update) schedule.every(12).to(24).hours.do(thread, GEOIPHANDLER.update)
for server in CONFIG.tautulli_servers: for server in CONFIG.tautulli_servers:
TAUTULLI = TautulliAPI(server, DBMANAGER, GEOIPHANDLER) TAUTULLI = TautulliAPI(server, DBMANAGER, GEOIPHANDLER)
if server.get_activity: if server.get_activity:
at_time = schedule.every(server.get_activity_run_seconds).seconds at_time = schedule.every(server.get_activity_run_seconds).seconds
at_time.do(QUEUE.put, TAUTULLI.get_activity).tag("tautulli-{}-get_activity".format(server.id)) at_time.do(thread, TAUTULLI.get_activity).tag("tautulli-{}-get_activity".format(server.id))
if server.get_stats: if server.get_stats:
at_time = schedule.every(server.get_stats_run_seconds).seconds at_time = schedule.every(server.get_stats_run_seconds).seconds
at_time.do(QUEUE.put, TAUTULLI.get_stats).tag("tautulli-{}-get_stats".format(server.id)) at_time.do(thread, TAUTULLI.get_stats).tag("tautulli-{}-get_stats".format(server.id))
if CONFIG.radarr_enabled: if CONFIG.radarr_enabled:
for server in CONFIG.radarr_servers: for server in CONFIG.radarr_servers:
RADARR = RadarrAPI(server, DBMANAGER) RADARR = RadarrAPI(server, DBMANAGER)
if server.get_missing: if server.get_missing:
at_time = schedule.every(server.get_missing_run_seconds).seconds at_time = schedule.every(server.get_missing_run_seconds).seconds
at_time.do(QUEUE.put, RADARR.get_missing).tag("radarr-{}-get_missing".format(server.id)) at_time.do(thread, RADARR.get_missing).tag("radarr-{}-get_missing".format(server.id))
if server.queue: if server.queue:
at_time = schedule.every(server.queue_run_seconds).seconds at_time = schedule.every(server.queue_run_seconds).seconds
at_time.do(QUEUE.put, RADARR.get_queue).tag("radarr-{}-get_queue".format(server.id)) at_time.do(thread, RADARR.get_queue).tag("radarr-{}-get_queue".format(server.id))
if CONFIG.lidarr_enabled: if CONFIG.lidarr_enabled:
for server in CONFIG.lidarr_servers: for server in CONFIG.lidarr_servers:
LIDARR = LidarrAPI(server, DBMANAGER) LIDARR = LidarrAPI(server, DBMANAGER)
if server.queue: if server.queue:
at_time = schedule.every(server.queue_run_seconds).seconds at_time = schedule.every(server.queue_run_seconds).seconds
at_time.do(QUEUE.put, LIDARR.get_queue, None).tag("lidarr-{}-get_queue".format(server.id)) at_time.do(thread, LIDARR.get_queue).tag("lidarr-{}-get_queue".format(server.id))
if server.missing_days > 0: if server.missing_days > 0:
at_time = schedule.every(server.missing_days_run_seconds).seconds at_time = schedule.every(server.missing_days_run_seconds).seconds
at_time.do(QUEUE.put, (LIDARR.get_calendar, "Missing")).tag( at_time.do(thread, (LIDARR.get_calendar, "Missing")).tag(
"lidarr-{}-get_missing".format(server.id)) "lidarr-{}-get_missing".format(server.id))
if server.future_days > 0: if server.future_days > 0:
at_time = schedule.every(server.future_days_run_seconds).seconds at_time = schedule.every(server.future_days_run_seconds).seconds
at_time.do(QUEUE.put, (LIDARR.get_calendar, "Future")).tag("lidarr-{}-get_future".format( at_time.do(thread, (LIDARR.get_calendar, "Future")).tag("lidarr-{}-get_future".format(
server.id)) server.id))
if CONFIG.ombi_enabled: if CONFIG.ombi_enabled:
@ -159,26 +155,26 @@ if __name__ == "__main__":
OMBI = OmbiAPI(server, DBMANAGER) OMBI = OmbiAPI(server, DBMANAGER)
if server.request_type_counts: if server.request_type_counts:
at_time = schedule.every(server.request_type_run_seconds).seconds at_time = schedule.every(server.request_type_run_seconds).seconds
at_time.do(QUEUE.put, OMBI.get_request_counts).tag("ombi-{}-get_request_counts".format(server.id)) at_time.do(thread, OMBI.get_request_counts).tag("ombi-{}-get_request_counts".format(server.id))
if server.request_total_counts: if server.request_total_counts:
at_time = schedule.every(server.request_total_run_seconds).seconds at_time = schedule.every(server.request_total_run_seconds).seconds
at_time.do(QUEUE.put, OMBI.get_all_requests).tag("ombi-{}-get_all_requests".format(server.id)) at_time.do(thread, OMBI.get_all_requests).tag("ombi-{}-get_all_requests".format(server.id))
if server.issue_status_counts: if server.issue_status_counts:
at_time = schedule.every(server.issue_status_run_seconds).seconds at_time = schedule.every(server.issue_status_run_seconds).seconds
at_time.do(QUEUE.put, OMBI.get_issue_counts).tag("ombi-{}-get_issue_counts".format(server.id)) at_time.do(thread, OMBI.get_issue_counts).tag("ombi-{}-get_issue_counts".format(server.id))
if CONFIG.sickchill_enabled: if CONFIG.sickchill_enabled:
for server in CONFIG.sickchill_servers: for server in CONFIG.sickchill_servers:
SICKCHILL = SickChillAPI(server, DBMANAGER) SICKCHILL = SickChillAPI(server, DBMANAGER)
if server.get_missing: if server.get_missing:
at_time = schedule.every(server.get_missing_run_seconds).seconds at_time = schedule.every(server.get_missing_run_seconds).seconds
at_time.do(QUEUE.put, SICKCHILL.get_missing).tag("sickchill-{}-get_missing".format(server.id)) at_time.do(thread, SICKCHILL.get_missing).tag("sickchill-{}-get_missing".format(server.id))
if CONFIG.unifi_enabled: if CONFIG.unifi_enabled:
for server in CONFIG.unifi_servers: for server in CONFIG.unifi_servers:
UNIFI = UniFiAPI(server, DBMANAGER) UNIFI = UniFiAPI(server, DBMANAGER)
at_time = schedule.every(server.get_usg_stats_run_seconds).seconds at_time = schedule.every(server.get_usg_stats_run_seconds).seconds
at_time.do(QUEUE.put, UNIFI.get_usg_stats).tag("unifi-{}-get_usg_stats".format(server.id)) at_time.do(thread, UNIFI.get_usg_stats).tag("unifi-{}-get_usg_stats".format(server.id))
# Run all on startup # Run all on startup
SERVICES_ENABLED = [CONFIG.ombi_enabled, CONFIG.radarr_enabled, CONFIG.tautulli_enabled, CONFIG.unifi_enabled, SERVICES_ENABLED = [CONFIG.ombi_enabled, CONFIG.radarr_enabled, CONFIG.tautulli_enabled, CONFIG.unifi_enabled,
@ -187,9 +183,6 @@ if __name__ == "__main__":
vl.logger.error("All services disabled. Exiting") vl.logger.error("All services disabled. Exiting")
exit(1) exit(1)
WORKER = Thread(target=thread)
WORKER.start()
schedule.run_all() schedule.run_all()
while schedule.jobs: while schedule.jobs: