d_mcknight/Varken
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

bit more import scoping and added filter for using /location with domain

Browse source
This commit is contained in:
Nicholas St. Germain 2018-12-17 20:19:41 -06:00
parent 61fcf3b80c
commit fbc8488b07
3 changed files with 33 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

58
varken/helpers.py
View file

@ -1,17 +1,17 @@
import os from time import time
import time from hashlib import md5
import tarfile from tarfile import open
import hashlib from logging import getLogger
import urllib3 from geoip2.database import Reader
import geoip2.database from urllib3 import disable_warnings
import logging from os import stat, remove, makedirs
from json.decoder import JSONDecodeError
from os.path import abspath, join
from requests.exceptions import InvalidSchema, SSLError, ConnectionError
from urllib.request import urlretrieve from urllib.request import urlretrieve
from json.decoder import JSONDecodeError
from os.path import abspath, join, basename
from urllib3.exceptions import InsecureRequestWarning
from requests.exceptions import InvalidSchema, SSLError, ConnectionError
logger = logging.getLogger() logger = getLogger()
def geoip_download(data_folder): def geoip_download(data_folder):
@ -23,48 +23,48 @@ def geoip_download(data_folder):
logger.info('Downloading GeoLite2 from %s', url) logger.info('Downloading GeoLite2 from %s', url)
urlretrieve(url, tar_dbfile) urlretrieve(url, tar_dbfile)
tar = tarfile.open(tar_dbfile, 'r:gz') tar = open(tar_dbfile, 'r:gz')
logging.debug('Opening GeoLite2 tar file : %s', tar_dbfile) logger.debug('Opening GeoLite2 tar file : %s', tar_dbfile)
for files in tar.getmembers(): for files in tar.getmembers():
if 'GeoLite2-City.mmdb' in files.name: if 'GeoLite2-City.mmdb' in files.name:
logging.debug('"GeoLite2-City.mmdb" FOUND in tar file') logger.debug('"GeoLite2-City.mmdb" FOUND in tar file')
files.name = os.path.basename(files.name) files.name = basename(files.name)
tar.extract(files, datafolder) tar.extract(files, datafolder)
logging.debug('%s has been extracted to %s', files, datafolder) logger.debug('%s has been extracted to %s', files, datafolder)
os.remove(tar_dbfile) remove(tar_dbfile)
def geo_lookup(ipaddress, data_folder): def geo_lookup(ipaddress, data_folder):
datafolder = data_folder datafolder = data_folder
logging.debug('Reading GeoLite2 DB from %s', datafolder) logger.debug('Reading GeoLite2 DB from %s', datafolder)
dbfile = abspath(join(datafolder, 'GeoLite2-City.mmdb')) dbfile = abspath(join(datafolder, 'GeoLite2-City.mmdb'))
now = time.time() now = time()
try: try:
dbinfo = os.stat(dbfile) dbinfo = stat(dbfile)
db_age = now - dbinfo.st_ctime db_age = now - dbinfo.st_ctime
if db_age > (35 * 86400): if db_age > (35 * 86400):
logging.info('GeoLite2 DB is older than 35 days. Attempting to re-download...') logger.info('GeoLite2 DB is older than 35 days. Attempting to re-download...')
os.remove(dbfile) remove(dbfile)
geoip_download(datafolder) geoip_download(datafolder)
except FileNotFoundError: except FileNotFoundError:
logging.error('GeoLite2 DB not found. Attempting to download...') logger.error('GeoLite2 DB not found. Attempting to download...')
geoip_download(datafolder) geoip_download(datafolder)
reader = geoip2.database.Reader(dbfile) reader = Reader(dbfile)
return reader.city(ipaddress) return reader.city(ipaddress)
def hashit(string): def hashit(string):
encoded = string.encode() encoded = string.encode()
hashed = hashlib.md5(encoded).hexdigest() hashed = md5(encoded).hexdigest()
return hashed return hashed
@ -75,7 +75,7 @@ def connection_handler(session, request, verify):
v = verify v = verify
return_json = False return_json = False
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) disable_warnings(InsecureRequestWarning)
try: try:
get = s.send(r, verify=v) get = s.send(r, verify=v)
@ -106,10 +106,10 @@ def connection_handler(session, request, verify):
def mkdir_p(path): def mkdir_p(path):
templogger = logging.getLogger('temp') templogger = getLogger('temp')
try: try:
templogger.info('Creating folder %s ', path) templogger.info('Creating folder %s ', path)
os.makedirs(path, exist_ok=True) makedirs(path, exist_ok=True)
except Exception as e: except Exception as e:
templogger.error('Could not create folder %s : %s ', path, e) templogger.error('Could not create folder %s : %s ', path, e)

3
varken/iniparser.py
View file

@ -44,6 +44,9 @@ class INIParser(object):
filtered_strings = [section.get(k) for key, section in self.config.items() filtered_strings = [section.get(k) for key, section in self.config.items()
for k in section if k in BlacklistFilter.blacklisted_strings] for k in section if k in BlacklistFilter.blacklisted_strings]
self.filtered_strings = list(filter(None, filtered_strings)) self.filtered_strings = list(filter(None, filtered_strings))
# Added matching for domains that use /locations. ConnectionPool ignores the location in logs
domains_only = list([ string.split('/')[0] for string in filtered_strings if '/' in string ])
self.filtered_strings.extend(domains_only)
for handler in self.logger.handlers: for handler in self.logger.handlers:
handler.addFilter(BlacklistFilter(set(self.filtered_strings))) handler.addFilter(BlacklistFilter(set(self.filtered_strings)))

2
varken/varkenlogger.py
View file

@ -45,7 +45,7 @@ class VarkenLogger(object):
self.log_level = INFO self.log_level = INFO
# Make the log directory if it does not exist # Make the log directory if it does not exist
mkdir_p('{}/{}'.format(self.data_folder, BlacklistFilter.log_folder)) mkdir_p(f'{self.data_folder}/{BlacklistFilter.log_folder}')
# Create the Logger # Create the Logger
self.logger = getLogger() self.logger = getLogger()