#!/usr/bin/env python3
#
# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
# Version 1.0 - 20191107
#
# Requires:
#   - Python 3.x
#       
import json
import urllib.request
import argparse
import sys
from urllib.parse import urlparse
import re

parser = argparse.ArgumentParser(description="Collect all page events from Netskope API and process domains by category and confidence")
parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
parser.add_argument("token", type=str, help="Tenant API Token")
parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)")
parser.add_argument("-r", "--records", type=int, default=100, help="# of records (default: 100)")
parser.add_argument("-v", "--verbose", action='store_true', help="verbose")
parser.add_argument("-d", "--debug", action='store_true', help="debug")

try:
	args = parser.parse_args()
	tenant = args.tenant
	token = args.token
	timeperiod = args.timeperiod
	records = args.records
	verbose = args.verbose
	debug = args.debug

except argparse.ArgumentError as e:
	print(str(e))

cursor_up = '\x1b[1A'
erase_line = '\x1b[2K'
cct_list = ["Cloud Storage", "Webmail"]
ccl_list = ["low", "poor"]
whitelist = re.compile("bla")
ioc_list = []
i = 0

if verbose:
	print("Using Categories: ", end='', flush=True)
	print(", ".join(map(str,cct_list)))
	print("Using Rating: ", end='', flush=True)
	print(", ".join(map(str,ccl_list)))
	print(f"Applying Whitelist for: {whitelist.pattern}")

def get_json(type):
	domain = "goskope.com"
	url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}"
	req = urllib.request.Request(url)
	with urllib.request.urlopen(req) as response:
		content = response.read()
	json_data = json.loads(content)
	if debug: print (json_data)
	return(json_data)

print()
print("Processing...", end='', flush=True)
json_content = get_json("page")
sys.stdout.write(cursor_up)
sys.stdout.write(erase_line)
print()

if verbose:
	print(f"{'#':>4}  {'Domain':<50s} Confidence")
	print("#######################################################################")

for index, data in enumerate(json_content['data']):
	if not "domain" in data:
		domain = urlparse(data["url"]).netloc
	else:
		domain = data["domain"]
	if whitelist.search(domain):
		continue
	if data["category"] in cct_list:
		if data["ccl"] in ccl_list:
			if domain not in ioc_list:
				i += 1
				if verbose: print(f"{i:>4}) {domain:<50s} {data['ccl']}")
				ioc_list.append(domain)
				if i == records:
					break

if verbose: print()
print(", ".join(map(str,ioc_list)))