Initial commit

.gitignore

@@ -0,0 +1,3 @@
+raw
+*.cnf
+zscaler.txt

Netskope_APIEvents-01.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+import json
+import urllib.request
+import argparse
+import collections
+from operator import itemgetter
+
+parser = argparse.ArgumentParser(description="API Call to collect data")
+parser.add_argument("tenant", type=str, help="Tenant Name")
+parser.add_argument("token", type=str, help="Tenat API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod (default: 604800)")
+
+try:
+	args = parser.parse_args()
+	tenant = args.tenant
+	token = args.token
+	timeperiod = args.timeperiod
+
+except argparse.ArgumentError as e:
+	print(str(e))
+
+def print_dict(dict):
+	for key, value in sorted(dict.items(), key = itemgetter(1), reverse = True):
+		print ("{:<35s}{:5d}".format(key, value))
+
+base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod)
+
+req = urllib.request.Request(base_url)
+with urllib.request.urlopen(req) as response:
+	content = response.read()
+json_content = json.loads(content)
+
+domains = collections.Counter()
+categories = collections.Counter()
+
+for i in range (0, len (json_content['data'])):
+	domain = json_content["data"][i]["domain"]
+	ccl = json_content["data"][i]["ccl"]
+	category = json_content["data"][i]["category"]
+	domains[domain] += 1
+	categories[category][count] += 1
+
+
+#print ("===== Domains =====")
+#print_dict(domains)
+
+
+print ("\n===== Categories =====")
+print_dict(categories)
+

Netskope_APIEvents-02.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+import json
+import urllib.request
+import argparse
+from collections import Counter
+from operator import itemgetter
+
+parser = argparse.ArgumentParser(description="API Call to collect data")
+parser.add_argument("tenant", type=str, help="Tenant Name")
+parser.add_argument("token", type=str, help="Tenat API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod (default: 604800)")
+
+try:
+	args = parser.parse_args()
+	tenant = args.tenant
+	token = args.token
+	timeperiod = args.timeperiod
+
+except argparse.ArgumentError as e:
+	print(str(e))
+
+def print_dict(dict, json_content):
+	for key, value in sorted(dict.items(), key = itemgetter(1), reverse = True):
+		print ("{:<35s}{:5d}".format(key, value))
+
+base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod)
+req = urllib.request.Request(base_url)
+with urllib.request.urlopen(req) as response:
+	content = response.read()
+json_content = json.loads(content)
+
+domains = Counter(data['domain'] for data in json_content['data'])
+categories = Counter(data['category'] for data in json_content['data'])
+
+print ("==== Domains ===")
+print_dict (domains, json_content)
+print ("\n")
+print ("==== Categories ===")
+print_dict (categories)

Netskope_APIEvents-03.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191028
+#
+# Requires:
+#   - Python 3.x
+#       
+import json
+import urllib.request
+import argparse
+from collections import Counter
+from operator import itemgetter
+
+parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope")
+parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
+parser.add_argument("token", type=str, help="Tenat API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 604800)")
+parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)")
+parser.add_argument("-s", "--show", action='store_true', help="Show category hits")
+
+try:
+	args = parser.parse_args()
+	tenant = args.tenant
+	token = args.token
+	timeperiod = args.timeperiod
+	rows = args.rows
+	show = args.show
+
+except argparse.ArgumentError as e:
+	print(str(e))
+
+base_url = "https://{}.goskope.com/api/v1/events?token={}&type=application&timeperiod={}".format(tenant, token, timeperiod)
+
+req = urllib.request.Request(base_url)
+with urllib.request.urlopen(req) as response:
+	content = response.read()
+json_content = json.loads(content)
+
+domain_count = Counter()
+domain_category = {}
+category_count = Counter()
+rows = None if rows == 0 else rows
+
+for i in range (0, len (json_content['data'])):
+	domain = json_content["data"][i]["domain"]
+	ccl = json_content["data"][i]["ccl"]
+	category = json_content["data"][i]["category"]
+	domain_count[domain] += 1
+	domain_category[domain] = category
+	category_count[category] += 1
+
+top_domains = domain_count.most_common(rows)
+print ("{:<40s}{:>5s} - {}".format("Domain", "Hits", "Category"))
+print ("################################################################################")
+for i in top_domains: 
+	print ("{:<40s}{:5d} - {}".format(i[0], i[1], domain_category[i[0]]))
+
+print ("")
+if show:
+	top_categories = category_count.most_common()
+	print ("{:<40s}{:>5s}".format("Category", "Hits"))
+	print ("################################################################################")
+	for i in top_categories: 
+		print ("{:<40s}{:5d}".format(i[0], i[1]))
+

Netskope_APIEvents-04.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191028
+#
+# Requires:
+#   - Python 3.x
+#       
+import json
+import urllib.request
+import argparse
+import collections
+from operator import itemgetter
+
+parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope")
+parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
+parser.add_argument("token", type=str, help="Tenat API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 604800)")
+parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)")
+parser.add_argument("-s", "--show", action='store_true', help="Show category hits")
+
+try:
+	args = parser.parse_args()
+	tenant = args.tenant
+	token = args.token
+	timeperiod = args.timeperiod
+	rows = args.rows
+	show = args.show
+
+except argparse.ArgumentError as e:
+	print(str(e))
+
+base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod)
+
+req = urllib.request.Request(base_url)
+with urllib.request.urlopen(req) as response:
+	content = response.read()
+json_content = json.loads(content)
+
+#site = {'data': []}
+site = collections.defaultdict(list);
+
+rows = None if rows == 0 else rows
+
+for i in range (0, len (json_content['data'])):
+	json_site = json_content["data"][i]["site"]
+	json_domain = json_content["data"][i]["domain"]
+
+	if json_domain not in site[json_site]:
+		site[json_site].append(json_domain)
+	#print (json_site, "-", json_domain)
+
+print (site)
+       
+for key, value in sorted(site.items(), key = itemgetter(0), reverse = False):
+	print ("{:<35s}".format(key), end="")
+	for i in value:
+		print ("{},".format(i), end="")
+	print ("")
+
+
+#top_domains = domain_count.most_common(rows)
+#print ("{:<40s}{:>5s} - {}".format("Domain", "Hits", "Category"))
+#print ("################################################################################")
+#for i in top_domains:
+	#print ("{:<40s}{:5d} - {}".format(i[0], i[1], domain_category[i[0]]))
+        
+

Netskope_APIEvents-05.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191028
+#
+# Requires:
+#   - Python 3.x
+#       
+import json
+import urllib.request
+import argparse
+from collections import Counter
+from operator import itemgetter
+
+parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope")
+parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
+parser.add_argument("token", type=str, help="Tenat API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)")
+parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)")
+parser.add_argument("-s", "--show", action='store_true', help="Show category hits")
+parser.add_argument("-d", "--debug", action='store_true', help="debug")
+
+try:
+	args = parser.parse_args()
+	tenant = args.tenant
+	token = args.token
+	timeperiod = args.timeperiod
+	rows = args.rows
+	show = args.show
+	debug = args.debug
+
+except argparse.ArgumentError as e:
+	print(str(e))
+
+domain_count = Counter()
+domain_category = {}
+category_count = Counter()
+rows = None if rows == 0 else rows
+
+def get_json(type):
+	domain = "goskope.com"
+	url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}"
+	req = urllib.request.Request(url)
+	with urllib.request.urlopen(req) as response:
+		content = response.read()
+	json_data = json.loads(content)
+	if debug: print (json_data)
+	return(json_data)
+
+json_content = get_json("application")
+for i in range (0, len (json_content['data'])):
+	domain = json_content["data"][i]["domain"]
+	ccl = json_content["data"][i]["ccl"]
+	category = json_content["data"][i]["category"]
+	domain_count[domain] += 1
+	domain_category[domain] = category
+	category_count[category] += 1
+
+top_domains = domain_count.most_common(rows)
+print (f"{'Domain':<40s}{'Hits':>5s} - Category")
+print ("################################################################################")
+for i in top_domains: 
+	print (f"{i[0]:<40s}{i[1]:5d} - {domain_category[i[0]]}")
+
+print ("")
+if show:
+	top_categories = category_count.most_common()
+	print (f"{'Category':<40s}{'Hits':>5s}")
+	print ("################################################################################")
+	for i in top_categories: 
+		print (f"{i[0]:<40s}{i[1]:5d}")

Netskope_APIEvents-06.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191107
+#
+# Requires:
+#   - Python 3.x
+#       
+import json
+import urllib.request
+import argparse
+import sys
+from urllib.parse import urlparse
+import re
+
+parser = argparse.ArgumentParser(description="Collect all page events from Netskope API and process domains by category and confidence")
+parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
+parser.add_argument("token", type=str, help="Tenant API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)")
+parser.add_argument("-r", "--records", type=int, default=100, help="# of records (default: 100)")
+parser.add_argument("-v", "--verbose", action='store_true', help="verbose")
+parser.add_argument("-d", "--debug", action='store_true', help="debug")
+
+try:
+	args = parser.parse_args()
+	tenant = args.tenant
+	token = args.token
+	timeperiod = args.timeperiod
+	records = args.records
+	verbose = args.verbose
+	debug = args.debug
+
+except argparse.ArgumentError as e:
+	print(str(e))
+
+cursor_up = '\x1b[1A'
+erase_line = '\x1b[2K'
+cct_list = ["Cloud Storage", "Webmail"]
+ccl_list = ["low", "poor"]
+whitelist = re.compile("bla")
+ioc_list = []
+i = 0
+
+if verbose:
+	print("Using Categories: ", end='', flush=True)
+	print(", ".join(map(str,cct_list)))
+	print("Using Rating: ", end='', flush=True)
+	print(", ".join(map(str,ccl_list)))
+	print(f"Applying Whitelist for: {whitelist.pattern}")
+
+def get_json(type):
+	domain = "goskope.com"
+	url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}"
+	req = urllib.request.Request(url)
+	with urllib.request.urlopen(req) as response:
+		content = response.read()
+	json_data = json.loads(content)
+	if debug: print (json_data)
+	return(json_data)
+
+print()
+print("Processing...", end='', flush=True)
+json_content = get_json("page")
+sys.stdout.write(cursor_up)
+sys.stdout.write(erase_line)
+print()
+
+if verbose:
+	print(f"{'#':>4}  {'Domain':<50s} Confidence")
+	print("#######################################################################")
+
+for index, data in enumerate(json_content['data']):
+	if not "domain" in data:
+		domain = urlparse(data["url"]).netloc
+	else:
+		domain = data["domain"]
+	if whitelist.search(domain):
+		continue
+	if data["category"] in cct_list:
+		if data["ccl"] in ccl_list:
+			if domain not in ioc_list:
+				i += 1
+				if verbose: print(f"{i:>4}) {domain:<50s} {data['ccl']}")
+				ioc_list.append(domain)
+				if i == records:
+					break
+
+if verbose: print()
+print(", ".join(map(str,ioc_list)))

Netskope_APIEvents-07.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191107
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# Requires:
+#   - Python 3.x
+#       
+import json
+import urllib.request
+import argparse
+import sys
+import urllib.parse
+import re
+import requests
+
+parser = argparse.ArgumentParser(description="Collect all page events from Netskope API and process domains by category and confidence")
+parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
+parser.add_argument("token", type=str, help="Tenant API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)")
+parser.add_argument("-r", "--records", type=int, default=100, help="# of records (default: 100)")
+parser.add_argument("-v", "--verbose", action='store_true', help="verbose")
+parser.add_argument("-d", "--debug", action='store_true', help="print raw json data")
+
+try:
+	args = parser.parse_args()
+	tenant = args.tenant
+	token = args.token
+	timeperiod = args.timeperiod
+	records = args.records
+	verbose = args.verbose
+	debug = args.debug
+
+except argparse.ArgumentError as e:
+	print(str(e))
+
+cct_list = ["Cloud Storage", "Webmail"]
+ccl_list = ["low", "poor"]
+whitelist = re.compile("yahoo")
+ioc_list = []
+
+if verbose:
+	print("Using Categories: ", end='', flush=True)
+	print(", ".join(map(str,cct_list)))
+	print("Using Rating: ", end='', flush=True)
+	print(", ".join(map(str,ccl_list)))
+	print(f"Applying Whitelist: {whitelist.pattern}")
+	print()
+	print(f"{'#':>4}  {'Domain':<50s} Confidence")
+	print("#######################################################################")
+
+def get_json(type):
+	domain = "goskope.com"
+	url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}"
+	req = urllib.request.Request(url)
+	with urllib.request.urlopen(req) as response:
+		content = response.read()
+	json_data = json.loads(content)
+	if debug: print (json_data)
+	return(json_data)
+
+def parse_json(json_content):
+	i = 0
+	for index, data in enumerate(json_content['data']):
+		if not "domain" in data:
+			domain = urllib.parse.urlparse(data["url"]).netloc
+		else:
+			domain = data["domain"]
+		if whitelist.search(domain):
+			continue
+		if data["category"] in cct_list:
+			if data["ccl"] in ccl_list:
+				if domain not in ioc_list:
+					i += 1
+					if verbose: print(f"{i:>4}) {domain:<50s} {data['ccl']}")
+					ioc_list.append(domain)
+
+	return ioc_list
+	#domain_list = ", ".join(map(str,ioc_list[:records]))
+	#return domain_list
+
+json = get_json("page")
+print(parse_json(json))

Netskope_APIEvents-08.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191107
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# Requires:
+#   - Python 3.x
+#       
+import os
+import sys
+import json
+import time
+import re
+import logging
+import urllib.parse
+import requests
+
+NTSKP_TENANT = 'https://astrazeneca.eu.goskope.com'
+NTSKP_TOKEN = '604d0a3b26ea9b22c3ec42130ebbfa8e'
+NTSKP_PERIOD = '2592000'
+cct_list = ["Cloud Storage", "Webmail"]
+ccl_list = ["low", "poor"]
+whitelist = re.compile("yahoo")
+ioc_list = []
+
+ZS_MAX_DOMAINS = 2
+headers = {'Content-Type': 'application/json', 'Cache-Control': 'no-cache', 'User-Agent': 'Netskope_ZscalerImporter1.0'}
+PROXY=''
+
+logging.basicConfig(level=logging.DEBUG)
+logging = logging.getLogger('zsc')
+
+def ntskp_get_domains(headers):
+	uri = f"{NTSKP_TENANT}/api/v1/events?token={NTSKP_TOKEN}&type=page&timeperiod={NTSKP_PERIOD}"
+	try:
+		r = requests.get(uri, headers=headers, proxies=PROXY)
+		r.raise_for_status()
+	except Exception as e:
+		logging.error('Error: ' + str(e))
+		sys.exit(1)
+	json = r.json()
+	limit = (len(json['data']))
+	
+	for item in json['data']:
+		if not "domain" in item:
+			domain = urllib.parse.urlparse(item['url']).netloc
+		else:
+			domain = item['domain']
+		if whitelist.search(domain):
+			continue
+		if item['category'] in cct_list:
+			if item['ccl'] in ccl_list:
+				if domain not in ioc_list:
+					print(f"{domain:<50s} {item['ccl']}")
+					endtime = item['timestamp']
+					ioc_list.append(domain)
+	print(limit)
+	print(endtime)
+	starttime = endtime - (10 * 60)
+	print(ioc_list[:ZS_MAX_DOMAINS])
+	return ioc_list[:ZS_MAX_DOMAINS]
+
+
+ntskp_get_domains(headers)
+
+now = int(time.time() * 1000)
+print(now)
+#print(str(time.ctime(int(time.time()))))

Netskope_APIEvents-09.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191028
+#
+# Collects all the page events, counts all the domain hits and category hits
+#
+# Requires:
+#   - Python 3.x
+#       
+import json
+import urllib.request
+import argparse
+from collections import Counter
+from operator import itemgetter
+
+parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope")
+parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
+parser.add_argument("token", type=str, help="Tenat API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)")
+parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)")
+parser.add_argument("-s", "--show", action='store_true', help="Show category hits")
+parser.add_argument("-d", "--debug", action='store_true', help="debug")
+
+try:
+	args = parser.parse_args()
+	tenant = args.tenant
+	token = args.token
+	timeperiod = args.timeperiod
+	rows = args.rows
+	show = args.show
+	debug = args.debug
+
+except argparse.ArgumentError as e:
+	print(str(e))
+
+domain_count = Counter()
+domain_category = {}
+domain_ccl = {}
+domain_cci = {}
+category_count = Counter()
+rows = None if rows == 0 else rows
+
+def get_json(type):
+	domain = "goskope.com"
+	url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}"
+	req = urllib.request.Request(url)
+	with urllib.request.urlopen(req) as response:
+		content = response.read()
+	json_data = json.loads(content)
+	if debug: print (json_data)
+	print(json.dumps(json_data, indent=4, sort_keys=True))
+	return(json_data)
+
+json_content = get_json("page")
+for i in range (0, len (json_content['data'])):
+	domain = json_content["data"][i]["domain"]
+	ccl = json_content["data"][i]["ccl"]
+	category = json_content["data"][i]["category"]
+	#ccl = json_content["data"][i]["ccl"]
+	cci = json_content["data"][i]["cci"]
+	domain_count[domain] += 1
+	domain_category[domain] = category
+	domain_ccl[domain] = ccl
+	domain_cci[domain] = cci
+category_count[category] += 1
+
+top_domains = domain_count.most_common(rows)
+print (f"{'Domain':<40s}{'Hits':>5s} - Category")
+print ("################################################################################")
+for i in top_domains: 
+	print (f"{i[0]:<40s}{i[1]:5d} - {domain_category[i[0]]} - {domain_ccl[i[0]]}")
+
+print ("")
+if show:
+	top_categories = category_count.most_common()
+	print (f"{'Category':<40s}{'Hits':>5s}")
+	print ("################################################################################")
+	for i in top_categories: 
+		print (f"{i[0]:<40s}{i[1]:5d}")

Netskope_APIEvents-10.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+#
+import os
+import sys
+import json
+import time
+import re
+import logging
+import urllib.parse
+import requests
+import configparser
+from datetime import datetime
+
+###############################################
+
+CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"
+if not os.path.isfile(CONFIG_FILE):
+	logging.error(f"The config file {CONFIG_FILE} doesn't exist")
+	sys.exit(1)
+config = configparser.RawConfigParser()
+config.read(CONFIG_FILE)
+NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT')
+NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN')
+NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD')
+NTSKP_SCORE = config.get('netskope', 'NTSKP_SCORE')
+NTSKP_CATEGORIES = config.get('netskope', 'NTSKP_CATEGORIES')
+NTSKP_CONFIDENCE = config.get('netskope', 'NTSKP_CONFIDENCE')
+PROXY = config.get('general', 'PROXY')
+
+###############################################
+
+# Use a custom user-agent string
+UA_STRING = 'NetskopeAPICollector1.0'
+
+# Set logging.INFO to logging.DEBUG for debug information
+logging.basicConfig(level=logging.INFO)
+logging = logging.getLogger('NetskopeAPICollector')
+
+###############################################
+
+def ntskp_get_domains(headers):
+	skip = 0
+	filename = f"/home/mischa/netskope/api-{datetime.now().strftime('%Y%m%d')}.txt"
+	logging.info(f"File {filename} created")
+	ssl_session = requests.Session()
+	logging.debug(f"{ssl_session}")
+
+	while True:
+		uri = f'{NTSKP_TENANT}/api/v1/events?token={NTSKP_TOKEN}&type=page&timeperiod={NTSKP_PERIOD}&skip={skip}'
+		try:
+			r = ssl_session.get(uri, headers=headers, proxies=PROXY)
+			r.raise_for_status()
+		except Exception as e:
+			logging.error(f'Error: {str(e)}')
+			sys.exit(1)
+		json = r.json()
+		#if json['data']:
+		if 'data' in json:
+			if len(json['data']) <= 5000:
+				skip += 5000
+				filter_file = open(filename, "a")
+				logging.debug(f"File {filename} opened")
+				for item in json['data']:
+					if not 'domain' in item:
+						domain = urllib.parse.urlparse(item['url']).netloc
+					else:
+						domain = item['domain']
+
+					#if NTSKP_SAFELIST.search(domain):
+						#print(domain)
+
+					#if item['ccl'] in NTSKP_CONFIDENCE:
+					utctime = datetime.utcfromtimestamp(item['timestamp']).strftime('%Y-%m-%d %H:%M:%S')
+					filter_file.write(f"{utctime},{domain},{item['cci']},{item['category']},{item['ccl']},{item['user']}\n")
+				filter_file.close()
+				logging.debug(f"File {filename} closed")
+				logging.debug(f"Next request, skip: {skip}")
+			else:
+				logging.info(f"No more data to collect")
+				break
+		else:
+			logging.info(f"No more data to collect")
+			break
+		if skip == 500000:
+			logging.info(f"Reached limit")
+			break
+
+###############################################
+
+request_headers = {'Content-Type': 'application/json', 'Cache-Control': 'no-cache', 'User-Agent': UA_STRING}
+ntskp_get_domains(request_headers)

Netskope_APIReport-01.pl

@@ -0,0 +1,86 @@
+#!/usr/bin/perl -w
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use HTTP::Tiny;
+use JSON::PP;
+use Text::CSV;
+use File::Temp;
+use MIME::Lite;
+
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+
+my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+my $response = HTTP::Tiny->new->get($uri);
+my $json = JSON::PP->new->utf8->decode($response->{'content'});
+my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+my %files;
+
+for my $widget (@{$data}) {
+	my $tmp_file = File::Temp->new(UNLINK => 0, TEMPLATE => 'tempXXXXX', DIR => '/tmp');
+	$files{$tmp_file} = $widget->{'name'};
+	$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+	$response = HTTP::Tiny->new->get($uri);
+	open my $fh_out, ">", $tmp_file;
+	print $fh_out $response->{'content'};
+	close $fh_out;
+}
+
+my $out_email = "azblocklist.csv";
+my $out_zscaler = "zscaler.txt";
+open my $fh_email, ">", $out_email;
+open my $fh_zscaler, ">", $out_zscaler;
+
+for my $item (keys %files) {
+	my $count = 0;
+	my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+	open my $fh, "<", $item;
+	my $header = $csv->getline($fh);
+
+	print "$files{$item}\n";
+	print $fh_email "$files{$item}\n";
+
+	while (my $row = $csv->getline($fh)) {
+		last if ($count == 30);
+		if ($row->[1] =~ m/,/) {
+			my @domains = split "," , $row->[1];
+			for my $domain (@domains) {
+				print "$domain,";
+				print $fh_email "$domain,";
+				print $fh_zscaler "$domain\n";
+			}
+		} else {
+			print "$row->[1],";
+			print $fh_email "$row->[1],";
+			print $fh_zscaler "$row->[1]\n";
+		}
+		$count++;
+	}
+	print "\n";
+	print $fh_email "\n";
+	close $fh;
+	unlink $item;
+}
+close $fh_email;
+close $fh_zscaler;
+
+my $msg = MIME::Lite->new(
+    From    => 'mischa@high5.nl',
+    To      => 'mischa@netskope.com',
+    Cc      => 'mischa@high5.nl',
+    Subject => 'AztraZeneca Netskope Blocklist',
+    Type    => 'TEXT',
+    Data    => "Domains pushed to Zscaler for blocking\n\n"
+);
+$msg->attach(
+    Type     => 'text/csv',
+    Path     => $out_email,
+    Filename => $out_email
+);
+$msg->send('smtp','mail.high5.nl', Debug=>0);
+unlink $out_email;

Netskope_OPLPUploader-01.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# OPLPUploader.sh - Version 1.0 - 20200113
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# find ${LOCALDIR} -type f -name "*.csv" -maxdepth 1 | sort -V
+
+HOST="ftp://" 
+LOCALDIR="/tmp/files"
+REMOTEDIR="/nslogs/user/upload/custom-ASML_SplunkCurlv1/"
+USER=''
+PASS=''
+GLOB='*.csv'
+LOG="/tmp/script.log"
+
+if [ -d ${LOCALDIR} ]; then
+	cd ${LOCALDIR}
+else
+	echo "$(date "+%Y-%m-%d %T") ${LOCALDIR} doesn't exist" | tee -a ${LOG}
+	exit 1
+fi
+
+lftp ${HOST} <<- UPLOAD
+	user "${USER}" "${PASS}"
+	cd "${REMOTEDIR}"
+	mput -E "${GLOB}"
+UPLOAD
+
+if [ ! $? -eq 0 ]; then
+	echo "$(date "+%Y-%m-%d %T") unable to upload files" | tee -a ${LOG}
+	exit 1
+fi

Netskope_ZScalerImporter-01.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019-2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.py - Version 2.0 - 20200611
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+import os
+import sys
+import re
+import json
+import csv
+import time
+import logging
+import urllib.parse
+import requests
+import configparser
+
+###############################################
+
+CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"
+if not os.path.isfile(CONFIG_FILE):
+	logging.error(f"The config file {CONFIG_FILE} doesn't exist")
+	sys.exit(1)
+config = configparser.RawConfigParser()
+config.read(CONFIG_FILE)
+NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT')
+NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN')
+ZS_MAX_DOMAINS = int(config.get('zscaler', 'ZS_MAX_DOMAINS'))
+ZS_BASE_URI = config.get('zscaler', 'ZS_BASE_URI')
+ZS_API_KEY = config.get('zscaler', 'ZS_API_KEY')
+ZS_API_USERNAME = config.get('zscaler', 'ZS_API_USERNAME')
+ZS_API_PASSWORD = config.get('zscaler', 'ZS_API_PASSWORD')
+ZS_CATEGORY_NAME = config.get('zscaler', 'ZS_CATEGORY_NAME')
+ZS_CATEGORY_DESC = config.get('zscaler', 'ZS_CATEGORY_DESC')
+PROXY = config.get('general', 'PROXY')
+
+###############################################
+
+# Use a custom user-agent string
+UA_STRING = 'Netskope_ZScalerImporter1.0'
+
+# Set logging.INFO to logging.DEBUG for debug information
+logging.basicConfig(level=logging.DEBUG)
+logging = logging.getLogger('Netskope_ZScalerImporter')
+
+def ntskp_get_domains():
+	ioc_list = []
+	with open('zscaler.txt') as f:
+		ioc_list = f.read().splitlines()
+	logging.debug(ioc_list[:ZS_MAX_DOMAINS])
+	return ioc_list[:ZS_MAX_DOMAINS]
+
+def zs_auth(headers):
+	# Authenticatie against ZScaler API, fetch and return JSESSIONID
+	now = int(time.time() * 1000)
+	n = str(now)[-6:]
+	r = str(int(n) >> 1).zfill(6)
+	key = ""
+	for i in range(0, len(str(n)), 1):
+		key += ZS_API_KEY[int(str(n)[i])]
+	for j in range(0, len(str(r)), 1):
+		key += ZS_API_KEY[int(str(r)[j])+2]
+
+	uri = f'{ZS_BASE_URI}/authenticatedSession'
+	body = {'apiKey': key,
+		'username': ZS_API_USERNAME,
+		'password': ZS_API_PASSWORD,
+		'timestamp': now}
+	try:
+		r = requests.post(uri, data=json.dumps(body), headers=headers, proxies=PROXY)
+		r.raise_for_status()
+		jsessionid = re.sub(r';.*$', "", r.headers['Set-Cookie'])
+	except Exception as e:
+		logging.error(f'Error: {str(e)}')
+		sys.exit(1)
+	return jsessionid
+
+def zs_get_categories(headers):
+	# Find any existing categories matching ZS_CATEGORY_NAME
+	uri = f'{ZS_BASE_URI}/urlCategories/lite'
+	try:
+		r = requests.get(uri, headers=headers, proxies=PROXY)
+		r.raise_for_status()
+	except Exception as e:
+		logging.error(f'Error: {str(e)}')
+		sys.exit(1)
+	data = r.json()
+	for item in data:
+		if item.get('configuredName') == ZS_CATEGORY_NAME:
+			return item.get('id')
+	return None
+
+def zs_update_categories(headers, domains, id = None):
+	# Update the ZS_CATEGORY_NAME with blocklist from Netskope
+	description = f'{ZS_CATEGORY_DESC}\n\nLast Updated: {str(time.ctime(int(time.time())))}'
+	body = {'configuredName': ZS_CATEGORY_NAME,
+		'customCategory': 'true',
+		'superCategory': 'SECURITY',
+		'urls': domains,
+		'description': description}
+	try:
+		if id == None:
+			uri = f'{ZS_BASE_URI}/urlCategories'
+			r = requests.post(uri, json=body, headers=headers, proxies=PROXY)
+		else:
+			uri = f'{ZS_BASE_URI}/urlCategories/{str(id)}'
+			r = requests.put(uri, json=body, headers=headers, proxies=PROXY)
+		r.raise_for_status()
+	except Exception as e:
+		logging.error(f'Error: {str(e)}')
+		sys.exit(1)
+	return None
+
+def zs_logout(headers):
+	# Logout from ZScaler
+	uri = f'{ZS_BASE_URI}/authenticatedSession'
+	try:
+		r = requests.delete(uri, headers=headers, proxies=PROXY)
+		r.raise_for_status()
+	except Exception as e:
+		logging.error(f'Error: {str(e)}')
+		sys.exit(1)
+	return None
+
+##############################################
+
+request_headers = {'Content-Type': 'application/json', 'Cache-Control': 'no-cache', 'User-Agent': UA_STRING}
+domains = ntskp_get_domains()
+request_headers['Cookie'] = zs_auth(request_headers)
+zs_update_categories(request_headers, domains, zs_get_categories(request_headers))
+zs_logout(request_headers)
+logging.info(f'Netskope added {str(len(domains))} domains added to ZScaler custom URL category {ZS_CATEGORY_NAME}')

Netskope_ZScalerImporter-02.pl

@@ -0,0 +1,192 @@
+#!/usr/bin/perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use File::Temp;
+use MIME::Lite;
+
+my $VERBOSE = 1;
+my $DEBUG = 1;
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $FILENAME = $config->{general}{FILENAME};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %headers = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->attach( Type => 'text/csv', Path => $FILENAME, Filename => $FILENAME);
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "SMTP $FROM -> $TO - CSV" if $VERBOSE;
+	unlink $FILENAME;
+}
+
+sub check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'Error', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n",
+		);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "SMTP $FROM -> $TO - ERROR" if $VERBOSE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	### Collect widget IDs
+	my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+	my $request = HTTP::Tiny->new('default_headers' => \%headers);
+	my $response = $request->get($uri);
+	check_return($response->{'status'}, $response->{'content'}, $uri);
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+	if (!$data) { check_return(404, $response->{'content'}, "No Widget Data"); }
+	my %files;
+
+
+	### Collect widget data and write to CSV
+	for my $widget (@{$data}) {
+		my $tmp_file = File::Temp->new(UNLINK => 0, TEMPLATE => 'tempXXXXX', DIR => '/tmp');
+		$files{$tmp_file} = $widget->{'name'};
+		$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+		$response = $request->get($uri);
+		print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if $DEBUG;
+		check_return($response->{'status'}, $response->{'content'}, $uri);
+
+		open my $fh_out, ">", $tmp_file;
+		print $fh_out $response->{'content'};
+		close $fh_out;
+	}
+
+	### Process domains from CSV
+	my @blocklist;
+	open my $fh_email_out, ">", $FILENAME;
+
+	for my $csv_file (keys %files) {
+		my $count = 0;
+		my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+		open my $fh_in, "<", $csv_file;
+		my $header = $csv->getline($fh_in);
+
+		print "\n## Widget Name: $files{$csv_file}\nDomains: " if $VERBOSE;
+		print $fh_email_out "$files{$csv_file}\n";
+
+		while (my $row = $csv->getline($fh_in)) {
+			last if ($count == 30);
+			print "$row->[1]," if $VERBOSE;
+			print $fh_email_out "$row->[1],";
+			push @blocklist, $row->[1];
+			$count++;
+		}
+		print "\n" if $VERBOSE;
+		print $fh_email_out "\n";
+		close $fh_in;
+		unlink $csv_file;
+	}
+	close $fh_email_out;
+	return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+	my @domains = @{$_[0]};
+
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	my $request = HTTP::Tiny->new('default_headers' => \%headers, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	### Push Domains
+	$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+	my $method = defined($id) ? "put" : "post";
+	my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+	$#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
+	$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+	$response = $request->$method($uri, {'content' => $body});
+	check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running..." if $VERBOSE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $VERBOSE;

Netskope_ZScalerImporter-03.pl

@@ -0,0 +1,179 @@
+#!/usr/bin/perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $VERBOSE = 1;
+my $DEBUG = 0;
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV;
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "SMTP $FROM -> $TO - CSV" if $VERBOSE;
+}
+
+sub check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'Error', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n",
+		);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "SMTP $FROM -> $TO - ERROR" if $VERBOSE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	### Collect widget IDs
+	my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+	my $response = $request->get($uri);
+	check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+	if (!$data) { check_return(404, $response->{'content'}, "No Widget Data"); }
+	my %csv_content;
+
+	### Collect widget data and write to CSV
+	for my $widget (@{$data}) {
+		$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+		$response = $request->get($uri);
+		print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if $DEBUG;
+		check_return($response->{'status'}, $response->{'content'}, $uri);
+		$csv_content{$widget->{'name'}} = $response->{'content'};
+	}
+
+	### Process domains from CSV
+	my @blocklist;
+	for my $widget_name (keys %csv_content) {
+		my $count = 0;
+		my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+		open my $fh_in, "<", \$csv_content{$widget_name};
+		my $header = $csv->getline($fh_in);
+
+		print "\n## Widget Name: $widget_name\n## Domains: " if $VERBOSE;
+		$EMAIL_CSV .= "$widget_name\n";
+		while (my $row = $csv->getline($fh_in)) {
+			last if ($count == 30);
+			print "$row->[1]," if $VERBOSE;
+			$EMAIL_CSV .= "$row->[1],";
+			push @blocklist, $row->[1];
+			$count++;
+		}
+		print "\n" if $VERBOSE;
+		$EMAIL_CSV .= "\n";
+	}
+	return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+	my @domains = @{$_[0]};
+
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	### Push Domains
+	$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+	my $method = defined($id) ? "put" : "post";
+	my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+	$#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
+	$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+	$response = $request->$method($uri, {'content' => $body});
+	check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running..." if $VERBOSE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $VERBOSE;

Netskope_ZScalerImporter-04.pl

@@ -0,0 +1,183 @@
+#!/usr/bin/perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "VERBOSE";
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status =~ /^2/ && $LOGMODE) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n";
+		print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+	}	
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+		);
+		$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	### Collect widget IDs
+	my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+	my $response = $request->get($uri);
+	check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+	if (!$data) { check_return(404, $response->{'content'}, "No Widget Data"); }
+	my %csv_content;
+
+	### Collect widget data and write to CSV
+	for my $widget (@{$data}) {
+		$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+		$response = $request->get($uri);
+		print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+		check_return($response->{'status'}, $response->{'content'}, $uri);
+		$csv_content{$widget->{'name'}} = $response->{'content'};
+	}
+
+	### Process domains from CSV
+	my @blocklist;
+	for my $widget_name (keys %csv_content) {
+		my $count = 0;
+		my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+		open my $fh_in, "<", \$csv_content{$widget_name};
+		my $header = $csv->getline($fh_in);
+
+		print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+		$EMAIL_CSV .= "$widget_name\n";
+		while (my $row = $csv->getline($fh_in)) {
+			last if ($count == 30);
+			print "$row->[1]," if $LOGMODE;
+			$EMAIL_CSV .= "$row->[1],";
+			push @blocklist, $row->[1];
+			$count++;
+		}
+		print "\n\n" if $LOGMODE;
+		$EMAIL_CSV .= "\n";
+	}
+	return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+	my @domains = @{$_[0]};
+
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	### Push Domains
+	$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+	my $method = defined($id) ? "put" : "post";
+	my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+	$#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
+	$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+	$response = $request->$method($uri, {'content' => $body});
+	check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;

Netskope_ZScalerImporter-05.pl

@@ -0,0 +1,189 @@
+#!/usr/bin/perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status =~ /^2/ && $LOGMODE) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n";
+		print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+	}	
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+		);
+		$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	### Collect widget IDs
+	my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+	my $response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+	if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+	my %csv_content;
+
+	### Collect widget data and write to CSV
+	for my $widget (@{$data}) {
+		$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+		$response = $request->get($uri);
+		print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+		_check_return($response->{'status'}, $response->{'content'}, $uri);
+		$csv_content{$widget->{'name'}} = $response->{'content'};
+	}
+
+	### Process domains from CSV
+	my @blocklist;
+	for my $widget_name (keys %csv_content) {
+		my $count = 0;
+		my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+		open my $fh_in, "<", \$csv_content{$widget_name};
+		my $header = $csv->getline($fh_in);
+
+		print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+		$EMAIL_CSV .= "$widget_name\n";
+		DOMAIN:
+		while (my $row = $csv->getline($fh_in)) {
+			last DOMAIN if ($count == $MAX_DOMAIN);
+			if ($row->[4] < $USER_COUNT) {
+				print "$row->[1]," if ($LOGMODE ne "DEBUG");
+				print "$row->[0] - $row->[1] - $row->[2], $row->[3], $row->[4]\n" if ($LOGMODE eq "DEBUG");
+				$EMAIL_CSV .= "$row->[1],";
+				push @blocklist, $row->[1];
+				$count++;
+			}
+		}
+		print "\n\n" if $LOGMODE;
+		$EMAIL_CSV .= "\n";
+	}
+	return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+	my @domains = @{$_[0]};
+
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	### Push Domains
+	$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+	my $method = defined($id) ? "put" : "post";
+	my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+	$#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
+	$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+	$response = $request->$method($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;

Netskope_ZScalerImporter-06.pl

@@ -0,0 +1,190 @@
+#!/usr/bin/perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "VERBOSE";
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status =~ /^2/ && $LOGMODE) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n";
+		print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+	}	
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+		);
+		$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	### Collect widget IDs
+	my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+	my $response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+	if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+	my %csv_content;
+
+	### Collect widget data and write to CSV
+	for my $widget (@{$data}) {
+		$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+		$response = $request->get($uri);
+		print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+		_check_return($response->{'status'}, $response->{'content'}, $uri);
+		$csv_content{$widget->{'name'}} = $response->{'content'};
+	}
+
+	### Process domains from CSV
+	my @blocklist;
+	for my $widget_name (keys %csv_content) {
+		my $count = 0;
+		my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+		open my $fh_in, "<", \$csv_content{$widget_name};
+		$csv->column_names($csv->getline($fh_in));
+		# "Application","Domain","Category","CCI","Users"
+
+		print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+		$EMAIL_CSV .= "$widget_name\n";
+		DOMAIN:
+		while (my $row = $csv->getline_hr($fh_in)) {
+			last DOMAIN if ($count == $MAX_DOMAIN);
+			if ($row->{'Users'} < $USER_COUNT) {
+				print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+				print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+				$EMAIL_CSV .= "$row->{'Domain'},";
+				push @blocklist, $row->{'Domain'};
+				$count++;
+			}
+		}
+		print "\n\n" if $LOGMODE;
+		$EMAIL_CSV .= "\n";
+	}
+	return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+	my @domains = @{$_[0]};
+
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	### Push Domains
+	$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+	my $method = defined($id) ? "put" : "post";
+	my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+	$#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
+	$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+	$response = $request->$method($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;

Netskope_ZScalerImporter-07.pl

@@ -0,0 +1,190 @@
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status =~ /^2/ && $LOGMODE) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n";
+		print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+	}	
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+		);
+		$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	### Collect widget IDs
+	my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+	my $response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+	if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+	my %csv_content;
+
+	### Collect widget data and write to CSV
+	for my $widget (@{$data}) {
+		$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+		$response = $request->get($uri);
+		print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+		_check_return($response->{'status'}, $response->{'content'}, $uri);
+		$csv_content{$widget->{'name'}} = $response->{'content'};
+	}
+
+	### Process domains from CSV
+	my @blocklist;
+	for my $widget_name (keys %csv_content) {
+		my $count = 0;
+		my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+		open my $fh_in, "<", \$csv_content{$widget_name};
+		$csv->column_names($csv->getline($fh_in));
+		# "Application","Domain","Category","CCI","Users"
+
+		print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+		$EMAIL_CSV .= "$widget_name\n";
+		DOMAIN:
+		while (my $row = $csv->getline_hr($fh_in)) {
+			last DOMAIN if ($count == $MAX_DOMAIN);
+			if ($row->{'Users'} < $USER_COUNT) {
+				print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+				print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+				$EMAIL_CSV .= "$row->{'Domain'},";
+				push @blocklist, $row->{'Domain'};
+				$count++;
+			}
+		}
+		print "\n\n" if $LOGMODE;
+		$EMAIL_CSV .= "\n";
+	}
+	return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+	my @domains = @{$_[0]};
+
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	### Push Domains
+	$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+	my $method = defined($id) ? "put" : "post";
+	my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+	splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+	$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+	$response = $request->$method($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;

Netskope_ZScalerImporter-08.pl

@@ -0,0 +1,199 @@
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status =~ /^2/ && $LOGMODE) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n";
+		print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+	}	
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+		);
+		$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	### Collect widget IDs
+	my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+	my $response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+	if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+	my %csv_content;
+
+	### Collect widget data and write to CSV
+	for my $widget (@{$data}) {
+		$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+		$response = $request->get($uri);
+		print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+		_check_return($response->{'status'}, $response->{'content'}, $uri);
+		$csv_content{$widget->{'name'}} = $response->{'content'};
+	}
+
+	### Process domains from CSV
+	my @blocklist;
+	for my $widget_name (keys %csv_content) {
+		my $count = 0;
+		my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+		open my $fh_in, "<", \$csv_content{$widget_name};
+		$csv->column_names($csv->getline($fh_in));
+		# "Application","Domain","Category","CCI","Users"
+
+		print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+		$EMAIL_CSV .= "$widget_name\n";
+		DOMAIN:
+		while (my $row = $csv->getline_hr($fh_in)) {
+			last DOMAIN if ($count == $MAX_DOMAIN);
+			if ($row->{'Users'} < $USER_COUNT) {
+				print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+				print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+				if ($row->{'Domain'} =~ /,/) {
+					push @blocklist, split (/,/, $row->{'Domain'});
+				} else {
+					push @blocklist, $row->{'Domain'};
+				}	
+				$EMAIL_CSV .= "$row->{'Domain'},";
+				$count++;
+			}
+		}
+		print "\n\n" if $LOGMODE;
+		$EMAIL_CSV .= "\n";
+	}
+	return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+	my @domains = @{$_[0]};
+
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	### Push Domains
+	$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+	my $method = defined($id) ? "put" : "post";
+	my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+	splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+	$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+# DEBUG
+	print "$body\n";
+
+	$response = $request->$method($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;

Netskope_ZScalerImporter-09.pl

@@ -0,0 +1,198 @@
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl
+# Version 3.0 - 20200615 - rewrite to Perl
+# Version 3.1 - 20200812 - split domains when comma separated in CSV
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status =~ /^2/ && $LOGMODE) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n";
+		print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+	}	
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+		);
+		$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	### Collect widget IDs
+	my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+	my $response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+	if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+	my %csv_content;
+
+	### Collect widget data and write to CSV
+	for my $widget (@{$data}) {
+		$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+		$response = $request->get($uri);
+		print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+		_check_return($response->{'status'}, $response->{'content'}, $uri);
+		$csv_content{$widget->{'name'}} = $response->{'content'};
+	}
+
+	### Process domains from CSV
+	my @blocklist;
+	for my $widget_name (keys %csv_content) {
+		my $count = 0;
+		my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+		open my $fh_in, "<", \$csv_content{$widget_name};
+		$csv->column_names($csv->getline($fh_in));
+		# "Application","Domain","Category","CCI","Users"
+
+		print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+		$EMAIL_CSV .= "$widget_name\n";
+		DOMAIN:
+		while (my $row = $csv->getline_hr($fh_in)) {
+			last DOMAIN if ($count == $MAX_DOMAIN);
+			if ($row->{'Users'} < $USER_COUNT) {
+				print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+				print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+				if ($row->{'Domain'} =~ /,/) {
+					push @blocklist, split (/,/, $row->{'Domain'});
+				} else {
+					push @blocklist, $row->{'Domain'};
+				}	
+				$EMAIL_CSV .= "$row->{'Domain'},";
+				$count++;
+			}
+		}
+		print "\n\n" if $LOGMODE;
+		$EMAIL_CSV .= "\n";
+	}
+	return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+	my @domains = @{$_[0]};
+
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	### Push Domains
+	$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+	my $method = defined($id) ? "put" : "post";
+	my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+	splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+	$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+	print "$body\n" if ($LOGMODE eq "DEBUG");
+
+	$response = $request->$method($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;

Netskope_ZScalerImporter-10.pl

@@ -0,0 +1,207 @@
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl
+# Version 3.0 - 20200615 - rewrite to Perl
+# Version 3.1 - 20200812 - split domains when comma separated in CSV
+# Version 3.2 - 20200826 - added all fields to CSV export
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status =~ /^2/ && $LOGMODE) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n";
+		print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+	}	
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+		);
+		$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	### Collect widget IDs
+	my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+	my $response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+	if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+	my %csv_content;
+
+	### Collect widget data and write to CSV
+	for my $widget (@{$data}) {
+		$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+		$response = $request->get($uri);
+		print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+		_check_return($response->{'status'}, $response->{'content'}, $uri);
+		$csv_content{$widget->{'name'}} = $response->{'content'};
+	}
+
+	### Process domains from CSV
+	my @blocklist;
+	for my $widget_name (keys %csv_content) {
+		my $count = 0;
+		my $domain;
+		my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+		open my $fh_in, "<", \$csv_content{$widget_name};
+		$csv->column_names($csv->getline($fh_in));
+		# "Application","Domain","Category","CCI","Users"
+
+		print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+		$EMAIL_CSV .= "$widget_name\n";
+		$EMAIL_CSV .= "Application,Domain,Category,CCI,Users\n";
+
+		DOMAIN:
+		while (my $row = $csv->getline_hr($fh_in)) {
+			last DOMAIN if ($count == $MAX_DOMAIN);
+			if ($row->{'Users'} < $USER_COUNT) {
+				print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+				print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+				if ($row->{'Domain'} =~ /,/) {
+					push @blocklist, split (/,/, $row->{'Domain'});
+					$domain = $row->{'Domain'} =~ s/,/ /gr;
+				} else {
+					push @blocklist, $row->{'Domain'};
+					$domain = $row->{'Domain'};
+					
+				}	
+				#$EMAIL_CSV .= "$row->{'Domain'},";
+				#$EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Users'}\n";
+				$EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Users'}\n";
+				$count++;
+			}
+		}
+		print "\n\n" if $LOGMODE;
+		$EMAIL_CSV .= "\n";
+	}
+	return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+	my @domains = @{$_[0]};
+
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	### Push Domains
+	$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+	my $method = defined($id) ? "put" : "post";
+	my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+	splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+	$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+	print "$body\n" if ($LOGMODE eq "DEBUG");
+
+	$response = $request->$method($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+#zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;

Netskope_ZScalerImporter-11.pl

@@ -0,0 +1,205 @@
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl
+# Version 3.0 - 20200615 - rewrite to Perl
+# Version 3.1 - 20200812 - split domains when comma separated in CSV
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status =~ /^2/ && $LOGMODE) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n";
+		print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+	}	
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+		);
+		$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	### Collect widget IDs
+	my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+	my $response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+	if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+	my %csv_content;
+
+	### Collect widget data and write to CSV
+	for my $widget (@{$data}) {
+		$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+		$response = $request->get($uri);
+		print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+		_check_return($response->{'status'}, $response->{'content'}, $uri);
+		$csv_content{$widget->{'name'}} = $response->{'content'};
+	}
+
+	### Process domains from CSV
+	my @blocklist;
+	for my $widget_name (keys %csv_content) {
+		my $count = 0;
+		my $domain;
+		my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+		open my $fh_in, "<", \$csv_content{$widget_name};
+		my @headers = $csv->column_names($csv->getline($fh_in));
+		print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE;
+
+		print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+		$EMAIL_CSV .= "$widget_name\n";
+		$EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n";
+
+		DOMAIN:
+		while (my $row = $csv->getline_hr($fh_in)) {
+			last DOMAIN if ($count == $MAX_DOMAIN);
+			next DOMAIN if ($row->{'Blocked Events'} > 0);
+			if ($row->{'Users'} < $USER_COUNT) {
+				print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+				print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+				if ($row->{'Domain'} =~ /,/) {
+					push @blocklist, split (/,/, $row->{'Domain'});
+					$domain = $row->{'Domain'} =~ s/,/ /gr;
+				} else {
+					push @blocklist, $row->{'Domain'};
+					$domain = $row->{'Domain'};
+					
+				}	
+				$EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+				$count++;
+			}
+		}
+		print "\n\n" if $LOGMODE;
+		$EMAIL_CSV .= "\n";
+	}
+	return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler {
+	my @domains = @{$_[0]};
+
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	### Push Domains
+	$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+	my $method = defined($id) ? "put" : "post";
+	my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+	splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+	$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+	print "$body\n" if ($LOGMODE eq "DEBUG");
+
+	$response = $request->$method($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+#zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;

Netskope_ZScalerImporter-12.pl

@@ -0,0 +1,276 @@
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl
+# Version 3.0 - 20200615 - rewrite to Perl
+# Version 3.1 - 20200812 - split domains when comma separated in CSV
+# Version 3.2 - 20200909 - de-duplication of Zscaler URL category
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status =~ /^2/ && $LOGMODE) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n";
+		print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+	}	
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+		);
+		$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	my @existing_domains = @{$_[0]};
+
+	### Collect widget IDs
+	my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+	my $response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+	if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+	my %csv_content;
+
+	### Collect widget data and write to CSV
+	for my $widget (@{$data}) {
+		$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+		$response = $request->get($uri);
+		print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+		_check_return($response->{'status'}, $response->{'content'}, $uri);
+		$csv_content{$widget->{'name'}} = $response->{'content'};
+	}
+
+	### Process domains from CSV
+	my @blocklist;
+	for my $widget_name (keys %csv_content) {
+		my $count = 0;
+		my $domain;
+		my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+		open my $fh_in, "<", \$csv_content{$widget_name};
+		my @headers = $csv->column_names($csv->getline($fh_in));
+		print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE;
+
+		print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+		$EMAIL_CSV .= "$widget_name\n";
+		$EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n";
+
+		DOMAIN: while (my $row = $csv->getline_hr($fh_in)) {
+			last DOMAIN if ($count == $MAX_DOMAIN);
+			#next DOMAIN if ($row->{'Blocked Events'} > 0);
+			if ($row->{'Users'} < $USER_COUNT) {
+				print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+				print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+				if ($row->{'Domain'} =~ /,/) {
+					PARSE:
+					for my $item (split (/,/, $row->{'Domain'})) {
+						next PARSE if (grep(/$item/, @existing_domains));
+						push @blocklist, $item;
+						$domain .= $item . " ";
+					}
+					if ($domain) {
+						$EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+						$count++;
+					}
+				} else {
+					next DOMAIN if (!grep(/$row->{'Domain'}/, @existing_domains));
+					push @blocklist, $row->{'Domain'};
+					$EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+					$count++;
+				}
+			}
+		}
+		print "\n\n" if $LOGMODE;
+		$EMAIL_CSV .= "\n";
+		print "COUNT: $count\n";
+	}
+	return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler_get {
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	$uri = "$ZS_BASE_URI/urlCategories/$id";
+	my $method = "get";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	$json = JSON::PP->new->utf8->decode($response->{'content'});
+        my $data = $json->{'urls'};
+	my @convert = ();
+	for my $item (@{$data}) {
+		push @convert, $item;
+	}
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	return @convert;
+}
+
+
+
+sub zscaler_push {
+	my @domains = @{$_[0]};
+
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	### Push Domains
+	$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories";
+	my $method = defined($id) ? "put" : "post";
+	my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+	splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+	$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+	print "$body\n" if ($LOGMODE eq "DEBUG");
+
+	$response = $request->$method($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @existing_domains = zscaler_get();
+my @domains = netskope(\@existing_domains);
+print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE;
+zscaler_push(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;

Netskope_ZScalerImporter-13.pl

@@ -0,0 +1,277 @@
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl
+# Version 3.0 - 20200615 - rewrite to Perl
+# Version 3.1 - 20200812 - split domains when comma separated in CSV
+# Version 3.2 - 20200909 - de-duplication of Zscaler URL category
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status =~ /^2/ && $LOGMODE) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n";
+		print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+	}	
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+		);
+		$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	my @existing_domains = @{$_[0]};
+
+	### Collect widget IDs
+	my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+	my $response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+	if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+	my %csv_content;
+
+	### Collect widget data and write to CSV
+	for my $widget (@{$data}) {
+		$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+		$response = $request->get($uri);
+		print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+		_check_return($response->{'status'}, $response->{'content'}, $uri);
+		$csv_content{$widget->{'name'}} = $response->{'content'};
+	}
+
+	### Process domains from CSV
+	my @blocklist;
+	for my $widget_name (keys %csv_content) {
+		my $count = 0;
+		my $domain;
+		my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+		open my $fh_in, "<", \$csv_content{$widget_name};
+		my @headers = $csv->column_names($csv->getline($fh_in));
+		print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE;
+
+		print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+		$EMAIL_CSV .= "$widget_name\n";
+		$EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n";
+
+		DOMAIN: while (my $row = $csv->getline_hr($fh_in)) {
+			last DOMAIN if ($count == $MAX_DOMAIN);
+			#next DOMAIN if ($row->{'Blocked Events'} > 0);
+			if ($row->{'Users'} < $USER_COUNT) {
+				print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+				print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+				next DOMAIN if ($row->{'Domain'} =~ "n/a");
+				if ($row->{'Domain'} =~ /,/) {
+					PARSE:
+					for my $item (split (/,/, $row->{'Domain'})) {
+						next PARSE if (grep(/$item/, @existing_domains));
+						push @blocklist, $item;
+						$domain .= $item . " ";
+					}
+					if ($domain) {
+						$EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+						$count++;
+					}
+				} else {
+					next DOMAIN if (!grep(/$row->{'Domain'}/, @existing_domains));
+					push @blocklist, $row->{'Domain'};
+					$EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+					$count++;
+				}
+			}
+		}
+		print "\n\n" if $LOGMODE;
+		$EMAIL_CSV .= "\n";
+		print "COUNT: $count\n";
+	}
+	return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler_get {
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	$uri = "$ZS_BASE_URI/urlCategories/$id";
+	my $method = "get";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	$json = JSON::PP->new->utf8->decode($response->{'content'});
+        my $data = $json->{'urls'};
+	my @convert = ();
+	for my $item (@{$data}) {
+		push @convert, $item;
+	}
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	return @convert;
+}
+
+
+
+sub zscaler_push {
+	my @domains = @{$_[0]};
+
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	### Push Domains
+	$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories";
+	my $method = defined($id) ? "put" : "post";
+	my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+	splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+	$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+	print "$body\n" if ($LOGMODE eq "DEBUG");
+
+	$response = $request->$method($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @existing_domains = zscaler_get();
+my @domains = netskope(\@existing_domains);
+print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE;
+zscaler_push(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;

Netskope_ZScalerImporter-14.pl

@@ -0,0 +1,280 @@
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl
+# Version 3.0 - 20200615 - rewrite to Perl
+# Version 3.1 - 20200812 - split domains when comma separated in CSV
+# Version 3.2 - 20200909 - de-duplication of Zscaler URL category
+# Version 3.3 - 20210121 - filter our entries when domain "n/a"
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+#my $LOGMODE = "";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status =~ /^2/ && $LOGMODE) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n";
+		print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+	}	
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+		);
+		$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	my @existing_domains = @{$_[0]};
+
+	### Collect widget IDs
+	my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+	my $response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+	if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+	my %csv_content;
+
+	### Collect widget data and write to CSV
+	for my $widget (@{$data}) {
+		$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+		$response = $request->get($uri);
+		print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+		_check_return($response->{'status'}, $response->{'content'}, $uri);
+		$csv_content{$widget->{'name'}} = $response->{'content'};
+	}
+
+	### Process domains from CSV
+	my @blocklist;
+	for my $widget_name (keys %csv_content) {
+		my $count = 0;
+		my $domain;
+		my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+		open my $fh_in, "<", \$csv_content{$widget_name};
+		my @headers = $csv->column_names($csv->getline($fh_in));
+		print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE;
+
+		print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+		$EMAIL_CSV .= "$widget_name\n";
+		$EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n";
+
+		DOMAIN: while (my $row = $csv->getline_hr($fh_in)) {
+			last DOMAIN if ($count == $MAX_DOMAIN);
+			next DOMAIN if ($row->{'Blocked Events'} > 0);
+			if ($row->{'Users'} < $USER_COUNT) {
+				print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+				print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+				next DOMAIN if ($row->{'Domain'} =~ "n/a");
+				if ($row->{'Domain'} =~ /,/) {
+					PARSE:
+					for my $item (split (/,/, $row->{'Domain'})) {
+						next PARSE if (grep(/$item/, @existing_domains));
+						push @blocklist, $item;
+						$domain .= $item . " ";
+					}
+					if ($domain) {
+						$EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+						$count++;
+					}
+				} else {
+					next DOMAIN if (grep(/$row->{'Domain'}/, @existing_domains));
+					push @blocklist, $row->{'Domain'};
+					$EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+					$count++;
+				}
+			}
+		}
+		print "\n\n" if $LOGMODE;
+		$EMAIL_CSV .= "\n";
+		print "COUNT: $count\n";
+	}
+	return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler_get {
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	$uri = "$ZS_BASE_URI/urlCategories/$id";
+	my $method = "get";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	$json = JSON::PP->new->utf8->decode($response->{'content'});
+        my $data = $json->{'urls'};
+	my @convert = ();
+	for my $item (@{$data}) {
+		push @convert, $item;
+	}
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	return @convert;
+}
+
+
+
+sub zscaler_push {
+	my @domains = @{$_[0]};
+
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	### Push Domains
+	$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories?action=ADD_TO_LIST";
+	my $method = defined($id) ? "put" : "post";
+	my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+	splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+	$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+	print "$body\n" if ($LOGMODE eq "DEBUG");
+
+	$response = $request->$method($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @existing_domains = zscaler_get();
+my @domains = netskope(\@existing_domains);
+print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE;
+zscaler_push(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;

Netskope_ZScalerImporter-wip.pl

@@ -0,0 +1,276 @@
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl
+# Version 3.0 - 20200615 - rewrite to Perl
+# Version 3.1 - 20200812 - split domains when comma separated in CSV
+# Version 3.2 - 20200909 - de-duplication of Zscaler URL category
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.016;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "DEBUG";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status =~ /^2/ && $LOGMODE) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n";
+		print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+	}	
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+		);
+		$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	my @existing_domains = @{$_[0]};
+
+	### Collect widget IDs
+	my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+	my $response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+	if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+	my %csv_content;
+
+	### Collect widget data and write to CSV
+	for my $widget (@{$data}) {
+		$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+		$response = $request->get($uri);
+		print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+		_check_return($response->{'status'}, $response->{'content'}, $uri);
+		$csv_content{$widget->{'name'}} = $response->{'content'};
+	}
+
+	### Process domains from CSV
+	my @blocklist;
+	for my $widget_name (keys %csv_content) {
+		my $count = 0;
+		my $domain;
+		my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+		open my $fh_in, "<", \$csv_content{$widget_name};
+		my @headers = $csv->column_names($csv->getline($fh_in));
+		print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE;
+
+		print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+		$EMAIL_CSV .= "$widget_name\n";
+		$EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n";
+
+		DOMAIN: while (my $row = $csv->getline_hr($fh_in)) {
+			last DOMAIN if ($count == $MAX_DOMAIN);
+			#next DOMAIN if ($row->{'Blocked Events'} > 0);
+			if ($row->{'Users'} < $USER_COUNT) {
+				print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+				print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+				if ($row->{'Domain'} =~ /,/) {
+					PARSE:
+					for my $item (split (/,/, $row->{'Domain'})) {
+						next PARSE if (grep(/$item/, @existing_domains));
+						push @blocklist, $item;
+						$domain .= $item . " ";
+					}
+					if ($domain) {
+						$EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+						$count++;
+					}
+				} else {
+					next DOMAIN if (grep(/$row->{'Domain'}/, @existing_domains));
+					push @blocklist, $row->{'Domain'};
+					$EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
+					$count++;
+				}
+			}
+		}
+		print "\n\n" if $LOGMODE;
+		$EMAIL_CSV .= "\n";
+		print "COUNT: $count\n";
+	}
+	return @blocklist;
+}
+
+### Zscaler ###
+
+sub zscaler_get {
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	$uri = "$ZS_BASE_URI/urlCategories/$id";
+	my $method = "get";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	$json = JSON::PP->new->utf8->decode($response->{'content'});
+        my $data = $json->{'urls'};
+	my @convert = ();
+	for my $item (@{$data}) {
+		push @convert, $item;
+	}
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	return @convert;
+}
+
+
+
+sub zscaler_push {
+	my @domains = @{$_[0]};
+
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	### Push Domains
+	$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories";
+	my $method = defined($id) ? "put" : "post";
+	my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+	splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+	$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+	print "$body\n" if ($LOGMODE eq "DEBUG");
+
+	$response = $request->$method($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @existing_domains = zscaler_get();
+my @domains = netskope(\@existing_domains);
+print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE;
+zscaler_push(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;

httpstat.py

@@ -0,0 +1,351 @@
+#!/usr/bin/env python
+# coding: utf-8
+# References:
+# man curl
+# https://curl.haxx.se/libcurl/c/curl_easy_getinfo.html
+# https://curl.haxx.se/libcurl/c/easy_getinfo_options.html
+# http://blog.kenweiner.com/2014/11/http-request-timings-with-curl.html
+
+from __future__ import print_function
+
+import os
+import json
+import sys
+import logging
+import tempfile
+import subprocess
+
+
+__version__ = '1.2.1'
+
+
+PY3 = sys.version_info >= (3,)
+
+if PY3:
+    xrange = range
+
+
+# Env class is copied from https://github.com/reorx/getenv/blob/master/getenv.py
+class Env(object):
+    prefix = 'HTTPSTAT'
+    _instances = []
+
+    def __init__(self, key):
+        self.key = key.format(prefix=self.prefix)
+        Env._instances.append(self)
+
+    def get(self, default=None):
+        return os.environ.get(self.key, default)
+
+
+ENV_SHOW_BODY = Env('{prefix}_SHOW_BODY')
+ENV_SHOW_IP = Env('{prefix}_SHOW_IP')
+ENV_SHOW_SPEED = Env('{prefix}_SHOW_SPEED')
+ENV_SAVE_BODY = Env('{prefix}_SAVE_BODY')
+ENV_CURL_BIN = Env('{prefix}_CURL_BIN')
+ENV_DEBUG = Env('{prefix}_DEBUG')
+
+
+curl_format = """{
+"time_namelookup": %{time_namelookup},
+"time_connect": %{time_connect},
+"time_appconnect": %{time_appconnect},
+"time_pretransfer": %{time_pretransfer},
+"time_redirect": %{time_redirect},
+"time_starttransfer": %{time_starttransfer},
+"time_total": %{time_total},
+"speed_download": %{speed_download},
+"speed_upload": %{speed_upload},
+"remote_ip": "%{remote_ip}",
+"remote_port": "%{remote_port}",
+"local_ip": "%{local_ip}",
+"local_port": "%{local_port}"
+}"""
+
+https_template = """
+  DNS Lookup   TCP Connection   TLS Handshake   Server Processing   Content Transfer
+[   {a0000}  |     {a0001}    |    {a0002}    |      {a0003}      |      {a0004}     ]
+             |                |               |                   |                  |
+    namelookup:{b0000}        |               |                   |                  |
+                        connect:{b0001}       |                   |                  |
+                                    pretransfer:{b0002}           |                  |
+                                                      starttransfer:{b0003}          |
+                                                                                 total:{b0004}
+"""[1:]
+
+http_template = """
+  DNS Lookup   TCP Connection   Server Processing   Content Transfer
+[   {a0000}  |     {a0001}    |      {a0003}      |      {a0004}     ]
+             |                |                   |                  |
+    namelookup:{b0000}        |                   |                  |
+                        connect:{b0001}           |                  |
+                                      starttransfer:{b0003}          |
+                                                                 total:{b0004}
+"""[1:]
+
+
+# Color code is copied from https://github.com/reorx/python-terminal-color/blob/master/color_simple.py
+ISATTY = sys.stdout.isatty()
+
+
+def make_color(code):
+    def color_func(s):
+        if not ISATTY:
+            return s
+        tpl = '\x1b[{}m{}\x1b[0m'
+        return tpl.format(code, s)
+    return color_func
+
+
+red = make_color(31)
+green = make_color(32)
+yellow = make_color(33)
+blue = make_color(34)
+magenta = make_color(35)
+cyan = make_color(36)
+
+bold = make_color(1)
+underline = make_color(4)
+
+grayscale = {(i - 232): make_color('38;5;' + str(i)) for i in xrange(232, 256)}
+
+
+def quit(s, code=0):
+    if s is not None:
+        print(s)
+    sys.exit(code)
+
+
+def print_help():
+    help = """
+Usage: httpstat URL [CURL_OPTIONS]
+       httpstat -h | --help
+       httpstat --version
+
+Arguments:
+  URL     url to request, could be with or without `http(s)://` prefix
+
+Options:
+  CURL_OPTIONS  any curl supported options, except for -w -D -o -S -s,
+                which are already used internally.
+  -h --help     show this screen.
+  --version     show version.
+
+Environments:
+  HTTPSTAT_SHOW_BODY    Set to `true` to show response body in the output,
+                        note that body length is limited to 1023 bytes, will be
+                        truncated if exceeds. Default is `false`.
+  HTTPSTAT_SHOW_IP      By default httpstat shows remote and local IP/port address.
+                        Set to `false` to disable this feature. Default is `true`.
+  HTTPSTAT_SHOW_SPEED   Set to `true` to show download and upload speed.
+                        Default is `false`.
+  HTTPSTAT_SAVE_BODY    By default httpstat stores body in a tmp file,
+                        set to `false` to disable this feature. Default is `true`
+  HTTPSTAT_CURL_BIN     Indicate the curl bin path to use. Default is `curl`
+                        from current shell $PATH.
+  HTTPSTAT_DEBUG        Set to `true` to see debugging logs. Default is `false`
+"""[1:-1]
+    print(help)
+
+
+def main():
+    args = sys.argv[1:]
+    if not args:
+        print_help()
+        quit(None, 0)
+
+    # get envs
+    show_body = 'true' in ENV_SHOW_BODY.get('false').lower()
+    show_ip = 'true' in ENV_SHOW_IP.get('true').lower()
+    show_speed = 'true'in ENV_SHOW_SPEED.get('false').lower()
+    save_body = 'true' in ENV_SAVE_BODY.get('true').lower()
+    curl_bin = ENV_CURL_BIN.get('curl')
+    is_debug = 'true' in ENV_DEBUG.get('false').lower()
+
+    # configure logging
+    if is_debug:
+        log_level = logging.DEBUG
+    else:
+        log_level = logging.INFO
+    logging.basicConfig(level=log_level)
+    lg = logging.getLogger('httpstat')
+
+    # log envs
+    lg.debug('Envs:\n%s', '\n'.join('  {}={}'.format(i.key, i.get('')) for i in Env._instances))
+    lg.debug('Flags: %s', dict(
+        show_body=show_body,
+        show_ip=show_ip,
+        show_speed=show_speed,
+        save_body=save_body,
+        curl_bin=curl_bin,
+        is_debug=is_debug,
+    ))
+
+    # get url
+    url = args[0]
+    if url in ['-h', '--help']:
+        print_help()
+        quit(None, 0)
+    elif url == '--version':
+        print('httpstat {}'.format(__version__))
+        quit(None, 0)
+
+    curl_args = args[1:]
+
+    # check curl args
+    exclude_options = [
+        '-w', '--write-out',
+        '-D', '--dump-header',
+        '-o', '--output',
+        '-s', '--silent',
+    ]
+    for i in exclude_options:
+        if i in curl_args:
+            quit(yellow('Error: {} is not allowed in extra curl args'.format(i)), 1)
+
+    # tempfile for output
+    bodyf = tempfile.NamedTemporaryFile(delete=False)
+    bodyf.close()
+
+    headerf = tempfile.NamedTemporaryFile(delete=False)
+    headerf.close()
+
+    # run cmd
+    cmd_env = os.environ.copy()
+    cmd_env.update(
+        LC_ALL='C',
+    )
+    cmd_core = [curl_bin, '-w', curl_format, '-D', headerf.name, '-o', bodyf.name, '-s', '-S']
+    cmd = cmd_core + curl_args + [url]
+    lg.debug('cmd: %s', cmd)
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=cmd_env)
+    out, err = p.communicate()
+    if PY3:
+        out, err = out.decode(), err.decode()
+    lg.debug('out: %s', out)
+
+    # print stderr
+    if p.returncode == 0:
+        if err:
+            print(grayscale[16](err))
+    else:
+        _cmd = list(cmd)
+        _cmd[2] = '<output-format>'
+        _cmd[4] = '<tempfile>'
+        _cmd[6] = '<tempfile>'
+        print('> {}'.format(' '.join(_cmd)))
+        quit(yellow('curl error: {}'.format(err)), p.returncode)
+
+    # parse output
+    try:
+        d = json.loads(out)
+    except ValueError as e:
+        print(yellow('Could not decode json: {}'.format(e)))
+        print('curl result:', p.returncode, grayscale[16](out), grayscale[16](err))
+        quit(None, 1)
+    for k in d:
+        if k.startswith('time_'):
+            d[k] = int(d[k] * 1000)
+
+    # calculate ranges
+    d.update(
+        range_dns=d['time_namelookup'],
+        range_connection=d['time_connect'] - d['time_namelookup'],
+        range_ssl=d['time_pretransfer'] - d['time_connect'],
+        range_server=d['time_starttransfer'] - d['time_pretransfer'],
+        range_transfer=d['time_total'] - d['time_starttransfer'],
+    )
+
+    # ip
+    if show_ip:
+        s = 'Connected to {}:{} from {}:{}'.format(
+            cyan(d['remote_ip']), cyan(d['remote_port']),
+            d['local_ip'], d['local_port'],
+        )
+        print(s)
+        print()
+
+    # print header & body summary
+    with open(headerf.name, 'r') as f:
+        headers = f.read().strip()
+    # remove header file
+    lg.debug('rm header file %s', headerf.name)
+    os.remove(headerf.name)
+
+    for loop, line in enumerate(headers.split('\n')):
+        if loop == 0:
+            p1, p2 = tuple(line.split('/'))
+            print(green(p1) + grayscale[14]('/') + cyan(p2))
+        else:
+            pos = line.find(':')
+            print(grayscale[14](line[:pos + 1]) + cyan(line[pos + 1:]))
+
+    print()
+
+    # body
+    if show_body:
+        body_limit = 1024
+        with open(bodyf.name, 'r') as f:
+            body = f.read().strip()
+        body_len = len(body)
+
+        if body_len > body_limit:
+            print(body[:body_limit] + cyan('...'))
+            print()
+            s = '{} is truncated ({} out of {})'.format(green('Body'), body_limit, body_len)
+            if save_body:
+                s += ', stored in: {}'.format(bodyf.name)
+            print(s)
+        else:
+            print(body)
+    else:
+        if save_body:
+            print('{} stored in: {}'.format(green('Body'), bodyf.name))
+
+    # remove body file
+    if not save_body:
+        lg.debug('rm body file %s', bodyf.name)
+        os.remove(bodyf.name)
+
+    # print stat
+    if url.startswith('https://'):
+        template = https_template
+    else:
+        template = http_template
+
+    # colorize template first line
+    tpl_parts = template.split('\n')
+    tpl_parts[0] = grayscale[16](tpl_parts[0])
+    template = '\n'.join(tpl_parts)
+
+    def fmta(s):
+        return cyan('{:^7}'.format(str(s) + 'ms'))
+
+    def fmtb(s):
+        return cyan('{:<7}'.format(str(s) + 'ms'))
+
+    stat = template.format(
+        # a
+        a0000=fmta(d['range_dns']),
+        a0001=fmta(d['range_connection']),
+        a0002=fmta(d['range_ssl']),
+        a0003=fmta(d['range_server']),
+        a0004=fmta(d['range_transfer']),
+        # b
+        b0000=fmtb(d['time_namelookup']),
+        b0001=fmtb(d['time_connect']),
+        b0002=fmtb(d['time_pretransfer']),
+        b0003=fmtb(d['time_starttransfer']),
+        b0004=fmtb(d['time_total']),
+    )
+    print()
+    print(stat)
+
+    # speed, originally bytes per second
+    if show_speed:
+        print('speed_download: {:.1f} KiB/s, speed_upload: {:.1f} KiB/s'.format(
+            d['speed_download'] / 1024, d['speed_upload'] / 1024))
+
+
+if __name__ == '__main__':
+    main()

jsondump.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+import json
+import urllib.request
+import argparse
+import collections
+from operator import itemgetter
+
+parser = argparse.ArgumentParser(description="API Call to collect data")
+parser.add_argument("tenant", type=str, help="Tenant Name")
+parser.add_argument("token", type=str, help="Tenat API Token")
+parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod (default: 604800)")
+
+try:
+	args = parser.parse_args()
+	tenant = args.tenant
+	token = args.token
+	timeperiod = args.timeperiod
+
+except argparse.ArgumentError as e:
+	print(str(e))
+
+base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod)
+
+req = urllib.request.Request(base_url)
+with urllib.request.urlopen(req) as response:
+	content = response.read()
+json_content = json.loads(content)
+print(json.dumps(json_content, indent=4, sort_keys=True))

measure.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Version 1.0 - 20191028
+#
+# Measure timing for DNS lookup as well as HTTP page load
+#
+# Requires:
+#   - Python 3.x
+#       
+import argparse
+import socket
+import time
+import ssl
+import urllib.request
+from urllib.parse import urlparse
+
+parser = argparse.ArgumentParser(description="Measure load times", epilog="2019 (c) Netskope")
+parser.add_argument("url", type=str, help="url (eg. https://google.com)")
+
+try:
+	args = parser.parse_args()
+	url = args.url
+
+except argparse.ArgumentError as e:
+	print(str(e))
+
+print (url, "timing:")
+
+urlinfo = urlparse(url)
+request_headers = {'Cache-Control': 'no-cache', 'User-Agent': 'Mozilla/5.0'}
+no_cert_check = ssl.create_default_context()
+no_cert_check.check_hostname=False
+no_cert_check.verify_mode=ssl.CERT_NONE
+
+start = time.time()
+ip = socket.gethostbyname(urlinfo.netloc)
+dns_time = time.time()-start
+print ("DNS Lookup:\t{:.3f} seconds".format(dns_time))
+
+start = time.time()
+req = urllib.request.Request(url, headers=request_headers)
+content = urllib.request.urlopen(req, context=no_cert_check).read()
+load_time = time.time()-start
+print ("Page Load:\t{:.3f} seconds".format(load_time))
+print ("w/o DNS Lookup:\t{:.3f} seconds".format(load_time-dns_time))

ns.pl

@@ -0,0 +1,24 @@
+#!/usr/bin/perl -w
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use HTTP::Tiny;
+use Cpanel::JSON::XS;
+
+my $NTSKP_TENANT = "https://oss.de.goskope.com";
+my $NTSKP_TOKEN = "0cfe04c4237cc33dc7f383af5ddbe2e3";
+my $uri = "$NTSKP_TENANT/api/v1/alerts?token=$NTSKP_TOKEN&timeperiod=86400&groupby=application&query=access_method+eq+Client+and+action+eq+block";
+#my $uri = "$NTSKP_TENANT/api/v1/report?token=$NTSKP_TOKEN&timeperiod=86400&type=connection&groupby=application&query=app-cci-app-tag+eq+'Under_Review'";
+#my $uri = "$NTSKP_TENANT/api/v1/report?token=$NTSKP_TOKEN&timeperiod=86400&type=connection&groupby=application&query=app-cci-app-tag+eq+'Pending_GRC_Review'";
+my $response = HTTP::Tiny->new->get($uri);
+my $json = Cpanel::JSON::XS->new->utf8->decode($response->{'content'});
+my $data = $json->{'data'};
+for my $item (@{$data}) {
+	if (exists($item->{'app'})) {
+		print ".";
+		#print $item->{'app'} . ", ";
+		#say $item->{'sessions'};
+	}
+}
+say "";

ntskp-api-01.pl

@@ -0,0 +1,45 @@
+#!/usr/bin/perl -w
+use strict;
+use warnings;
+use autodie;
+use POSIX qw(strftime);
+use Cpanel::JSON::XS;
+
+my $file;
+{
+	local $/;
+	open my $fh, "<", "amsjson.txt";
+	$file = <$fh>;
+	close $fh;
+}
+
+my $json = Cpanel::JSON::XS->new->utf8->decode($file);
+my $data = $json->{'data'};
+my $domain;
+my $cci;
+
+for (my $i = 0; $i < (@{$data}); $i++) {
+	#print "Timestamp: $data->[$i]->{'timestamp'}\n";
+	#print "Domain: $data->[$i]->{'domain'}\n";
+
+	if (!$data->[$i]->{'domain'}) {
+		my $url = $data->[$i]->{'url'};
+		$url =~ s!^https?://(?:www\.)?!!i;
+		$url =~ s!/.*!!;
+		$url =~ s/[\?\#\:].*//;
+		$domain	= $url;
+	} else {
+		$domain = $data->[$i]->{'domain'};
+	}
+	if ($data->[$i]->{'cci'}) {
+		$cci = $data->[$i]->{'cci'};
+	} else {
+		$cci = 'none';
+	}
+
+	#print "Category: $data->[$i]->{'category'}\n";
+	#print "CCI: $data->[$i]->{'ccl'}\n";
+	#print "User: $data->[$i]->{'user'}\n";
+	my $timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime($data->[$i]->{'timestamp'}));
+	print "$timestamp,$domain,$cci,$data->[$i]->{'category'},$data->[$i]->{'ccl'}\n";
+}

ntskp-api-02.pl

@@ -0,0 +1,69 @@
+#!/usr/bin/perl -w
+use strict;
+use warnings;
+use autodie;
+use POSIX qw(strftime);
+use Config::Tiny;
+use HTTP::Tiny;
+use Cpanel::JSON::XS;
+
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_PERIOD = $config->{netskope}{NTSKP_PERIOD};
+my $NTSKP_SCORE = $config->{netskope}{NTSKP_SCORE};
+my $NTSKP_CATEGORIES = $config->{netskope}{NTSKP_CATEGORIES};
+
+my $uri;
+my $skip = 0;
+my $response;
+my $json;
+my $data;
+my $length;
+my $domain;
+my $cci;
+
+my $file_out = "extracted-" . strftime("%Y%m%d", localtime) . ".txt";
+print "File: $file_out\n";
+print "Tenant: $NTSKP_TENANT\n";
+
+while ($skip < 500000) {
+	$uri = "$NTSKP_TENANT/api/v1/events?token=$NTSKP_TOKEN&type=page&timeperiod=$NTSKP_PERIOD&skip=$skip";
+	$response = HTTP::Tiny->new->get($uri);
+	print "HTTP: $response->{status} $response->{reason}\n";
+	$json = Cpanel::JSON::XS->new->utf8->decode($response->{content});
+	print "API: $json->{'status'}\n";
+	$data = $json->{'data'};
+
+	$length = (@{$data});
+	if ($length == 0) {
+		print "All data collected\n";
+		last;
+	}
+
+	open my $fh_out, ">>", $file_out;
+	for (my $i = 0; $i < $length; $i++) {
+		if (!$data->[$i]->{'domain'}) {
+			my $url = $data->[$i]->{'url'};
+			$url =~ s!^https?://(?:www\.)?!!i;
+			$url =~ s!/.*!!;
+			$url =~ s/[\?\#\:].*//;
+			$domain	= $url;
+		} else {
+			$domain = $data->[$i]->{'domain'};
+		}
+		if ($data->[$i]->{'cci'}) {
+			$cci = $data->[$i]->{'cci'};
+		} else {
+			$cci = 'none';
+		}
+	
+		my $timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime($data->[$i]->{'timestamp'}));
+		print $fh_out "$timestamp,$domain,$cci,$data->[$i]->{'category'},$data->[$i]->{'ccl'},$data->[$i]->{'user'}\n";
+	}
+	close $fh_out;
+	$skip += 5000;
+	#print "Next batch $skip\n";
+}
+print "Done\n";

ntskp-api-03.pl

@@ -0,0 +1,21 @@
+#!/usr/bin/perl -w
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use HTTP::Tiny;
+#use Cpanel::JSON::XS;
+use JSON::PP;
+
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}->{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}->{NTSKP_TOKEN};
+my $NTSKP_PERIOD = $config->{netskope}->{NTSKP_PERIOD};
+
+my $uri = "$NTSKP_TENANT/api/v1/events?token=$NTSKP_TOKEN&type=page&timeperiod=$NTSKP_PERIOD";
+my $response = HTTP::Tiny->new->get($uri);
+#my $json = Cpanel::JSON::XS->new->indent(1)->encode($response->{content});
+my $json = JSON::PP->new->pretty(1)->encode($response->{content});
+print $json;

ntskp-api-04.pl

@@ -0,0 +1,21 @@
+#!/usr/bin/perl -w
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use HTTP::Tiny;
+#use Cpanel::JSON::XS;
+use JSON::PP;
+
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}->{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}->{NTSKP_TOKEN};
+my $NTSKP_PERIOD = $config->{netskope}->{NTSKP_PERIOD};
+
+my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=498";
+my $response = HTTP::Tiny->new->get($uri);
+#my $json = Cpanel::JSON::XS->new->indent(1)->encode($response->{content});
+my $json = JSON::PP->new->pretty(1)->encode($response->{content});
+print $json;

ntskp-api-05.pl

@@ -0,0 +1,59 @@
+#!/usr/bin/perl -w
+use strict;
+use warnings;
+use autodie;
+use POSIX qw(strftime);
+use Config::Tiny;
+use HTTP::Tiny;
+use JSON::PP;
+
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_PERIOD = $config->{netskope}{NTSKP_PERIOD};
+my $NTSKP_SCORE = $config->{netskope}{NTSKP_SCORE};
+my $NTSKP_CATEGORIES = $config->{netskope}{NTSKP_CATEGORIES};
+my $from_email = 'mischa@netskope.com';
+my $to_email = 'mischa@netskope.com';
+my $subject = "AZ Blocklist Report";
+
+#print "Tenant: $NTSKP_TENANT\n";
+
+my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=498";
+my $response = HTTP::Tiny->new->get($uri);
+#print "HTTP: $response->{status} $response->{reason}\n";
+my $json = JSON::PP->new->utf8->decode($response->{content});
+#print "API: $json->{'status'}\n";
+my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+
+my $length = (@{$data});
+if ($length == 0) {
+	print "No widgets found\n";
+	last;
+}
+
+open my $fh_email, "|-", "/usr/sbin/sendmail -t";
+printf $fh_email "To: %s\n", $to_email;
+printf $fh_email "From: %s\n", $from_email;
+printf $fh_email "Subject: %s\n\n", $subject;
+
+for (my $i = 0; $i < $length; $i++) {
+	print "$data->[$i]->{'id'} - $data->[$i]->{'name'}\n";
+	print $fh_email "$data->[$i]->{'id'} - $data->[$i]->{'name'}\n";
+	$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$data->[$i]->{'id'}";
+	$response = HTTP::Tiny->new->get($uri);
+	#print "HTTP: $response->{status} $response->{reason}\n";
+	my $count = 0;
+	foreach (split(/\r\n/, $response->{content})) {
+		last if ($count == 30);
+		my @fields = split(/,/);
+		next if ($fields[1] =~ '"');
+		print "$fields[1],";
+		print $fh_email "$fields[1],";
+		$count++;
+	}
+	print "\n";
+	print $fh_email "\n";
+}
+close $fh_email;

ntskp-api-06.pl

@@ -0,0 +1,20 @@
+#!/usr/bin/perl -w
+use strict;
+use warnings;
+use autodie;
+use POSIX qw(strftime);
+use File::Temp qw/ tempfile tempdir /;
+use Text::CSV;
+
+my $file = "widget-18465-20200611.txt";
+open my $fh, "<", $file;
+
+my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+
+my $header = $csv->getline($fh);
+
+while (my $row = $csv->getline($fh)) {
+	print "$row->[1]\n";
+}
+close $fh;
+

ntskp-phishing.txt

@@ -0,0 +1,22 @@
+<html><head>
+<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
+</head>
+<body style="word-wrap: break-word; -webkit-nbsp-mode: space; line-break: after-white-space;" class="">
+<meta http-equiv="Content-Type" content="text/html; charset=us-ascii" class="">
+<div style="word-wrap: break-word; -webkit-nbsp-mode: space; line-break: after-white-space;" class="">
+Unfortunately, the service you are using has been targeted by a
+very sophisticated password attack. To protect you, the security
+team recommended that we reset all customer passwords immediately.
+Effective immediately, you will be required to reset your service
+password before you can login again. To reset your password please
+use your regular service password reset link.
+<div class=""><br class=""></div>
+<div class="">
+<a href="https://docs.google.com/forms/d/e/1FAIpQLSeuq5BnjifExxv7hmY5yaC0xBDyAk1IlEbxCEnqhG72brnGmQ/viewform?fbzx=-5005650978102270379" class="">Password Reset</a>
+<br class="">
+<div class=""><br class=""></div></div>
+<div class="">Regards,</div>
+<div class=""><br class=""></div>
+<div class="">Your friendly neighbourhood scammer</div>
+<div class=""><br class=""></div></div>
+</body></html>

ntskp-send.sh

@@ -0,0 +1,12 @@
+# high5.nl
+#cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@high5.nl
+#cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@high5.nl
+# ntskp.com
+cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" herman.akker23@ntskp.com
+cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" herman.akker23@ntskp.com
+# Microsoft
+cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com
+cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com
+# Died
+#cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com
+#cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com

ntskp-spam.txt

@@ -0,0 +1,17 @@
+<html><head>
+<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
+</head>
+<body style="word-wrap: break-word; -webkit-nbsp-mode: space; line-break: after-white-space;" class="">
+<meta http-equiv="Content-Type" content="text/html; charset=us-ascii" class="">
+<div style="word-wrap: break-word; -webkit-nbsp-mode: space; line-break: after-white-space;" class="">
+Please have a look at this CV at your earliest convenience.
+<div class=""><br class=""></div>
+<div class="">
+<a href="http://jn.gs/I7a" class="">LinkedIn CV</a>
+<br class="">
+<div class=""><br class=""></div></div>
+<div class="">Regards,</div>
+<div class=""><br class=""></div>
+<div class="">Your friendly neighbourhood scammer</div>
+<div class=""><br class=""></div></div>
+</body></html>

oss1.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+import os
+import sys
+import json
+import requests
+import configparser
+
+###############################################
+# Look for oss.cnf file in current working directory
+CONFIG_FILE = "./oss.cnf"
+if not os.path.isfile(CONFIG_FILE):
+	logging.error(f"The config file {CONFIG_FILE} doesn't exist")
+	sys.exit(1)
+config = configparser.RawConfigParser()
+config.read(CONFIG_FILE)
+NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT')
+NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN')
+NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD')
+
+###############################################
+
+ssl_session = requests.Session()
+uri = f"{NTSKP_TENANT}/api/v1/alerts?token={NTSKP_TOKEN}&timeperiod={NTSKP_PERIOD}&groupby=application&query=access_method+eq+Client+and+action+eq+block"
+try:
+	r = ssl_session.get(uri)
+	r.raise_for_status()
+except Exception as e:
+	logging.error(f'Error: {str(e)}')
+	sys.exit(1)
+json = r.json()
+if 'data' in json:
+	for item in json['data']:
+		if 'app' in item:
+			print(f"{item['app']} - {item['category']}", end=', ')
+	print()

oss2.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+import os
+import sys
+import json
+import requests
+import configparser
+
+###############################################
+# Look for oss.cnf file in current working directory
+CONFIG_FILE = "./oss.cnf"
+if not os.path.isfile(CONFIG_FILE):
+	logging.error(f"The config file {CONFIG_FILE} doesn't exist")
+	sys.exit(1)
+config = configparser.RawConfigParser()
+config.read(CONFIG_FILE)
+NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT')
+NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN')
+NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD')
+
+###############################################
+
+ssl_session = requests.Session()
+uri = f"{NTSKP_TENANT}/api/v1/report?token={NTSKP_TOKEN}&timeperiod={NTSKP_PERIOD}&type=connection&groupby=application&query=app-cci-app-tag+eq+'Under_Review'"
+try:
+	r = ssl_session.get(uri)
+	r.raise_for_status()
+except Exception as e:
+	logging.error(f'Error: {str(e)}')
+	sys.exit(1)
+json = r.json()
+if 'data' in json:
+	for item in json['data']:
+		print(f"{item['app']}", end=', ')
+	print()

oss3.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+import os
+import sys
+import json
+import requests
+import configparser
+
+###############################################
+# Look for oss.cnf file in current working directory
+CONFIG_FILE = "./oss.cnf"
+if not os.path.isfile(CONFIG_FILE):
+	logging.error(f"The config file {CONFIG_FILE} doesn't exist")
+	sys.exit(1)
+config = configparser.RawConfigParser()
+config.read(CONFIG_FILE)
+NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT')
+NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN')
+NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD')
+
+###############################################
+
+ssl_session = requests.Session()
+uri = f"{NTSKP_TENANT}/api/v1/report?token={NTSKP_TOKEN}&timeperiod={NTSKP_PERIOD}&type=connection&groupby=application&query=app-cci-app-tag+eq+'Pending_GRC_Review'"
+try:
+	r = ssl_session.get(uri)
+	r.raise_for_status()
+except Exception as e:
+	logging.error(f'Error: {str(e)}')
+	sys.exit(1)
+json = r.json()
+if 'data' in json:
+	for item in json['data']:
+		print(f"{item['app']}", end=', ')
+	print()

tbi.pl

@@ -0,0 +1,107 @@
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "";
+#my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my @CONFIG_FILES = grep { -e } ('./tbi.cnf');
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_TIMEPERIOD = $config->{netskope}{NTSKP_TIMEPERIOD};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status =~ /^2/ && $LOGMODE) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n";
+		print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+	}	
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+		);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	my $uri = "$NTSKP_TENANT/api/v1/alerts?token=$NTSKP_TOKEN&timeperiod=$NTSKP_TIMEPERIOD&type=policy";
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+	my $response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'};
+	printf "%-7s %-45s %-35s %s\n", "Action", "Page", "Policy", "Category";
+	say "#############################################################################################################################";
+
+	my @seen;
+	for my $item (@{$data}) {
+		if (exists($item->{'page'})) {
+			next if (grep {$_ eq $item->{'site'}} @seen);
+			printf "%-7s %-45s %-35s %s\n", $item->{'action'}, $item->{'page'}, $item->{'policy'}, $item->{'category'};
+			push @seen, $item->{'site'};
+		}
+	}
+
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+netskope();
+say "Completed." if $LOGMODE;

z.pl

@@ -0,0 +1,199 @@
+#!/usr/bin/env perl
+#
+# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
+# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# ZScaler integration with Netskope
+#
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+use MIME::Lite;
+
+my $LOGMODE = "";
+my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
+my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
+my $USER_COUNT = $config->{report}{USER_COUNT};
+my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+my $PROXY = $config->{general}{PROXY};
+my $SMTP = $config->{general}{SMTP};
+my $FROM = $config->{general}{FROM};
+my $TO = $config->{general}{TO};
+my $SUBJECT = $config->{general}{SUBJECT};
+my $TEXT = $config->{general}{TEXT} . "\n\n";
+my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
+my $EMAIL_CSV = "";
+
+### Netskope ###
+sub mail_csv {
+	my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
+	$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+	$msg->send('smtp', $SMTP, Debug=>0);
+	say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
+}
+
+sub _check_return {
+	my ($status, $content, $uri) = @_;
+	if ($status =~ /^2/ && $LOGMODE) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n";
+		print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
+	}	
+	if ($status !~ /^2/) {
+		print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
+		my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
+			Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
+		);
+		$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
+		$msg->send('smtp', $SMTP, Debug=>0);
+		say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
+		say "exit 1";
+		exit 1;
+	}
+}
+
+sub netskope {
+	### Collect widget IDs
+	my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+	my $response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
+	if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
+	my %csv_content;
+
+	### Collect widget data and write to CSV
+	for my $widget (@{$data}) {
+		$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
+		$response = $request->get($uri);
+		print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
+		_check_return($response->{'status'}, $response->{'content'}, $uri);
+		$csv_content{$widget->{'name'}} = $response->{'content'};
+	}
+
+	### Process domains from CSV
+	my @blocklist;
+	for my $widget_name (keys %csv_content) {
+		my $count = 0;
+		my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
+		open my $fh_in, "<", \$csv_content{$widget_name};
+		$csv->column_names($csv->getline($fh_in));
+		# "Application","Domain","Category","CCI","Users"
+
+		print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
+		$EMAIL_CSV .= "$widget_name\n";
+		DOMAIN:
+		while (my $row = $csv->getline_hr($fh_in)) {
+			last DOMAIN if ($count == $MAX_DOMAIN);
+			if ($row->{'Users'} < $USER_COUNT) {
+				print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
+				print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
+				$EMAIL_CSV .= "$row->{'Domain'},";
+				push @blocklist, $row->{'Domain'};
+				$count++;
+			}
+		}
+		print "\n\n" if $LOGMODE;
+		$EMAIL_CSV .= "\n";
+	}
+	return @blocklist;
+}
+
+#sub updateUrlList {
+	#my @domains = @{$_[0]};
+	#my $uri = "$NTSKP_TENANT/api/v1/updateUrlList?token=$NTSKP_TOKEN";
+	#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
+	#$body = JSON::PP->new->encode({name => $NTSKP_URL_CATEGORY, list => \@domains});
+	#my $response = $request->post($uri, {'content' => $body});
+	#_check_return($response->{'status'}, $response->{'content'}, $uri);
+#}
+
+### Zscaler ###
+
+sub zscaler {
+	my @domains = @{$_[0]};
+
+	### Authenticate 
+	my $now = int(gettimeofday * 1000);
+	my $n = substr($now, -6);
+	my $r = sprintf "%06d", $n >> 1;
+	my $key;
+	for my $i (0..length($n)-1) {
+		$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+	}
+	for my $i (0..length($r)-1) {
+		$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+	}
+	my $uri = "$ZS_BASE_URI/authenticatedSession";
+	my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+	my $jar = HTTP::CookieJar->new;
+	my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
+	my $response = $request->post($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Get filter list id
+	$uri = "$ZS_BASE_URI/urlCategories/lite";
+	$response = $request->get($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	my $json = JSON::PP->new->utf8->decode($response->{'content'});
+	my $id;
+	for my $item (@{$json}) {
+		if (exists($item->{'configuredName'})) {
+			if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+				$id = $item->{'id'};
+			}
+		}
+	}
+
+	### Push Domains
+	$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+	my $method = defined($id) ? "put" : "post";
+	my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+	splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
+	$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+	$response = $request->$method($uri, {'content' => $body});
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+
+	### Delete authenticadSession
+	$uri = "$ZS_BASE_URI/authenticatedSession";
+	$response = $request->delete($uri);
+	_check_return($response->{'status'}, $response->{'content'}, $uri);
+}
+
+say "Running in $LOGMODE mode..." if $LOGMODE;
+my @domains = netskope();
+zscaler(\@domains);
+mail_csv();
+say "Completed." if $LOGMODE;

zscaler-api.pl

@@ -0,0 +1,87 @@
+#!/usr/bin/perl
+use 5.024;
+use strict;
+use warnings;
+use autodie;
+use Config::Tiny;
+use Time::HiRes qw(gettimeofday);
+use POSIX qw(strftime);
+use HTTP::Tiny;
+use HTTP::CookieJar;
+use JSON::PP;
+use Text::CSV;
+
+my $verbose = 1;
+my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
+my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
+my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
+my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
+my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
+my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
+my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
+my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
+my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
+my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
+my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
+my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
+
+say "Running...";
+
+# Authenticate
+my $now = int(gettimeofday * 1000);
+my $n = substr($now, -6);
+my $r = sprintf "%06d", $n >> 1;
+my $key;
+for my $i (0..length($n)-1) {
+	$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
+}
+for my $i (0..length($r)-1) {
+	$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
+}
+my $uri = "$ZS_BASE_URI/authenticatedSession";
+my $body = JSON::PP->new->space_after->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
+my $jar = HTTP::CookieJar->new;
+my $request = HTTP::Tiny->new('default_headers' => {"Content-Type" => "application/json", "Cache-Control" => "no-cache"}, 'cookie_jar' => $jar);
+my $response = $request->post($uri, {'content' => $body});
+if ($verbose) {
+	say "POST $uri";
+	say "BODY $body";
+	say "HTTP " . $response->{'status'};
+	say "COOKIE " . $jar->cookie_header($ZS_BASE_URI);
+}
+
+# Get filter list id
+$uri = "$ZS_BASE_URI/urlCategories/lite";
+$response = $request->get($uri);
+my $json = JSON::PP->new->utf8->decode($response->{'content'});
+my $id;
+for my $item (@{$json}) {
+	if (exists($item->{'configuredName'})) {
+		if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
+			$id = $item->{'id'};
+		}
+	}
+}
+
+# Push Domains
+my @domains = ('secomtrust.net', 'baidupcs.com', 'cloud.baidu.com');
+$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
+my $method = defined($id) ? "put" : "post";
+my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
+$#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
+$body = JSON::PP->new->space_after->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
+$response = $request->$method($uri, {'content' => $body});
+if ($verbose) {
+	say uc($method) . " $uri";
+	say "BODY $body";
+	say "HTTP " . $response->{'status'};
+	say "RESPONSE " . $response->{'content'} if ($response->{'status'} =~ /^4/);
+}
+
+# Delete authenticadSession
+$uri = "$ZS_BASE_URI/authenticatedSession";
+$response = $request->delete($uri);
+if ($verbose) {
+	say "DELETE $uri";
+	say "HTTP " . $response->{'status'};
+}