From 49cb994257709fa8d6782d1572da50197870a15d Mon Sep 17 00:00:00 2001 From: mischa Date: Mon, 18 Jul 2022 17:28:22 +0200 Subject: [PATCH] Initial commit --- .gitignore | 3 + Netskope_APIEvents-01.py | 51 +++++ Netskope_APIEvents-02.py | 40 ++++ Netskope_APIEvents-03.py | 66 ++++++ Netskope_APIEvents-04.py | 68 +++++++ Netskope_APIEvents-05.py | 71 +++++++ Netskope_APIEvents-06.py | 89 ++++++++ Netskope_APIEvents-07.py | 95 +++++++++ Netskope_APIEvents-08.py | 80 ++++++++ Netskope_APIEvents-09.py | 80 ++++++++ Netskope_APIEvents-10.py | 91 +++++++++ Netskope_APIReport-01.pl | 86 ++++++++ Netskope_OPLPUploader-01.sh | 44 ++++ Netskope_ZScalerImporter-01.py | 145 +++++++++++++ Netskope_ZScalerImporter-02.pl | 192 +++++++++++++++++ Netskope_ZScalerImporter-03.pl | 179 ++++++++++++++++ Netskope_ZScalerImporter-04.pl | 183 +++++++++++++++++ Netskope_ZScalerImporter-05.pl | 189 +++++++++++++++++ Netskope_ZScalerImporter-06.pl | 190 +++++++++++++++++ Netskope_ZScalerImporter-07.pl | 190 +++++++++++++++++ Netskope_ZScalerImporter-08.pl | 199 ++++++++++++++++++ Netskope_ZScalerImporter-09.pl | 198 ++++++++++++++++++ Netskope_ZScalerImporter-10.pl | 207 +++++++++++++++++++ Netskope_ZScalerImporter-11.pl | 205 +++++++++++++++++++ Netskope_ZScalerImporter-12.pl | 276 +++++++++++++++++++++++++ Netskope_ZScalerImporter-13.pl | 277 +++++++++++++++++++++++++ Netskope_ZScalerImporter-14.pl | 280 +++++++++++++++++++++++++ Netskope_ZScalerImporter-wip.pl | 276 +++++++++++++++++++++++++ httpstat.py | 351 ++++++++++++++++++++++++++++++++ jsondump.py | 29 +++ measure.py | 46 +++++ ns.pl | 24 +++ ntskp-api-01.pl | 45 ++++ ntskp-api-02.pl | 69 +++++++ ntskp-api-03.pl | 21 ++ ntskp-api-04.pl | 21 ++ ntskp-api-05.pl | 59 ++++++ ntskp-api-06.pl | 20 ++ ntskp-phishing.txt | 22 ++ ntskp-send.sh | 12 ++ ntskp-spam.txt | 17 ++ oss1.py | 35 ++++ oss2.py | 34 ++++ oss3.py | 34 ++++ tbi.pl | 107 ++++++++++ z.pl | 199 ++++++++++++++++++ zscaler-api.pl | 87 ++++++++ 47 files changed, 5282 insertions(+) create mode 100644 .gitignore create mode 100755 Netskope_APIEvents-01.py create mode 100755 Netskope_APIEvents-02.py create mode 100755 Netskope_APIEvents-03.py create mode 100755 Netskope_APIEvents-04.py create mode 100755 Netskope_APIEvents-05.py create mode 100755 Netskope_APIEvents-06.py create mode 100755 Netskope_APIEvents-07.py create mode 100755 Netskope_APIEvents-08.py create mode 100755 Netskope_APIEvents-09.py create mode 100755 Netskope_APIEvents-10.py create mode 100755 Netskope_APIReport-01.pl create mode 100755 Netskope_OPLPUploader-01.sh create mode 100755 Netskope_ZScalerImporter-01.py create mode 100755 Netskope_ZScalerImporter-02.pl create mode 100755 Netskope_ZScalerImporter-03.pl create mode 100755 Netskope_ZScalerImporter-04.pl create mode 100755 Netskope_ZScalerImporter-05.pl create mode 100755 Netskope_ZScalerImporter-06.pl create mode 100755 Netskope_ZScalerImporter-07.pl create mode 100755 Netskope_ZScalerImporter-08.pl create mode 100755 Netskope_ZScalerImporter-09.pl create mode 100755 Netskope_ZScalerImporter-10.pl create mode 100755 Netskope_ZScalerImporter-11.pl create mode 100755 Netskope_ZScalerImporter-12.pl create mode 100755 Netskope_ZScalerImporter-13.pl create mode 100755 Netskope_ZScalerImporter-14.pl create mode 100755 Netskope_ZScalerImporter-wip.pl create mode 100755 httpstat.py create mode 100755 jsondump.py create mode 100755 measure.py create mode 100755 ns.pl create mode 100755 ntskp-api-01.pl create mode 100755 ntskp-api-02.pl create mode 100755 ntskp-api-03.pl create mode 100755 ntskp-api-04.pl create mode 100755 ntskp-api-05.pl create mode 100755 ntskp-api-06.pl create mode 100644 ntskp-phishing.txt create mode 100755 ntskp-send.sh create mode 100644 ntskp-spam.txt create mode 100755 oss1.py create mode 100755 oss2.py create mode 100755 oss3.py create mode 100755 tbi.pl create mode 100755 z.pl create mode 100755 zscaler-api.pl diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c60d3d4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +raw +*.cnf +zscaler.txt diff --git a/Netskope_APIEvents-01.py b/Netskope_APIEvents-01.py new file mode 100755 index 0000000..7f7c586 --- /dev/null +++ b/Netskope_APIEvents-01.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import json +import urllib.request +import argparse +import collections +from operator import itemgetter + +parser = argparse.ArgumentParser(description="API Call to collect data") +parser.add_argument("tenant", type=str, help="Tenant Name") +parser.add_argument("token", type=str, help="Tenat API Token") +parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod (default: 604800)") + +try: + args = parser.parse_args() + tenant = args.tenant + token = args.token + timeperiod = args.timeperiod + +except argparse.ArgumentError as e: + print(str(e)) + +def print_dict(dict): + for key, value in sorted(dict.items(), key = itemgetter(1), reverse = True): + print ("{:<35s}{:5d}".format(key, value)) + +base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod) + +req = urllib.request.Request(base_url) +with urllib.request.urlopen(req) as response: + content = response.read() +json_content = json.loads(content) + +domains = collections.Counter() +categories = collections.Counter() + +for i in range (0, len (json_content['data'])): + domain = json_content["data"][i]["domain"] + ccl = json_content["data"][i]["ccl"] + category = json_content["data"][i]["category"] + domains[domain] += 1 + categories[category][count] += 1 + + +#print ("===== Domains =====") +#print_dict(domains) + + +print ("\n===== Categories =====") +print_dict(categories) + diff --git a/Netskope_APIEvents-02.py b/Netskope_APIEvents-02.py new file mode 100755 index 0000000..a342ac7 --- /dev/null +++ b/Netskope_APIEvents-02.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +import json +import urllib.request +import argparse +from collections import Counter +from operator import itemgetter + +parser = argparse.ArgumentParser(description="API Call to collect data") +parser.add_argument("tenant", type=str, help="Tenant Name") +parser.add_argument("token", type=str, help="Tenat API Token") +parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod (default: 604800)") + +try: + args = parser.parse_args() + tenant = args.tenant + token = args.token + timeperiod = args.timeperiod + +except argparse.ArgumentError as e: + print(str(e)) + +def print_dict(dict, json_content): + for key, value in sorted(dict.items(), key = itemgetter(1), reverse = True): + print ("{:<35s}{:5d}".format(key, value)) + +base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod) +req = urllib.request.Request(base_url) +with urllib.request.urlopen(req) as response: + content = response.read() +json_content = json.loads(content) + +domains = Counter(data['domain'] for data in json_content['data']) +categories = Counter(data['category'] for data in json_content['data']) + +print ("==== Domains ===") +print_dict (domains, json_content) +print ("\n") +print ("==== Categories ===") +print_dict (categories) diff --git a/Netskope_APIEvents-03.py b/Netskope_APIEvents-03.py new file mode 100755 index 0000000..6876e35 --- /dev/null +++ b/Netskope_APIEvents-03.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# +# Copyright 2019, Mischa Peters , Netskope. +# Version 1.0 - 20191028 +# +# Requires: +# - Python 3.x +# +import json +import urllib.request +import argparse +from collections import Counter +from operator import itemgetter + +parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope") +parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)") +parser.add_argument("token", type=str, help="Tenat API Token") +parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 604800)") +parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)") +parser.add_argument("-s", "--show", action='store_true', help="Show category hits") + +try: + args = parser.parse_args() + tenant = args.tenant + token = args.token + timeperiod = args.timeperiod + rows = args.rows + show = args.show + +except argparse.ArgumentError as e: + print(str(e)) + +base_url = "https://{}.goskope.com/api/v1/events?token={}&type=application&timeperiod={}".format(tenant, token, timeperiod) + +req = urllib.request.Request(base_url) +with urllib.request.urlopen(req) as response: + content = response.read() +json_content = json.loads(content) + +domain_count = Counter() +domain_category = {} +category_count = Counter() +rows = None if rows == 0 else rows + +for i in range (0, len (json_content['data'])): + domain = json_content["data"][i]["domain"] + ccl = json_content["data"][i]["ccl"] + category = json_content["data"][i]["category"] + domain_count[domain] += 1 + domain_category[domain] = category + category_count[category] += 1 + +top_domains = domain_count.most_common(rows) +print ("{:<40s}{:>5s} - {}".format("Domain", "Hits", "Category")) +print ("################################################################################") +for i in top_domains: + print ("{:<40s}{:5d} - {}".format(i[0], i[1], domain_category[i[0]])) + +print ("") +if show: + top_categories = category_count.most_common() + print ("{:<40s}{:>5s}".format("Category", "Hits")) + print ("################################################################################") + for i in top_categories: + print ("{:<40s}{:5d}".format(i[0], i[1])) + diff --git a/Netskope_APIEvents-04.py b/Netskope_APIEvents-04.py new file mode 100755 index 0000000..12bf977 --- /dev/null +++ b/Netskope_APIEvents-04.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# +# Copyright 2019, Mischa Peters , Netskope. +# Version 1.0 - 20191028 +# +# Requires: +# - Python 3.x +# +import json +import urllib.request +import argparse +import collections +from operator import itemgetter + +parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope") +parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)") +parser.add_argument("token", type=str, help="Tenat API Token") +parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 604800)") +parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)") +parser.add_argument("-s", "--show", action='store_true', help="Show category hits") + +try: + args = parser.parse_args() + tenant = args.tenant + token = args.token + timeperiod = args.timeperiod + rows = args.rows + show = args.show + +except argparse.ArgumentError as e: + print(str(e)) + +base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod) + +req = urllib.request.Request(base_url) +with urllib.request.urlopen(req) as response: + content = response.read() +json_content = json.loads(content) + +#site = {'data': []} +site = collections.defaultdict(list); + +rows = None if rows == 0 else rows + +for i in range (0, len (json_content['data'])): + json_site = json_content["data"][i]["site"] + json_domain = json_content["data"][i]["domain"] + + if json_domain not in site[json_site]: + site[json_site].append(json_domain) + #print (json_site, "-", json_domain) + +print (site) + +for key, value in sorted(site.items(), key = itemgetter(0), reverse = False): + print ("{:<35s}".format(key), end="") + for i in value: + print ("{},".format(i), end="") + print ("") + + +#top_domains = domain_count.most_common(rows) +#print ("{:<40s}{:>5s} - {}".format("Domain", "Hits", "Category")) +#print ("################################################################################") +#for i in top_domains: + #print ("{:<40s}{:5d} - {}".format(i[0], i[1], domain_category[i[0]])) + + diff --git a/Netskope_APIEvents-05.py b/Netskope_APIEvents-05.py new file mode 100755 index 0000000..bb0e807 --- /dev/null +++ b/Netskope_APIEvents-05.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# +# Copyright 2019, Mischa Peters , Netskope. +# Version 1.0 - 20191028 +# +# Requires: +# - Python 3.x +# +import json +import urllib.request +import argparse +from collections import Counter +from operator import itemgetter + +parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope") +parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)") +parser.add_argument("token", type=str, help="Tenat API Token") +parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)") +parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)") +parser.add_argument("-s", "--show", action='store_true', help="Show category hits") +parser.add_argument("-d", "--debug", action='store_true', help="debug") + +try: + args = parser.parse_args() + tenant = args.tenant + token = args.token + timeperiod = args.timeperiod + rows = args.rows + show = args.show + debug = args.debug + +except argparse.ArgumentError as e: + print(str(e)) + +domain_count = Counter() +domain_category = {} +category_count = Counter() +rows = None if rows == 0 else rows + +def get_json(type): + domain = "goskope.com" + url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}" + req = urllib.request.Request(url) + with urllib.request.urlopen(req) as response: + content = response.read() + json_data = json.loads(content) + if debug: print (json_data) + return(json_data) + +json_content = get_json("application") +for i in range (0, len (json_content['data'])): + domain = json_content["data"][i]["domain"] + ccl = json_content["data"][i]["ccl"] + category = json_content["data"][i]["category"] + domain_count[domain] += 1 + domain_category[domain] = category + category_count[category] += 1 + +top_domains = domain_count.most_common(rows) +print (f"{'Domain':<40s}{'Hits':>5s} - Category") +print ("################################################################################") +for i in top_domains: + print (f"{i[0]:<40s}{i[1]:5d} - {domain_category[i[0]]}") + +print ("") +if show: + top_categories = category_count.most_common() + print (f"{'Category':<40s}{'Hits':>5s}") + print ("################################################################################") + for i in top_categories: + print (f"{i[0]:<40s}{i[1]:5d}") diff --git a/Netskope_APIEvents-06.py b/Netskope_APIEvents-06.py new file mode 100755 index 0000000..2f4d2dd --- /dev/null +++ b/Netskope_APIEvents-06.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# +# Copyright 2019, Mischa Peters , Netskope. +# Version 1.0 - 20191107 +# +# Requires: +# - Python 3.x +# +import json +import urllib.request +import argparse +import sys +from urllib.parse import urlparse +import re + +parser = argparse.ArgumentParser(description="Collect all page events from Netskope API and process domains by category and confidence") +parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)") +parser.add_argument("token", type=str, help="Tenant API Token") +parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)") +parser.add_argument("-r", "--records", type=int, default=100, help="# of records (default: 100)") +parser.add_argument("-v", "--verbose", action='store_true', help="verbose") +parser.add_argument("-d", "--debug", action='store_true', help="debug") + +try: + args = parser.parse_args() + tenant = args.tenant + token = args.token + timeperiod = args.timeperiod + records = args.records + verbose = args.verbose + debug = args.debug + +except argparse.ArgumentError as e: + print(str(e)) + +cursor_up = '\x1b[1A' +erase_line = '\x1b[2K' +cct_list = ["Cloud Storage", "Webmail"] +ccl_list = ["low", "poor"] +whitelist = re.compile("bla") +ioc_list = [] +i = 0 + +if verbose: + print("Using Categories: ", end='', flush=True) + print(", ".join(map(str,cct_list))) + print("Using Rating: ", end='', flush=True) + print(", ".join(map(str,ccl_list))) + print(f"Applying Whitelist for: {whitelist.pattern}") + +def get_json(type): + domain = "goskope.com" + url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}" + req = urllib.request.Request(url) + with urllib.request.urlopen(req) as response: + content = response.read() + json_data = json.loads(content) + if debug: print (json_data) + return(json_data) + +print() +print("Processing...", end='', flush=True) +json_content = get_json("page") +sys.stdout.write(cursor_up) +sys.stdout.write(erase_line) +print() + +if verbose: + print(f"{'#':>4} {'Domain':<50s} Confidence") + print("#######################################################################") + +for index, data in enumerate(json_content['data']): + if not "domain" in data: + domain = urlparse(data["url"]).netloc + else: + domain = data["domain"] + if whitelist.search(domain): + continue + if data["category"] in cct_list: + if data["ccl"] in ccl_list: + if domain not in ioc_list: + i += 1 + if verbose: print(f"{i:>4}) {domain:<50s} {data['ccl']}") + ioc_list.append(domain) + if i == records: + break + +if verbose: print() +print(", ".join(map(str,ioc_list))) diff --git a/Netskope_APIEvents-07.py b/Netskope_APIEvents-07.py new file mode 100755 index 0000000..38dcecc --- /dev/null +++ b/Netskope_APIEvents-07.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# +# Copyright 2019, Mischa Peters , Netskope. +# Version 1.0 - 20191107 +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# Requires: +# - Python 3.x +# +import json +import urllib.request +import argparse +import sys +import urllib.parse +import re +import requests + +parser = argparse.ArgumentParser(description="Collect all page events from Netskope API and process domains by category and confidence") +parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)") +parser.add_argument("token", type=str, help="Tenant API Token") +parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)") +parser.add_argument("-r", "--records", type=int, default=100, help="# of records (default: 100)") +parser.add_argument("-v", "--verbose", action='store_true', help="verbose") +parser.add_argument("-d", "--debug", action='store_true', help="print raw json data") + +try: + args = parser.parse_args() + tenant = args.tenant + token = args.token + timeperiod = args.timeperiod + records = args.records + verbose = args.verbose + debug = args.debug + +except argparse.ArgumentError as e: + print(str(e)) + +cct_list = ["Cloud Storage", "Webmail"] +ccl_list = ["low", "poor"] +whitelist = re.compile("yahoo") +ioc_list = [] + +if verbose: + print("Using Categories: ", end='', flush=True) + print(", ".join(map(str,cct_list))) + print("Using Rating: ", end='', flush=True) + print(", ".join(map(str,ccl_list))) + print(f"Applying Whitelist: {whitelist.pattern}") + print() + print(f"{'#':>4} {'Domain':<50s} Confidence") + print("#######################################################################") + +def get_json(type): + domain = "goskope.com" + url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}" + req = urllib.request.Request(url) + with urllib.request.urlopen(req) as response: + content = response.read() + json_data = json.loads(content) + if debug: print (json_data) + return(json_data) + +def parse_json(json_content): + i = 0 + for index, data in enumerate(json_content['data']): + if not "domain" in data: + domain = urllib.parse.urlparse(data["url"]).netloc + else: + domain = data["domain"] + if whitelist.search(domain): + continue + if data["category"] in cct_list: + if data["ccl"] in ccl_list: + if domain not in ioc_list: + i += 1 + if verbose: print(f"{i:>4}) {domain:<50s} {data['ccl']}") + ioc_list.append(domain) + + return ioc_list + #domain_list = ", ".join(map(str,ioc_list[:records])) + #return domain_list + +json = get_json("page") +print(parse_json(json)) diff --git a/Netskope_APIEvents-08.py b/Netskope_APIEvents-08.py new file mode 100755 index 0000000..c8b33be --- /dev/null +++ b/Netskope_APIEvents-08.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# +# Copyright 2019, Mischa Peters , Netskope. +# Version 1.0 - 20191107 +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# Requires: +# - Python 3.x +# +import os +import sys +import json +import time +import re +import logging +import urllib.parse +import requests + +NTSKP_TENANT = 'https://astrazeneca.eu.goskope.com' +NTSKP_TOKEN = '604d0a3b26ea9b22c3ec42130ebbfa8e' +NTSKP_PERIOD = '2592000' +cct_list = ["Cloud Storage", "Webmail"] +ccl_list = ["low", "poor"] +whitelist = re.compile("yahoo") +ioc_list = [] + +ZS_MAX_DOMAINS = 2 +headers = {'Content-Type': 'application/json', 'Cache-Control': 'no-cache', 'User-Agent': 'Netskope_ZscalerImporter1.0'} +PROXY='' + +logging.basicConfig(level=logging.DEBUG) +logging = logging.getLogger('zsc') + +def ntskp_get_domains(headers): + uri = f"{NTSKP_TENANT}/api/v1/events?token={NTSKP_TOKEN}&type=page&timeperiod={NTSKP_PERIOD}" + try: + r = requests.get(uri, headers=headers, proxies=PROXY) + r.raise_for_status() + except Exception as e: + logging.error('Error: ' + str(e)) + sys.exit(1) + json = r.json() + limit = (len(json['data'])) + + for item in json['data']: + if not "domain" in item: + domain = urllib.parse.urlparse(item['url']).netloc + else: + domain = item['domain'] + if whitelist.search(domain): + continue + if item['category'] in cct_list: + if item['ccl'] in ccl_list: + if domain not in ioc_list: + print(f"{domain:<50s} {item['ccl']}") + endtime = item['timestamp'] + ioc_list.append(domain) + print(limit) + print(endtime) + starttime = endtime - (10 * 60) + print(ioc_list[:ZS_MAX_DOMAINS]) + return ioc_list[:ZS_MAX_DOMAINS] + + +ntskp_get_domains(headers) + +now = int(time.time() * 1000) +print(now) +#print(str(time.ctime(int(time.time())))) diff --git a/Netskope_APIEvents-09.py b/Netskope_APIEvents-09.py new file mode 100755 index 0000000..9e8322d --- /dev/null +++ b/Netskope_APIEvents-09.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# +# Copyright 2019, Mischa Peters , Netskope. +# Version 1.0 - 20191028 +# +# Collects all the page events, counts all the domain hits and category hits +# +# Requires: +# - Python 3.x +# +import json +import urllib.request +import argparse +from collections import Counter +from operator import itemgetter + +parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope") +parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)") +parser.add_argument("token", type=str, help="Tenat API Token") +parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)") +parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)") +parser.add_argument("-s", "--show", action='store_true', help="Show category hits") +parser.add_argument("-d", "--debug", action='store_true', help="debug") + +try: + args = parser.parse_args() + tenant = args.tenant + token = args.token + timeperiod = args.timeperiod + rows = args.rows + show = args.show + debug = args.debug + +except argparse.ArgumentError as e: + print(str(e)) + +domain_count = Counter() +domain_category = {} +domain_ccl = {} +domain_cci = {} +category_count = Counter() +rows = None if rows == 0 else rows + +def get_json(type): + domain = "goskope.com" + url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}" + req = urllib.request.Request(url) + with urllib.request.urlopen(req) as response: + content = response.read() + json_data = json.loads(content) + if debug: print (json_data) + print(json.dumps(json_data, indent=4, sort_keys=True)) + return(json_data) + +json_content = get_json("page") +for i in range (0, len (json_content['data'])): + domain = json_content["data"][i]["domain"] + ccl = json_content["data"][i]["ccl"] + category = json_content["data"][i]["category"] + #ccl = json_content["data"][i]["ccl"] + cci = json_content["data"][i]["cci"] + domain_count[domain] += 1 + domain_category[domain] = category + domain_ccl[domain] = ccl + domain_cci[domain] = cci +category_count[category] += 1 + +top_domains = domain_count.most_common(rows) +print (f"{'Domain':<40s}{'Hits':>5s} - Category") +print ("################################################################################") +for i in top_domains: + print (f"{i[0]:<40s}{i[1]:5d} - {domain_category[i[0]]} - {domain_ccl[i[0]]}") + +print ("") +if show: + top_categories = category_count.most_common() + print (f"{'Category':<40s}{'Hits':>5s}") + print ("################################################################################") + for i in top_categories: + print (f"{i[0]:<40s}{i[1]:5d}") diff --git a/Netskope_APIEvents-10.py b/Netskope_APIEvents-10.py new file mode 100755 index 0000000..c21d33f --- /dev/null +++ b/Netskope_APIEvents-10.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# +import os +import sys +import json +import time +import re +import logging +import urllib.parse +import requests +import configparser +from datetime import datetime + +############################################### + +CONFIG_FILE = "/home/mischa/netskope/netskope.cnf" +if not os.path.isfile(CONFIG_FILE): + logging.error(f"The config file {CONFIG_FILE} doesn't exist") + sys.exit(1) +config = configparser.RawConfigParser() +config.read(CONFIG_FILE) +NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT') +NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN') +NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD') +NTSKP_SCORE = config.get('netskope', 'NTSKP_SCORE') +NTSKP_CATEGORIES = config.get('netskope', 'NTSKP_CATEGORIES') +NTSKP_CONFIDENCE = config.get('netskope', 'NTSKP_CONFIDENCE') +PROXY = config.get('general', 'PROXY') + +############################################### + +# Use a custom user-agent string +UA_STRING = 'NetskopeAPICollector1.0' + +# Set logging.INFO to logging.DEBUG for debug information +logging.basicConfig(level=logging.INFO) +logging = logging.getLogger('NetskopeAPICollector') + +############################################### + +def ntskp_get_domains(headers): + skip = 0 + filename = f"/home/mischa/netskope/api-{datetime.now().strftime('%Y%m%d')}.txt" + logging.info(f"File {filename} created") + ssl_session = requests.Session() + logging.debug(f"{ssl_session}") + + while True: + uri = f'{NTSKP_TENANT}/api/v1/events?token={NTSKP_TOKEN}&type=page&timeperiod={NTSKP_PERIOD}&skip={skip}' + try: + r = ssl_session.get(uri, headers=headers, proxies=PROXY) + r.raise_for_status() + except Exception as e: + logging.error(f'Error: {str(e)}') + sys.exit(1) + json = r.json() + #if json['data']: + if 'data' in json: + if len(json['data']) <= 5000: + skip += 5000 + filter_file = open(filename, "a") + logging.debug(f"File {filename} opened") + for item in json['data']: + if not 'domain' in item: + domain = urllib.parse.urlparse(item['url']).netloc + else: + domain = item['domain'] + + #if NTSKP_SAFELIST.search(domain): + #print(domain) + + #if item['ccl'] in NTSKP_CONFIDENCE: + utctime = datetime.utcfromtimestamp(item['timestamp']).strftime('%Y-%m-%d %H:%M:%S') + filter_file.write(f"{utctime},{domain},{item['cci']},{item['category']},{item['ccl']},{item['user']}\n") + filter_file.close() + logging.debug(f"File {filename} closed") + logging.debug(f"Next request, skip: {skip}") + else: + logging.info(f"No more data to collect") + break + else: + logging.info(f"No more data to collect") + break + if skip == 500000: + logging.info(f"Reached limit") + break + +############################################### + +request_headers = {'Content-Type': 'application/json', 'Cache-Control': 'no-cache', 'User-Agent': UA_STRING} +ntskp_get_domains(request_headers) diff --git a/Netskope_APIReport-01.pl b/Netskope_APIReport-01.pl new file mode 100755 index 0000000..20338b5 --- /dev/null +++ b/Netskope_APIReport-01.pl @@ -0,0 +1,86 @@ +#!/usr/bin/perl -w +use strict; +use warnings; +use autodie; +use Config::Tiny; +use HTTP::Tiny; +use JSON::PP; +use Text::CSV; +use File::Temp; +use MIME::Lite; + +my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"; +my $config = Config::Tiny->read($CONFIG_FILE, 'utf8'); +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; + +my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; +my $response = HTTP::Tiny->new->get($uri); +my $json = JSON::PP->new->utf8->decode($response->{'content'}); +my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; +my %files; + +for my $widget (@{$data}) { + my $tmp_file = File::Temp->new(UNLINK => 0, TEMPLATE => 'tempXXXXX', DIR => '/tmp'); + $files{$tmp_file} = $widget->{'name'}; + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = HTTP::Tiny->new->get($uri); + open my $fh_out, ">", $tmp_file; + print $fh_out $response->{'content'}; + close $fh_out; +} + +my $out_email = "azblocklist.csv"; +my $out_zscaler = "zscaler.txt"; +open my $fh_email, ">", $out_email; +open my $fh_zscaler, ">", $out_zscaler; + +for my $item (keys %files) { + my $count = 0; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh, "<", $item; + my $header = $csv->getline($fh); + + print "$files{$item}\n"; + print $fh_email "$files{$item}\n"; + + while (my $row = $csv->getline($fh)) { + last if ($count == 30); + if ($row->[1] =~ m/,/) { + my @domains = split "," , $row->[1]; + for my $domain (@domains) { + print "$domain,"; + print $fh_email "$domain,"; + print $fh_zscaler "$domain\n"; + } + } else { + print "$row->[1],"; + print $fh_email "$row->[1],"; + print $fh_zscaler "$row->[1]\n"; + } + $count++; + } + print "\n"; + print $fh_email "\n"; + close $fh; + unlink $item; +} +close $fh_email; +close $fh_zscaler; + +my $msg = MIME::Lite->new( + From => 'mischa@high5.nl', + To => 'mischa@netskope.com', + Cc => 'mischa@high5.nl', + Subject => 'AztraZeneca Netskope Blocklist', + Type => 'TEXT', + Data => "Domains pushed to Zscaler for blocking\n\n" +); +$msg->attach( + Type => 'text/csv', + Path => $out_email, + Filename => $out_email +); +$msg->send('smtp','mail.high5.nl', Debug=>0); +unlink $out_email; diff --git a/Netskope_OPLPUploader-01.sh b/Netskope_OPLPUploader-01.sh new file mode 100755 index 0000000..7019a1c --- /dev/null +++ b/Netskope_OPLPUploader-01.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright 2019, Mischa Peters , Netskope. +# OPLPUploader.sh - Version 1.0 - 20200113 +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# find ${LOCALDIR} -type f -name "*.csv" -maxdepth 1 | sort -V + +HOST="ftp://" +LOCALDIR="/tmp/files" +REMOTEDIR="/nslogs/user/upload/custom-ASML_SplunkCurlv1/" +USER='' +PASS='' +GLOB='*.csv' +LOG="/tmp/script.log" + +if [ -d ${LOCALDIR} ]; then + cd ${LOCALDIR} +else + echo "$(date "+%Y-%m-%d %T") ${LOCALDIR} doesn't exist" | tee -a ${LOG} + exit 1 +fi + +lftp ${HOST} <<- UPLOAD + user "${USER}" "${PASS}" + cd "${REMOTEDIR}" + mput -E "${GLOB}" +UPLOAD + +if [ ! $? -eq 0 ]; then + echo "$(date "+%Y-%m-%d %T") unable to upload files" | tee -a ${LOG} + exit 1 +fi diff --git a/Netskope_ZScalerImporter-01.py b/Netskope_ZScalerImporter-01.py new file mode 100755 index 0000000..ec255af --- /dev/null +++ b/Netskope_ZScalerImporter-01.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# +# Copyright 2019-2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.py - Version 2.0 - 20200611 +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +import os +import sys +import re +import json +import csv +import time +import logging +import urllib.parse +import requests +import configparser + +############################################### + +CONFIG_FILE = "/home/mischa/netskope/netskope.cnf" +if not os.path.isfile(CONFIG_FILE): + logging.error(f"The config file {CONFIG_FILE} doesn't exist") + sys.exit(1) +config = configparser.RawConfigParser() +config.read(CONFIG_FILE) +NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT') +NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN') +ZS_MAX_DOMAINS = int(config.get('zscaler', 'ZS_MAX_DOMAINS')) +ZS_BASE_URI = config.get('zscaler', 'ZS_BASE_URI') +ZS_API_KEY = config.get('zscaler', 'ZS_API_KEY') +ZS_API_USERNAME = config.get('zscaler', 'ZS_API_USERNAME') +ZS_API_PASSWORD = config.get('zscaler', 'ZS_API_PASSWORD') +ZS_CATEGORY_NAME = config.get('zscaler', 'ZS_CATEGORY_NAME') +ZS_CATEGORY_DESC = config.get('zscaler', 'ZS_CATEGORY_DESC') +PROXY = config.get('general', 'PROXY') + +############################################### + +# Use a custom user-agent string +UA_STRING = 'Netskope_ZScalerImporter1.0' + +# Set logging.INFO to logging.DEBUG for debug information +logging.basicConfig(level=logging.DEBUG) +logging = logging.getLogger('Netskope_ZScalerImporter') + +def ntskp_get_domains(): + ioc_list = [] + with open('zscaler.txt') as f: + ioc_list = f.read().splitlines() + logging.debug(ioc_list[:ZS_MAX_DOMAINS]) + return ioc_list[:ZS_MAX_DOMAINS] + +def zs_auth(headers): + # Authenticatie against ZScaler API, fetch and return JSESSIONID + now = int(time.time() * 1000) + n = str(now)[-6:] + r = str(int(n) >> 1).zfill(6) + key = "" + for i in range(0, len(str(n)), 1): + key += ZS_API_KEY[int(str(n)[i])] + for j in range(0, len(str(r)), 1): + key += ZS_API_KEY[int(str(r)[j])+2] + + uri = f'{ZS_BASE_URI}/authenticatedSession' + body = {'apiKey': key, + 'username': ZS_API_USERNAME, + 'password': ZS_API_PASSWORD, + 'timestamp': now} + try: + r = requests.post(uri, data=json.dumps(body), headers=headers, proxies=PROXY) + r.raise_for_status() + jsessionid = re.sub(r';.*$', "", r.headers['Set-Cookie']) + except Exception as e: + logging.error(f'Error: {str(e)}') + sys.exit(1) + return jsessionid + +def zs_get_categories(headers): + # Find any existing categories matching ZS_CATEGORY_NAME + uri = f'{ZS_BASE_URI}/urlCategories/lite' + try: + r = requests.get(uri, headers=headers, proxies=PROXY) + r.raise_for_status() + except Exception as e: + logging.error(f'Error: {str(e)}') + sys.exit(1) + data = r.json() + for item in data: + if item.get('configuredName') == ZS_CATEGORY_NAME: + return item.get('id') + return None + +def zs_update_categories(headers, domains, id = None): + # Update the ZS_CATEGORY_NAME with blocklist from Netskope + description = f'{ZS_CATEGORY_DESC}\n\nLast Updated: {str(time.ctime(int(time.time())))}' + body = {'configuredName': ZS_CATEGORY_NAME, + 'customCategory': 'true', + 'superCategory': 'SECURITY', + 'urls': domains, + 'description': description} + try: + if id == None: + uri = f'{ZS_BASE_URI}/urlCategories' + r = requests.post(uri, json=body, headers=headers, proxies=PROXY) + else: + uri = f'{ZS_BASE_URI}/urlCategories/{str(id)}' + r = requests.put(uri, json=body, headers=headers, proxies=PROXY) + r.raise_for_status() + except Exception as e: + logging.error(f'Error: {str(e)}') + sys.exit(1) + return None + +def zs_logout(headers): + # Logout from ZScaler + uri = f'{ZS_BASE_URI}/authenticatedSession' + try: + r = requests.delete(uri, headers=headers, proxies=PROXY) + r.raise_for_status() + except Exception as e: + logging.error(f'Error: {str(e)}') + sys.exit(1) + return None + +############################################## + +request_headers = {'Content-Type': 'application/json', 'Cache-Control': 'no-cache', 'User-Agent': UA_STRING} +domains = ntskp_get_domains() +request_headers['Cookie'] = zs_auth(request_headers) +zs_update_categories(request_headers, domains, zs_get_categories(request_headers)) +zs_logout(request_headers) +logging.info(f'Netskope added {str(len(domains))} domains added to ZScaler custom URL category {ZS_CATEGORY_NAME}') diff --git a/Netskope_ZScalerImporter-02.pl b/Netskope_ZScalerImporter-02.pl new file mode 100755 index 0000000..49e0ce1 --- /dev/null +++ b/Netskope_ZScalerImporter-02.pl @@ -0,0 +1,192 @@ +#!/usr/bin/perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615 +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.024; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use File::Temp; +use MIME::Lite; + +my $VERBOSE = 1; +my $DEBUG = 1; +my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"; +my $config = Config::Tiny->read($CONFIG_FILE, 'utf8'); +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $FILENAME = $config->{general}{FILENAME}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %headers = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->attach( Type => 'text/csv', Path => $FILENAME, Filename => $FILENAME); + $msg->send('smtp', $SMTP, Debug=>0); + say "SMTP $FROM -> $TO - CSV" if $VERBOSE; + unlink $FILENAME; +} + +sub check_return { + my ($status, $content, $uri) = @_; + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'Error', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n", + ); + $msg->send('smtp', $SMTP, Debug=>0); + say "SMTP $FROM -> $TO - ERROR" if $VERBOSE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + ### Collect widget IDs + my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; + my $request = HTTP::Tiny->new('default_headers' => \%headers); + my $response = $request->get($uri); + check_return($response->{'status'}, $response->{'content'}, $uri); + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + if (!$data) { check_return(404, $response->{'content'}, "No Widget Data"); } + my %files; + + + ### Collect widget data and write to CSV + for my $widget (@{$data}) { + my $tmp_file = File::Temp->new(UNLINK => 0, TEMPLATE => 'tempXXXXX', DIR => '/tmp'); + $files{$tmp_file} = $widget->{'name'}; + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = $request->get($uri); + print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if $DEBUG; + check_return($response->{'status'}, $response->{'content'}, $uri); + + open my $fh_out, ">", $tmp_file; + print $fh_out $response->{'content'}; + close $fh_out; + } + + ### Process domains from CSV + my @blocklist; + open my $fh_email_out, ">", $FILENAME; + + for my $csv_file (keys %files) { + my $count = 0; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh_in, "<", $csv_file; + my $header = $csv->getline($fh_in); + + print "\n## Widget Name: $files{$csv_file}\nDomains: " if $VERBOSE; + print $fh_email_out "$files{$csv_file}\n"; + + while (my $row = $csv->getline($fh_in)) { + last if ($count == 30); + print "$row->[1]," if $VERBOSE; + print $fh_email_out "$row->[1],"; + push @blocklist, $row->[1]; + $count++; + } + print "\n" if $VERBOSE; + print $fh_email_out "\n"; + close $fh_in; + unlink $csv_file; + } + close $fh_email_out; + return @blocklist; +} + +### Zscaler ### + +sub zscaler { + my @domains = @{$_[0]}; + + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + my $request = HTTP::Tiny->new('default_headers' => \%headers, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + ### Push Domains + $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories"; + my $method = defined($id) ? "put" : "post"; + my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); + $#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains; + $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); + $response = $request->$method($uri, {'content' => $body}); + check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + check_return($response->{'status'}, $response->{'content'}, $uri); +} + +say "Running..." if $VERBOSE; +my @domains = netskope(); +zscaler(\@domains); +mail_csv(); +say "Completed." if $VERBOSE; diff --git a/Netskope_ZScalerImporter-03.pl b/Netskope_ZScalerImporter-03.pl new file mode 100755 index 0000000..325664c --- /dev/null +++ b/Netskope_ZScalerImporter-03.pl @@ -0,0 +1,179 @@ +#!/usr/bin/perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615 +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.024; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use MIME::Lite; + +my $VERBOSE = 1; +my $DEBUG = 0; +my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"; +my $config = Config::Tiny->read($CONFIG_FILE, 'utf8'); +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); +my $EMAIL_CSV; + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "SMTP $FROM -> $TO - CSV" if $VERBOSE; +} + +sub check_return { + my ($status, $content, $uri) = @_; + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'Error', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n", + ); + $msg->send('smtp', $SMTP, Debug=>0); + say "SMTP $FROM -> $TO - ERROR" if $VERBOSE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + ### Collect widget IDs + my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS); + my $response = $request->get($uri); + check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + if (!$data) { check_return(404, $response->{'content'}, "No Widget Data"); } + my %csv_content; + + ### Collect widget data and write to CSV + for my $widget (@{$data}) { + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = $request->get($uri); + print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if $DEBUG; + check_return($response->{'status'}, $response->{'content'}, $uri); + $csv_content{$widget->{'name'}} = $response->{'content'}; + } + + ### Process domains from CSV + my @blocklist; + for my $widget_name (keys %csv_content) { + my $count = 0; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh_in, "<", \$csv_content{$widget_name}; + my $header = $csv->getline($fh_in); + + print "\n## Widget Name: $widget_name\n## Domains: " if $VERBOSE; + $EMAIL_CSV .= "$widget_name\n"; + while (my $row = $csv->getline($fh_in)) { + last if ($count == 30); + print "$row->[1]," if $VERBOSE; + $EMAIL_CSV .= "$row->[1],"; + push @blocklist, $row->[1]; + $count++; + } + print "\n" if $VERBOSE; + $EMAIL_CSV .= "\n"; + } + return @blocklist; +} + +### Zscaler ### + +sub zscaler { + my @domains = @{$_[0]}; + + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + ### Push Domains + $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories"; + my $method = defined($id) ? "put" : "post"; + my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); + $#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains; + $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); + $response = $request->$method($uri, {'content' => $body}); + check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + check_return($response->{'status'}, $response->{'content'}, $uri); +} + +say "Running..." if $VERBOSE; +my @domains = netskope(); +zscaler(\@domains); +mail_csv(); +say "Completed." if $VERBOSE; diff --git a/Netskope_ZScalerImporter-04.pl b/Netskope_ZScalerImporter-04.pl new file mode 100755 index 0000000..12ba9cd --- /dev/null +++ b/Netskope_ZScalerImporter-04.pl @@ -0,0 +1,183 @@ +#!/usr/bin/perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615 +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.024; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use MIME::Lite; + +my $LOGMODE = "VERBOSE"; +my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"; +my $config = Config::Tiny->read($CONFIG_FILE, 'utf8'); +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); +my $EMAIL_CSV = ""; + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE; +} + +sub check_return { + my ($status, $content, $uri) = @_; + if ($status =~ /^2/ && $LOGMODE) { + print "URI: $uri\nHTTP RESPONSE: $status\n"; + print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG"); + } + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n", + ); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + ### Collect widget IDs + my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS); + my $response = $request->get($uri); + check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + if (!$data) { check_return(404, $response->{'content'}, "No Widget Data"); } + my %csv_content; + + ### Collect widget data and write to CSV + for my $widget (@{$data}) { + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = $request->get($uri); + print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG"); + check_return($response->{'status'}, $response->{'content'}, $uri); + $csv_content{$widget->{'name'}} = $response->{'content'}; + } + + ### Process domains from CSV + my @blocklist; + for my $widget_name (keys %csv_content) { + my $count = 0; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh_in, "<", \$csv_content{$widget_name}; + my $header = $csv->getline($fh_in); + + print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE; + $EMAIL_CSV .= "$widget_name\n"; + while (my $row = $csv->getline($fh_in)) { + last if ($count == 30); + print "$row->[1]," if $LOGMODE; + $EMAIL_CSV .= "$row->[1],"; + push @blocklist, $row->[1]; + $count++; + } + print "\n\n" if $LOGMODE; + $EMAIL_CSV .= "\n"; + } + return @blocklist; +} + +### Zscaler ### + +sub zscaler { + my @domains = @{$_[0]}; + + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + ### Push Domains + $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories"; + my $method = defined($id) ? "put" : "post"; + my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); + $#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains; + $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); + $response = $request->$method($uri, {'content' => $body}); + check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + check_return($response->{'status'}, $response->{'content'}, $uri); +} + +say "Running in $LOGMODE mode..." if $LOGMODE; +my @domains = netskope(); +zscaler(\@domains); +mail_csv(); +say "Completed." if $LOGMODE; diff --git a/Netskope_ZScalerImporter-05.pl b/Netskope_ZScalerImporter-05.pl new file mode 100755 index 0000000..064f34b --- /dev/null +++ b/Netskope_ZScalerImporter-05.pl @@ -0,0 +1,189 @@ +#!/usr/bin/perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615 +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.024; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use MIME::Lite; + +my $LOGMODE = "DEBUG"; +my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"; +my $config = Config::Tiny->read($CONFIG_FILE, 'utf8'); +my $USER_COUNT = $config->{report}{USER_COUNT}; +my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN}; +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); +my $EMAIL_CSV = ""; + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE; +} + +sub _check_return { + my ($status, $content, $uri) = @_; + if ($status =~ /^2/ && $LOGMODE) { + print "URI: $uri\nHTTP RESPONSE: $status\n"; + print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG"); + } + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n", + ); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + ### Collect widget IDs + my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS); + my $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); } + my %csv_content; + + ### Collect widget data and write to CSV + for my $widget (@{$data}) { + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = $request->get($uri); + print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG"); + _check_return($response->{'status'}, $response->{'content'}, $uri); + $csv_content{$widget->{'name'}} = $response->{'content'}; + } + + ### Process domains from CSV + my @blocklist; + for my $widget_name (keys %csv_content) { + my $count = 0; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh_in, "<", \$csv_content{$widget_name}; + my $header = $csv->getline($fh_in); + + print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE; + $EMAIL_CSV .= "$widget_name\n"; + DOMAIN: + while (my $row = $csv->getline($fh_in)) { + last DOMAIN if ($count == $MAX_DOMAIN); + if ($row->[4] < $USER_COUNT) { + print "$row->[1]," if ($LOGMODE ne "DEBUG"); + print "$row->[0] - $row->[1] - $row->[2], $row->[3], $row->[4]\n" if ($LOGMODE eq "DEBUG"); + $EMAIL_CSV .= "$row->[1],"; + push @blocklist, $row->[1]; + $count++; + } + } + print "\n\n" if $LOGMODE; + $EMAIL_CSV .= "\n"; + } + return @blocklist; +} + +### Zscaler ### + +sub zscaler { + my @domains = @{$_[0]}; + + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + ### Push Domains + $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories"; + my $method = defined($id) ? "put" : "post"; + my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); + $#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains; + $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); + $response = $request->$method($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); +} + +say "Running in $LOGMODE mode..." if $LOGMODE; +my @domains = netskope(); +zscaler(\@domains); +mail_csv(); +say "Completed." if $LOGMODE; diff --git a/Netskope_ZScalerImporter-06.pl b/Netskope_ZScalerImporter-06.pl new file mode 100755 index 0000000..b197625 --- /dev/null +++ b/Netskope_ZScalerImporter-06.pl @@ -0,0 +1,190 @@ +#!/usr/bin/perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615 +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.024; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use MIME::Lite; + +my $LOGMODE = "VERBOSE"; +my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"; +my $config = Config::Tiny->read($CONFIG_FILE, 'utf8'); +my $USER_COUNT = $config->{report}{USER_COUNT}; +my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN}; +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); +my $EMAIL_CSV = ""; + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE; +} + +sub _check_return { + my ($status, $content, $uri) = @_; + if ($status =~ /^2/ && $LOGMODE) { + print "URI: $uri\nHTTP RESPONSE: $status\n"; + print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG"); + } + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n", + ); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + ### Collect widget IDs + my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS); + my $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); } + my %csv_content; + + ### Collect widget data and write to CSV + for my $widget (@{$data}) { + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = $request->get($uri); + print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG"); + _check_return($response->{'status'}, $response->{'content'}, $uri); + $csv_content{$widget->{'name'}} = $response->{'content'}; + } + + ### Process domains from CSV + my @blocklist; + for my $widget_name (keys %csv_content) { + my $count = 0; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh_in, "<", \$csv_content{$widget_name}; + $csv->column_names($csv->getline($fh_in)); + # "Application","Domain","Category","CCI","Users" + + print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE; + $EMAIL_CSV .= "$widget_name\n"; + DOMAIN: + while (my $row = $csv->getline_hr($fh_in)) { + last DOMAIN if ($count == $MAX_DOMAIN); + if ($row->{'Users'} < $USER_COUNT) { + print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG"); + print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG"); + $EMAIL_CSV .= "$row->{'Domain'},"; + push @blocklist, $row->{'Domain'}; + $count++; + } + } + print "\n\n" if $LOGMODE; + $EMAIL_CSV .= "\n"; + } + return @blocklist; +} + +### Zscaler ### + +sub zscaler { + my @domains = @{$_[0]}; + + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + ### Push Domains + $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories"; + my $method = defined($id) ? "put" : "post"; + my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); + $#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains; + $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); + $response = $request->$method($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); +} + +say "Running in $LOGMODE mode..." if $LOGMODE; +my @domains = netskope(); +zscaler(\@domains); +mail_csv(); +say "Completed." if $LOGMODE; diff --git a/Netskope_ZScalerImporter-07.pl b/Netskope_ZScalerImporter-07.pl new file mode 100755 index 0000000..60f318d --- /dev/null +++ b/Netskope_ZScalerImporter-07.pl @@ -0,0 +1,190 @@ +#!/usr/bin/env perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615 +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.024; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use MIME::Lite; + +my $LOGMODE = ""; +my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf"); +my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8'); +my $USER_COUNT = $config->{report}{USER_COUNT}; +my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN}; +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); +my $EMAIL_CSV = ""; + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE; +} + +sub _check_return { + my ($status, $content, $uri) = @_; + if ($status =~ /^2/ && $LOGMODE) { + print "URI: $uri\nHTTP RESPONSE: $status\n"; + print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG"); + } + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n", + ); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + ### Collect widget IDs + my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS); + my $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); } + my %csv_content; + + ### Collect widget data and write to CSV + for my $widget (@{$data}) { + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = $request->get($uri); + print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG"); + _check_return($response->{'status'}, $response->{'content'}, $uri); + $csv_content{$widget->{'name'}} = $response->{'content'}; + } + + ### Process domains from CSV + my @blocklist; + for my $widget_name (keys %csv_content) { + my $count = 0; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh_in, "<", \$csv_content{$widget_name}; + $csv->column_names($csv->getline($fh_in)); + # "Application","Domain","Category","CCI","Users" + + print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE; + $EMAIL_CSV .= "$widget_name\n"; + DOMAIN: + while (my $row = $csv->getline_hr($fh_in)) { + last DOMAIN if ($count == $MAX_DOMAIN); + if ($row->{'Users'} < $USER_COUNT) { + print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG"); + print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG"); + $EMAIL_CSV .= "$row->{'Domain'},"; + push @blocklist, $row->{'Domain'}; + $count++; + } + } + print "\n\n" if $LOGMODE; + $EMAIL_CSV .= "\n"; + } + return @blocklist; +} + +### Zscaler ### + +sub zscaler { + my @domains = @{$_[0]}; + + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + ### Push Domains + $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories"; + my $method = defined($id) ? "put" : "post"; + my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); + splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS; + $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); + $response = $request->$method($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); +} + +say "Running in $LOGMODE mode..." if $LOGMODE; +my @domains = netskope(); +zscaler(\@domains); +mail_csv(); +say "Completed." if $LOGMODE; diff --git a/Netskope_ZScalerImporter-08.pl b/Netskope_ZScalerImporter-08.pl new file mode 100755 index 0000000..8b8531e --- /dev/null +++ b/Netskope_ZScalerImporter-08.pl @@ -0,0 +1,199 @@ +#!/usr/bin/env perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615 +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.016; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use MIME::Lite; + +my $LOGMODE = "DEBUG"; +my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf"); +my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8'); +my $USER_COUNT = $config->{report}{USER_COUNT}; +my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN}; +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); +my $EMAIL_CSV = ""; + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv'); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE; +} + +sub _check_return { + my ($status, $content, $uri) = @_; + if ($status =~ /^2/ && $LOGMODE) { + print "URI: $uri\nHTTP RESPONSE: $status\n"; + print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG"); + } + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n", + ); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + ### Collect widget IDs + my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY); + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS); + my $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); } + my %csv_content; + + ### Collect widget data and write to CSV + for my $widget (@{$data}) { + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = $request->get($uri); + print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG"); + _check_return($response->{'status'}, $response->{'content'}, $uri); + $csv_content{$widget->{'name'}} = $response->{'content'}; + } + + ### Process domains from CSV + my @blocklist; + for my $widget_name (keys %csv_content) { + my $count = 0; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh_in, "<", \$csv_content{$widget_name}; + $csv->column_names($csv->getline($fh_in)); + # "Application","Domain","Category","CCI","Users" + + print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE; + $EMAIL_CSV .= "$widget_name\n"; + DOMAIN: + while (my $row = $csv->getline_hr($fh_in)) { + last DOMAIN if ($count == $MAX_DOMAIN); + if ($row->{'Users'} < $USER_COUNT) { + print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG"); + print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG"); + if ($row->{'Domain'} =~ /,/) { + push @blocklist, split (/,/, $row->{'Domain'}); + } else { + push @blocklist, $row->{'Domain'}; + } + $EMAIL_CSV .= "$row->{'Domain'},"; + $count++; + } + } + print "\n\n" if $LOGMODE; + $EMAIL_CSV .= "\n"; + } + return @blocklist; +} + +### Zscaler ### + +sub zscaler { + my @domains = @{$_[0]}; + + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar); + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + ### Push Domains + $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories"; + my $method = defined($id) ? "put" : "post"; + my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); + splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS; + $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); +# DEBUG + print "$body\n"; + + $response = $request->$method($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); +} + +say "Running in $LOGMODE mode..." if $LOGMODE; +my @domains = netskope(); +zscaler(\@domains); +mail_csv(); +say "Completed." if $LOGMODE; diff --git a/Netskope_ZScalerImporter-09.pl b/Netskope_ZScalerImporter-09.pl new file mode 100755 index 0000000..3a31849 --- /dev/null +++ b/Netskope_ZScalerImporter-09.pl @@ -0,0 +1,198 @@ +#!/usr/bin/env perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl +# Version 3.0 - 20200615 - rewrite to Perl +# Version 3.1 - 20200812 - split domains when comma separated in CSV +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.016; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use MIME::Lite; + +my $LOGMODE = "DEBUG"; +my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf"); +my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8'); +my $USER_COUNT = $config->{report}{USER_COUNT}; +my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN}; +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); +my $EMAIL_CSV = ""; + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv'); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE; +} + +sub _check_return { + my ($status, $content, $uri) = @_; + if ($status =~ /^2/ && $LOGMODE) { + print "URI: $uri\nHTTP RESPONSE: $status\n"; + print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG"); + } + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n", + ); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + ### Collect widget IDs + my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY); + my $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); } + my %csv_content; + + ### Collect widget data and write to CSV + for my $widget (@{$data}) { + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = $request->get($uri); + print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG"); + _check_return($response->{'status'}, $response->{'content'}, $uri); + $csv_content{$widget->{'name'}} = $response->{'content'}; + } + + ### Process domains from CSV + my @blocklist; + for my $widget_name (keys %csv_content) { + my $count = 0; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh_in, "<", \$csv_content{$widget_name}; + $csv->column_names($csv->getline($fh_in)); + # "Application","Domain","Category","CCI","Users" + + print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE; + $EMAIL_CSV .= "$widget_name\n"; + DOMAIN: + while (my $row = $csv->getline_hr($fh_in)) { + last DOMAIN if ($count == $MAX_DOMAIN); + if ($row->{'Users'} < $USER_COUNT) { + print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG"); + print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG"); + if ($row->{'Domain'} =~ /,/) { + push @blocklist, split (/,/, $row->{'Domain'}); + } else { + push @blocklist, $row->{'Domain'}; + } + $EMAIL_CSV .= "$row->{'Domain'},"; + $count++; + } + } + print "\n\n" if $LOGMODE; + $EMAIL_CSV .= "\n"; + } + return @blocklist; +} + +### Zscaler ### + +sub zscaler { + my @domains = @{$_[0]}; + + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + ### Push Domains + $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories"; + my $method = defined($id) ? "put" : "post"; + my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); + splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS; + $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); + print "$body\n" if ($LOGMODE eq "DEBUG"); + + $response = $request->$method($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); +} + +say "Running in $LOGMODE mode..." if $LOGMODE; +my @domains = netskope(); +zscaler(\@domains); +mail_csv(); +say "Completed." if $LOGMODE; diff --git a/Netskope_ZScalerImporter-10.pl b/Netskope_ZScalerImporter-10.pl new file mode 100755 index 0000000..9370930 --- /dev/null +++ b/Netskope_ZScalerImporter-10.pl @@ -0,0 +1,207 @@ +#!/usr/bin/env perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl +# Version 3.0 - 20200615 - rewrite to Perl +# Version 3.1 - 20200812 - split domains when comma separated in CSV +# Version 3.2 - 20200826 - added all fields to CSV export +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.016; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use MIME::Lite; + +my $LOGMODE = "DEBUG"; +my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf"); +my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8'); +my $USER_COUNT = $config->{report}{USER_COUNT}; +my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN}; +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); +my $EMAIL_CSV = ""; + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv'); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE; +} + +sub _check_return { + my ($status, $content, $uri) = @_; + if ($status =~ /^2/ && $LOGMODE) { + print "URI: $uri\nHTTP RESPONSE: $status\n"; + print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG"); + } + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n", + ); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + ### Collect widget IDs + my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY); + my $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); } + my %csv_content; + + ### Collect widget data and write to CSV + for my $widget (@{$data}) { + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = $request->get($uri); + print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG"); + _check_return($response->{'status'}, $response->{'content'}, $uri); + $csv_content{$widget->{'name'}} = $response->{'content'}; + } + + ### Process domains from CSV + my @blocklist; + for my $widget_name (keys %csv_content) { + my $count = 0; + my $domain; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh_in, "<", \$csv_content{$widget_name}; + $csv->column_names($csv->getline($fh_in)); + # "Application","Domain","Category","CCI","Users" + + print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE; + $EMAIL_CSV .= "$widget_name\n"; + $EMAIL_CSV .= "Application,Domain,Category,CCI,Users\n"; + + DOMAIN: + while (my $row = $csv->getline_hr($fh_in)) { + last DOMAIN if ($count == $MAX_DOMAIN); + if ($row->{'Users'} < $USER_COUNT) { + print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG"); + print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG"); + if ($row->{'Domain'} =~ /,/) { + push @blocklist, split (/,/, $row->{'Domain'}); + $domain = $row->{'Domain'} =~ s/,/ /gr; + } else { + push @blocklist, $row->{'Domain'}; + $domain = $row->{'Domain'}; + + } + #$EMAIL_CSV .= "$row->{'Domain'},"; + #$EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Users'}\n"; + $EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Users'}\n"; + $count++; + } + } + print "\n\n" if $LOGMODE; + $EMAIL_CSV .= "\n"; + } + return @blocklist; +} + +### Zscaler ### + +sub zscaler { + my @domains = @{$_[0]}; + + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + ### Push Domains + $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories"; + my $method = defined($id) ? "put" : "post"; + my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); + splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS; + $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); + print "$body\n" if ($LOGMODE eq "DEBUG"); + + $response = $request->$method($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); +} + +say "Running in $LOGMODE mode..." if $LOGMODE; +my @domains = netskope(); +#zscaler(\@domains); +mail_csv(); +say "Completed." if $LOGMODE; diff --git a/Netskope_ZScalerImporter-11.pl b/Netskope_ZScalerImporter-11.pl new file mode 100755 index 0000000..a2a7a5d --- /dev/null +++ b/Netskope_ZScalerImporter-11.pl @@ -0,0 +1,205 @@ +#!/usr/bin/env perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl +# Version 3.0 - 20200615 - rewrite to Perl +# Version 3.1 - 20200812 - split domains when comma separated in CSV +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.016; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use MIME::Lite; + +my $LOGMODE = ""; +my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf"); +my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8'); +my $USER_COUNT = $config->{report}{USER_COUNT}; +my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN}; +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); +my $EMAIL_CSV = ""; + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv'); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE; +} + +sub _check_return { + my ($status, $content, $uri) = @_; + if ($status =~ /^2/ && $LOGMODE) { + print "URI: $uri\nHTTP RESPONSE: $status\n"; + print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG"); + } + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n", + ); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + ### Collect widget IDs + my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY); + my $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); } + my %csv_content; + + ### Collect widget data and write to CSV + for my $widget (@{$data}) { + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = $request->get($uri); + print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG"); + _check_return($response->{'status'}, $response->{'content'}, $uri); + $csv_content{$widget->{'name'}} = $response->{'content'}; + } + + ### Process domains from CSV + my @blocklist; + for my $widget_name (keys %csv_content) { + my $count = 0; + my $domain; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh_in, "<", \$csv_content{$widget_name}; + my @headers = $csv->column_names($csv->getline($fh_in)); + print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE; + + print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE; + $EMAIL_CSV .= "$widget_name\n"; + $EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n"; + + DOMAIN: + while (my $row = $csv->getline_hr($fh_in)) { + last DOMAIN if ($count == $MAX_DOMAIN); + next DOMAIN if ($row->{'Blocked Events'} > 0); + if ($row->{'Users'} < $USER_COUNT) { + print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG"); + print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG"); + if ($row->{'Domain'} =~ /,/) { + push @blocklist, split (/,/, $row->{'Domain'}); + $domain = $row->{'Domain'} =~ s/,/ /gr; + } else { + push @blocklist, $row->{'Domain'}; + $domain = $row->{'Domain'}; + + } + $EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n"; + $count++; + } + } + print "\n\n" if $LOGMODE; + $EMAIL_CSV .= "\n"; + } + return @blocklist; +} + +### Zscaler ### + +sub zscaler { + my @domains = @{$_[0]}; + + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + ### Push Domains + $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories"; + my $method = defined($id) ? "put" : "post"; + my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); + splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS; + $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); + print "$body\n" if ($LOGMODE eq "DEBUG"); + + $response = $request->$method($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); +} + +say "Running in $LOGMODE mode..." if $LOGMODE; +my @domains = netskope(); +#zscaler(\@domains); +mail_csv(); +say "Completed." if $LOGMODE; diff --git a/Netskope_ZScalerImporter-12.pl b/Netskope_ZScalerImporter-12.pl new file mode 100755 index 0000000..6ff7457 --- /dev/null +++ b/Netskope_ZScalerImporter-12.pl @@ -0,0 +1,276 @@ +#!/usr/bin/env perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl +# Version 3.0 - 20200615 - rewrite to Perl +# Version 3.1 - 20200812 - split domains when comma separated in CSV +# Version 3.2 - 20200909 - de-duplication of Zscaler URL category +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.016; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use MIME::Lite; + +my $LOGMODE = "DEBUG"; +my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf"); +my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8'); +my $USER_COUNT = $config->{report}{USER_COUNT}; +my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN}; +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); +my $EMAIL_CSV = ""; + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv'); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE; +} + +sub _check_return { + my ($status, $content, $uri) = @_; + if ($status =~ /^2/ && $LOGMODE) { + print "URI: $uri\nHTTP RESPONSE: $status\n"; + print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG"); + } + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n", + ); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + my @existing_domains = @{$_[0]}; + + ### Collect widget IDs + my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY); + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS); + my $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); } + my %csv_content; + + ### Collect widget data and write to CSV + for my $widget (@{$data}) { + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = $request->get($uri); + print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG"); + _check_return($response->{'status'}, $response->{'content'}, $uri); + $csv_content{$widget->{'name'}} = $response->{'content'}; + } + + ### Process domains from CSV + my @blocklist; + for my $widget_name (keys %csv_content) { + my $count = 0; + my $domain; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh_in, "<", \$csv_content{$widget_name}; + my @headers = $csv->column_names($csv->getline($fh_in)); + print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE; + + print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE; + $EMAIL_CSV .= "$widget_name\n"; + $EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n"; + + DOMAIN: while (my $row = $csv->getline_hr($fh_in)) { + last DOMAIN if ($count == $MAX_DOMAIN); + #next DOMAIN if ($row->{'Blocked Events'} > 0); + if ($row->{'Users'} < $USER_COUNT) { + print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG"); + print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG"); + if ($row->{'Domain'} =~ /,/) { + PARSE: + for my $item (split (/,/, $row->{'Domain'})) { + next PARSE if (grep(/$item/, @existing_domains)); + push @blocklist, $item; + $domain .= $item . " "; + } + if ($domain) { + $EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n"; + $count++; + } + } else { + next DOMAIN if (!grep(/$row->{'Domain'}/, @existing_domains)); + push @blocklist, $row->{'Domain'}; + $EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n"; + $count++; + } + } + } + print "\n\n" if $LOGMODE; + $EMAIL_CSV .= "\n"; + print "COUNT: $count\n"; + } + return @blocklist; +} + +### Zscaler ### + +sub zscaler_get { + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar); + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + $uri = "$ZS_BASE_URI/urlCategories/$id"; + my $method = "get"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'urls'}; + my @convert = (); + for my $item (@{$data}) { + push @convert, $item; + } + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + return @convert; +} + + + +sub zscaler_push { + my @domains = @{$_[0]}; + + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar); + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + ### Push Domains + $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories"; + my $method = defined($id) ? "put" : "post"; + my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); + splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS; + $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); + print "$body\n" if ($LOGMODE eq "DEBUG"); + + $response = $request->$method($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); +} + +say "Running in $LOGMODE mode..." if $LOGMODE; +my @existing_domains = zscaler_get(); +my @domains = netskope(\@existing_domains); +print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE; +zscaler_push(\@domains); +mail_csv(); +say "Completed." if $LOGMODE; diff --git a/Netskope_ZScalerImporter-13.pl b/Netskope_ZScalerImporter-13.pl new file mode 100755 index 0000000..f758c2c --- /dev/null +++ b/Netskope_ZScalerImporter-13.pl @@ -0,0 +1,277 @@ +#!/usr/bin/env perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl +# Version 3.0 - 20200615 - rewrite to Perl +# Version 3.1 - 20200812 - split domains when comma separated in CSV +# Version 3.2 - 20200909 - de-duplication of Zscaler URL category +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.016; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use MIME::Lite; + +my $LOGMODE = "DEBUG"; +my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf"); +my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8'); +my $USER_COUNT = $config->{report}{USER_COUNT}; +my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN}; +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); +my $EMAIL_CSV = ""; + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv'); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE; +} + +sub _check_return { + my ($status, $content, $uri) = @_; + if ($status =~ /^2/ && $LOGMODE) { + print "URI: $uri\nHTTP RESPONSE: $status\n"; + print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG"); + } + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n", + ); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + my @existing_domains = @{$_[0]}; + + ### Collect widget IDs + my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY); + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS); + my $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); } + my %csv_content; + + ### Collect widget data and write to CSV + for my $widget (@{$data}) { + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = $request->get($uri); + print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG"); + _check_return($response->{'status'}, $response->{'content'}, $uri); + $csv_content{$widget->{'name'}} = $response->{'content'}; + } + + ### Process domains from CSV + my @blocklist; + for my $widget_name (keys %csv_content) { + my $count = 0; + my $domain; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh_in, "<", \$csv_content{$widget_name}; + my @headers = $csv->column_names($csv->getline($fh_in)); + print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE; + + print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE; + $EMAIL_CSV .= "$widget_name\n"; + $EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n"; + + DOMAIN: while (my $row = $csv->getline_hr($fh_in)) { + last DOMAIN if ($count == $MAX_DOMAIN); + #next DOMAIN if ($row->{'Blocked Events'} > 0); + if ($row->{'Users'} < $USER_COUNT) { + print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG"); + print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG"); + next DOMAIN if ($row->{'Domain'} =~ "n/a"); + if ($row->{'Domain'} =~ /,/) { + PARSE: + for my $item (split (/,/, $row->{'Domain'})) { + next PARSE if (grep(/$item/, @existing_domains)); + push @blocklist, $item; + $domain .= $item . " "; + } + if ($domain) { + $EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n"; + $count++; + } + } else { + next DOMAIN if (!grep(/$row->{'Domain'}/, @existing_domains)); + push @blocklist, $row->{'Domain'}; + $EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n"; + $count++; + } + } + } + print "\n\n" if $LOGMODE; + $EMAIL_CSV .= "\n"; + print "COUNT: $count\n"; + } + return @blocklist; +} + +### Zscaler ### + +sub zscaler_get { + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar); + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + $uri = "$ZS_BASE_URI/urlCategories/$id"; + my $method = "get"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'urls'}; + my @convert = (); + for my $item (@{$data}) { + push @convert, $item; + } + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + return @convert; +} + + + +sub zscaler_push { + my @domains = @{$_[0]}; + + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar); + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + ### Push Domains + $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories"; + my $method = defined($id) ? "put" : "post"; + my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); + splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS; + $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); + print "$body\n" if ($LOGMODE eq "DEBUG"); + + $response = $request->$method($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); +} + +say "Running in $LOGMODE mode..." if $LOGMODE; +my @existing_domains = zscaler_get(); +my @domains = netskope(\@existing_domains); +print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE; +zscaler_push(\@domains); +mail_csv(); +say "Completed." if $LOGMODE; diff --git a/Netskope_ZScalerImporter-14.pl b/Netskope_ZScalerImporter-14.pl new file mode 100755 index 0000000..1e83260 --- /dev/null +++ b/Netskope_ZScalerImporter-14.pl @@ -0,0 +1,280 @@ +#!/usr/bin/env perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl +# Version 3.0 - 20200615 - rewrite to Perl +# Version 3.1 - 20200812 - split domains when comma separated in CSV +# Version 3.2 - 20200909 - de-duplication of Zscaler URL category +# Version 3.3 - 20210121 - filter our entries when domain "n/a" +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.016; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use MIME::Lite; + +my $LOGMODE = "DEBUG"; +#my $LOGMODE = ""; +my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf"); +my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8'); +my $USER_COUNT = $config->{report}{USER_COUNT}; +my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN}; +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); +my $EMAIL_CSV = ""; + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv'); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE; +} + +sub _check_return { + my ($status, $content, $uri) = @_; + if ($status =~ /^2/ && $LOGMODE) { + print "URI: $uri\nHTTP RESPONSE: $status\n"; + print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG"); + } + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n", + ); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + my @existing_domains = @{$_[0]}; + + ### Collect widget IDs + my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY); + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS); + my $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); } + my %csv_content; + + ### Collect widget data and write to CSV + for my $widget (@{$data}) { + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = $request->get($uri); + print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG"); + _check_return($response->{'status'}, $response->{'content'}, $uri); + $csv_content{$widget->{'name'}} = $response->{'content'}; + } + + ### Process domains from CSV + my @blocklist; + for my $widget_name (keys %csv_content) { + my $count = 0; + my $domain; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh_in, "<", \$csv_content{$widget_name}; + my @headers = $csv->column_names($csv->getline($fh_in)); + print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE; + + print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE; + $EMAIL_CSV .= "$widget_name\n"; + $EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n"; + + DOMAIN: while (my $row = $csv->getline_hr($fh_in)) { + last DOMAIN if ($count == $MAX_DOMAIN); + next DOMAIN if ($row->{'Blocked Events'} > 0); + if ($row->{'Users'} < $USER_COUNT) { + print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG"); + print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG"); + next DOMAIN if ($row->{'Domain'} =~ "n/a"); + if ($row->{'Domain'} =~ /,/) { + PARSE: + for my $item (split (/,/, $row->{'Domain'})) { + next PARSE if (grep(/$item/, @existing_domains)); + push @blocklist, $item; + $domain .= $item . " "; + } + if ($domain) { + $EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n"; + $count++; + } + } else { + next DOMAIN if (grep(/$row->{'Domain'}/, @existing_domains)); + push @blocklist, $row->{'Domain'}; + $EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n"; + $count++; + } + } + } + print "\n\n" if $LOGMODE; + $EMAIL_CSV .= "\n"; + print "COUNT: $count\n"; + } + return @blocklist; +} + +### Zscaler ### + +sub zscaler_get { + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar); + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + $uri = "$ZS_BASE_URI/urlCategories/$id"; + my $method = "get"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'urls'}; + my @convert = (); + for my $item (@{$data}) { + push @convert, $item; + } + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + return @convert; +} + + + +sub zscaler_push { + my @domains = @{$_[0]}; + + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar); + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + ### Push Domains + $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories?action=ADD_TO_LIST"; + my $method = defined($id) ? "put" : "post"; + my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); + splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS; + $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); + print "$body\n" if ($LOGMODE eq "DEBUG"); + + $response = $request->$method($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); +} + +say "Running in $LOGMODE mode..." if $LOGMODE; +my @existing_domains = zscaler_get(); +my @domains = netskope(\@existing_domains); +print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE; +zscaler_push(\@domains); +mail_csv(); +say "Completed." if $LOGMODE; diff --git a/Netskope_ZScalerImporter-wip.pl b/Netskope_ZScalerImporter-wip.pl new file mode 100755 index 0000000..2f3b9a8 --- /dev/null +++ b/Netskope_ZScalerImporter-wip.pl @@ -0,0 +1,276 @@ +#!/usr/bin/env perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl +# Version 3.0 - 20200615 - rewrite to Perl +# Version 3.1 - 20200812 - split domains when comma separated in CSV +# Version 3.2 - 20200909 - de-duplication of Zscaler URL category +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.016; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use MIME::Lite; + +my $LOGMODE = "DEBUG"; +my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf"); +my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8'); +my $USER_COUNT = $config->{report}{USER_COUNT}; +my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN}; +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); +my $EMAIL_CSV = ""; + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv'); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE; +} + +sub _check_return { + my ($status, $content, $uri) = @_; + if ($status =~ /^2/ && $LOGMODE) { + print "URI: $uri\nHTTP RESPONSE: $status\n"; + print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG"); + } + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n", + ); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + my @existing_domains = @{$_[0]}; + + ### Collect widget IDs + my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY); + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS); + my $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); } + my %csv_content; + + ### Collect widget data and write to CSV + for my $widget (@{$data}) { + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = $request->get($uri); + print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG"); + _check_return($response->{'status'}, $response->{'content'}, $uri); + $csv_content{$widget->{'name'}} = $response->{'content'}; + } + + ### Process domains from CSV + my @blocklist; + for my $widget_name (keys %csv_content) { + my $count = 0; + my $domain; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh_in, "<", \$csv_content{$widget_name}; + my @headers = $csv->column_names($csv->getline($fh_in)); + print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE; + + print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE; + $EMAIL_CSV .= "$widget_name\n"; + $EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n"; + + DOMAIN: while (my $row = $csv->getline_hr($fh_in)) { + last DOMAIN if ($count == $MAX_DOMAIN); + #next DOMAIN if ($row->{'Blocked Events'} > 0); + if ($row->{'Users'} < $USER_COUNT) { + print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG"); + print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG"); + if ($row->{'Domain'} =~ /,/) { + PARSE: + for my $item (split (/,/, $row->{'Domain'})) { + next PARSE if (grep(/$item/, @existing_domains)); + push @blocklist, $item; + $domain .= $item . " "; + } + if ($domain) { + $EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n"; + $count++; + } + } else { + next DOMAIN if (grep(/$row->{'Domain'}/, @existing_domains)); + push @blocklist, $row->{'Domain'}; + $EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n"; + $count++; + } + } + } + print "\n\n" if $LOGMODE; + $EMAIL_CSV .= "\n"; + print "COUNT: $count\n"; + } + return @blocklist; +} + +### Zscaler ### + +sub zscaler_get { + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar); + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + $uri = "$ZS_BASE_URI/urlCategories/$id"; + my $method = "get"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'urls'}; + my @convert = (); + for my $item (@{$data}) { + push @convert, $item; + } + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + return @convert; +} + + + +sub zscaler_push { + my @domains = @{$_[0]}; + + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar); + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + ### Push Domains + $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories"; + my $method = defined($id) ? "put" : "post"; + my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); + splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS; + $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); + print "$body\n" if ($LOGMODE eq "DEBUG"); + + $response = $request->$method($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); +} + +say "Running in $LOGMODE mode..." if $LOGMODE; +my @existing_domains = zscaler_get(); +my @domains = netskope(\@existing_domains); +print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE; +zscaler_push(\@domains); +mail_csv(); +say "Completed." if $LOGMODE; diff --git a/httpstat.py b/httpstat.py new file mode 100755 index 0000000..df4f396 --- /dev/null +++ b/httpstat.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python +# coding: utf-8 +# References: +# man curl +# https://curl.haxx.se/libcurl/c/curl_easy_getinfo.html +# https://curl.haxx.se/libcurl/c/easy_getinfo_options.html +# http://blog.kenweiner.com/2014/11/http-request-timings-with-curl.html + +from __future__ import print_function + +import os +import json +import sys +import logging +import tempfile +import subprocess + + +__version__ = '1.2.1' + + +PY3 = sys.version_info >= (3,) + +if PY3: + xrange = range + + +# Env class is copied from https://github.com/reorx/getenv/blob/master/getenv.py +class Env(object): + prefix = 'HTTPSTAT' + _instances = [] + + def __init__(self, key): + self.key = key.format(prefix=self.prefix) + Env._instances.append(self) + + def get(self, default=None): + return os.environ.get(self.key, default) + + +ENV_SHOW_BODY = Env('{prefix}_SHOW_BODY') +ENV_SHOW_IP = Env('{prefix}_SHOW_IP') +ENV_SHOW_SPEED = Env('{prefix}_SHOW_SPEED') +ENV_SAVE_BODY = Env('{prefix}_SAVE_BODY') +ENV_CURL_BIN = Env('{prefix}_CURL_BIN') +ENV_DEBUG = Env('{prefix}_DEBUG') + + +curl_format = """{ +"time_namelookup": %{time_namelookup}, +"time_connect": %{time_connect}, +"time_appconnect": %{time_appconnect}, +"time_pretransfer": %{time_pretransfer}, +"time_redirect": %{time_redirect}, +"time_starttransfer": %{time_starttransfer}, +"time_total": %{time_total}, +"speed_download": %{speed_download}, +"speed_upload": %{speed_upload}, +"remote_ip": "%{remote_ip}", +"remote_port": "%{remote_port}", +"local_ip": "%{local_ip}", +"local_port": "%{local_port}" +}""" + +https_template = """ + DNS Lookup TCP Connection TLS Handshake Server Processing Content Transfer +[ {a0000} | {a0001} | {a0002} | {a0003} | {a0004} ] + | | | | | + namelookup:{b0000} | | | | + connect:{b0001} | | | + pretransfer:{b0002} | | + starttransfer:{b0003} | + total:{b0004} +"""[1:] + +http_template = """ + DNS Lookup TCP Connection Server Processing Content Transfer +[ {a0000} | {a0001} | {a0003} | {a0004} ] + | | | | + namelookup:{b0000} | | | + connect:{b0001} | | + starttransfer:{b0003} | + total:{b0004} +"""[1:] + + +# Color code is copied from https://github.com/reorx/python-terminal-color/blob/master/color_simple.py +ISATTY = sys.stdout.isatty() + + +def make_color(code): + def color_func(s): + if not ISATTY: + return s + tpl = '\x1b[{}m{}\x1b[0m' + return tpl.format(code, s) + return color_func + + +red = make_color(31) +green = make_color(32) +yellow = make_color(33) +blue = make_color(34) +magenta = make_color(35) +cyan = make_color(36) + +bold = make_color(1) +underline = make_color(4) + +grayscale = {(i - 232): make_color('38;5;' + str(i)) for i in xrange(232, 256)} + + +def quit(s, code=0): + if s is not None: + print(s) + sys.exit(code) + + +def print_help(): + help = """ +Usage: httpstat URL [CURL_OPTIONS] + httpstat -h | --help + httpstat --version + +Arguments: + URL url to request, could be with or without `http(s)://` prefix + +Options: + CURL_OPTIONS any curl supported options, except for -w -D -o -S -s, + which are already used internally. + -h --help show this screen. + --version show version. + +Environments: + HTTPSTAT_SHOW_BODY Set to `true` to show response body in the output, + note that body length is limited to 1023 bytes, will be + truncated if exceeds. Default is `false`. + HTTPSTAT_SHOW_IP By default httpstat shows remote and local IP/port address. + Set to `false` to disable this feature. Default is `true`. + HTTPSTAT_SHOW_SPEED Set to `true` to show download and upload speed. + Default is `false`. + HTTPSTAT_SAVE_BODY By default httpstat stores body in a tmp file, + set to `false` to disable this feature. Default is `true` + HTTPSTAT_CURL_BIN Indicate the curl bin path to use. Default is `curl` + from current shell $PATH. + HTTPSTAT_DEBUG Set to `true` to see debugging logs. Default is `false` +"""[1:-1] + print(help) + + +def main(): + args = sys.argv[1:] + if not args: + print_help() + quit(None, 0) + + # get envs + show_body = 'true' in ENV_SHOW_BODY.get('false').lower() + show_ip = 'true' in ENV_SHOW_IP.get('true').lower() + show_speed = 'true'in ENV_SHOW_SPEED.get('false').lower() + save_body = 'true' in ENV_SAVE_BODY.get('true').lower() + curl_bin = ENV_CURL_BIN.get('curl') + is_debug = 'true' in ENV_DEBUG.get('false').lower() + + # configure logging + if is_debug: + log_level = logging.DEBUG + else: + log_level = logging.INFO + logging.basicConfig(level=log_level) + lg = logging.getLogger('httpstat') + + # log envs + lg.debug('Envs:\n%s', '\n'.join(' {}={}'.format(i.key, i.get('')) for i in Env._instances)) + lg.debug('Flags: %s', dict( + show_body=show_body, + show_ip=show_ip, + show_speed=show_speed, + save_body=save_body, + curl_bin=curl_bin, + is_debug=is_debug, + )) + + # get url + url = args[0] + if url in ['-h', '--help']: + print_help() + quit(None, 0) + elif url == '--version': + print('httpstat {}'.format(__version__)) + quit(None, 0) + + curl_args = args[1:] + + # check curl args + exclude_options = [ + '-w', '--write-out', + '-D', '--dump-header', + '-o', '--output', + '-s', '--silent', + ] + for i in exclude_options: + if i in curl_args: + quit(yellow('Error: {} is not allowed in extra curl args'.format(i)), 1) + + # tempfile for output + bodyf = tempfile.NamedTemporaryFile(delete=False) + bodyf.close() + + headerf = tempfile.NamedTemporaryFile(delete=False) + headerf.close() + + # run cmd + cmd_env = os.environ.copy() + cmd_env.update( + LC_ALL='C', + ) + cmd_core = [curl_bin, '-w', curl_format, '-D', headerf.name, '-o', bodyf.name, '-s', '-S'] + cmd = cmd_core + curl_args + [url] + lg.debug('cmd: %s', cmd) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=cmd_env) + out, err = p.communicate() + if PY3: + out, err = out.decode(), err.decode() + lg.debug('out: %s', out) + + # print stderr + if p.returncode == 0: + if err: + print(grayscale[16](err)) + else: + _cmd = list(cmd) + _cmd[2] = '' + _cmd[4] = '' + _cmd[6] = '' + print('> {}'.format(' '.join(_cmd))) + quit(yellow('curl error: {}'.format(err)), p.returncode) + + # parse output + try: + d = json.loads(out) + except ValueError as e: + print(yellow('Could not decode json: {}'.format(e))) + print('curl result:', p.returncode, grayscale[16](out), grayscale[16](err)) + quit(None, 1) + for k in d: + if k.startswith('time_'): + d[k] = int(d[k] * 1000) + + # calculate ranges + d.update( + range_dns=d['time_namelookup'], + range_connection=d['time_connect'] - d['time_namelookup'], + range_ssl=d['time_pretransfer'] - d['time_connect'], + range_server=d['time_starttransfer'] - d['time_pretransfer'], + range_transfer=d['time_total'] - d['time_starttransfer'], + ) + + # ip + if show_ip: + s = 'Connected to {}:{} from {}:{}'.format( + cyan(d['remote_ip']), cyan(d['remote_port']), + d['local_ip'], d['local_port'], + ) + print(s) + print() + + # print header & body summary + with open(headerf.name, 'r') as f: + headers = f.read().strip() + # remove header file + lg.debug('rm header file %s', headerf.name) + os.remove(headerf.name) + + for loop, line in enumerate(headers.split('\n')): + if loop == 0: + p1, p2 = tuple(line.split('/')) + print(green(p1) + grayscale[14]('/') + cyan(p2)) + else: + pos = line.find(':') + print(grayscale[14](line[:pos + 1]) + cyan(line[pos + 1:])) + + print() + + # body + if show_body: + body_limit = 1024 + with open(bodyf.name, 'r') as f: + body = f.read().strip() + body_len = len(body) + + if body_len > body_limit: + print(body[:body_limit] + cyan('...')) + print() + s = '{} is truncated ({} out of {})'.format(green('Body'), body_limit, body_len) + if save_body: + s += ', stored in: {}'.format(bodyf.name) + print(s) + else: + print(body) + else: + if save_body: + print('{} stored in: {}'.format(green('Body'), bodyf.name)) + + # remove body file + if not save_body: + lg.debug('rm body file %s', bodyf.name) + os.remove(bodyf.name) + + # print stat + if url.startswith('https://'): + template = https_template + else: + template = http_template + + # colorize template first line + tpl_parts = template.split('\n') + tpl_parts[0] = grayscale[16](tpl_parts[0]) + template = '\n'.join(tpl_parts) + + def fmta(s): + return cyan('{:^7}'.format(str(s) + 'ms')) + + def fmtb(s): + return cyan('{:<7}'.format(str(s) + 'ms')) + + stat = template.format( + # a + a0000=fmta(d['range_dns']), + a0001=fmta(d['range_connection']), + a0002=fmta(d['range_ssl']), + a0003=fmta(d['range_server']), + a0004=fmta(d['range_transfer']), + # b + b0000=fmtb(d['time_namelookup']), + b0001=fmtb(d['time_connect']), + b0002=fmtb(d['time_pretransfer']), + b0003=fmtb(d['time_starttransfer']), + b0004=fmtb(d['time_total']), + ) + print() + print(stat) + + # speed, originally bytes per second + if show_speed: + print('speed_download: {:.1f} KiB/s, speed_upload: {:.1f} KiB/s'.format( + d['speed_download'] / 1024, d['speed_upload'] / 1024)) + + +if __name__ == '__main__': + main() diff --git a/jsondump.py b/jsondump.py new file mode 100755 index 0000000..9fb047b --- /dev/null +++ b/jsondump.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import json +import urllib.request +import argparse +import collections +from operator import itemgetter + +parser = argparse.ArgumentParser(description="API Call to collect data") +parser.add_argument("tenant", type=str, help="Tenant Name") +parser.add_argument("token", type=str, help="Tenat API Token") +parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod (default: 604800)") + +try: + args = parser.parse_args() + tenant = args.tenant + token = args.token + timeperiod = args.timeperiod + +except argparse.ArgumentError as e: + print(str(e)) + +base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod) + +req = urllib.request.Request(base_url) +with urllib.request.urlopen(req) as response: + content = response.read() +json_content = json.loads(content) +print(json.dumps(json_content, indent=4, sort_keys=True)) diff --git a/measure.py b/measure.py new file mode 100755 index 0000000..62d9476 --- /dev/null +++ b/measure.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# +# Copyright 2019, Mischa Peters , Netskope. +# Version 1.0 - 20191028 +# +# Measure timing for DNS lookup as well as HTTP page load +# +# Requires: +# - Python 3.x +# +import argparse +import socket +import time +import ssl +import urllib.request +from urllib.parse import urlparse + +parser = argparse.ArgumentParser(description="Measure load times", epilog="2019 (c) Netskope") +parser.add_argument("url", type=str, help="url (eg. https://google.com)") + +try: + args = parser.parse_args() + url = args.url + +except argparse.ArgumentError as e: + print(str(e)) + +print (url, "timing:") + +urlinfo = urlparse(url) +request_headers = {'Cache-Control': 'no-cache', 'User-Agent': 'Mozilla/5.0'} +no_cert_check = ssl.create_default_context() +no_cert_check.check_hostname=False +no_cert_check.verify_mode=ssl.CERT_NONE + +start = time.time() +ip = socket.gethostbyname(urlinfo.netloc) +dns_time = time.time()-start +print ("DNS Lookup:\t{:.3f} seconds".format(dns_time)) + +start = time.time() +req = urllib.request.Request(url, headers=request_headers) +content = urllib.request.urlopen(req, context=no_cert_check).read() +load_time = time.time()-start +print ("Page Load:\t{:.3f} seconds".format(load_time)) +print ("w/o DNS Lookup:\t{:.3f} seconds".format(load_time-dns_time)) diff --git a/ns.pl b/ns.pl new file mode 100755 index 0000000..b84efc9 --- /dev/null +++ b/ns.pl @@ -0,0 +1,24 @@ +#!/usr/bin/perl -w +use 5.024; +use strict; +use warnings; +use autodie; +use HTTP::Tiny; +use Cpanel::JSON::XS; + +my $NTSKP_TENANT = "https://oss.de.goskope.com"; +my $NTSKP_TOKEN = "0cfe04c4237cc33dc7f383af5ddbe2e3"; +my $uri = "$NTSKP_TENANT/api/v1/alerts?token=$NTSKP_TOKEN&timeperiod=86400&groupby=application&query=access_method+eq+Client+and+action+eq+block"; +#my $uri = "$NTSKP_TENANT/api/v1/report?token=$NTSKP_TOKEN&timeperiod=86400&type=connection&groupby=application&query=app-cci-app-tag+eq+'Under_Review'"; +#my $uri = "$NTSKP_TENANT/api/v1/report?token=$NTSKP_TOKEN&timeperiod=86400&type=connection&groupby=application&query=app-cci-app-tag+eq+'Pending_GRC_Review'"; +my $response = HTTP::Tiny->new->get($uri); +my $json = Cpanel::JSON::XS->new->utf8->decode($response->{'content'}); +my $data = $json->{'data'}; +for my $item (@{$data}) { + if (exists($item->{'app'})) { + print "."; + #print $item->{'app'} . ", "; + #say $item->{'sessions'}; + } +} +say ""; diff --git a/ntskp-api-01.pl b/ntskp-api-01.pl new file mode 100755 index 0000000..f515bbe --- /dev/null +++ b/ntskp-api-01.pl @@ -0,0 +1,45 @@ +#!/usr/bin/perl -w +use strict; +use warnings; +use autodie; +use POSIX qw(strftime); +use Cpanel::JSON::XS; + +my $file; +{ + local $/; + open my $fh, "<", "amsjson.txt"; + $file = <$fh>; + close $fh; +} + +my $json = Cpanel::JSON::XS->new->utf8->decode($file); +my $data = $json->{'data'}; +my $domain; +my $cci; + +for (my $i = 0; $i < (@{$data}); $i++) { + #print "Timestamp: $data->[$i]->{'timestamp'}\n"; + #print "Domain: $data->[$i]->{'domain'}\n"; + + if (!$data->[$i]->{'domain'}) { + my $url = $data->[$i]->{'url'}; + $url =~ s!^https?://(?:www\.)?!!i; + $url =~ s!/.*!!; + $url =~ s/[\?\#\:].*//; + $domain = $url; + } else { + $domain = $data->[$i]->{'domain'}; + } + if ($data->[$i]->{'cci'}) { + $cci = $data->[$i]->{'cci'}; + } else { + $cci = 'none'; + } + + #print "Category: $data->[$i]->{'category'}\n"; + #print "CCI: $data->[$i]->{'ccl'}\n"; + #print "User: $data->[$i]->{'user'}\n"; + my $timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime($data->[$i]->{'timestamp'})); + print "$timestamp,$domain,$cci,$data->[$i]->{'category'},$data->[$i]->{'ccl'}\n"; +} diff --git a/ntskp-api-02.pl b/ntskp-api-02.pl new file mode 100755 index 0000000..1dce7d9 --- /dev/null +++ b/ntskp-api-02.pl @@ -0,0 +1,69 @@ +#!/usr/bin/perl -w +use strict; +use warnings; +use autodie; +use POSIX qw(strftime); +use Config::Tiny; +use HTTP::Tiny; +use Cpanel::JSON::XS; + +my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"; +my $config = Config::Tiny->read($CONFIG_FILE, 'utf8'); +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_PERIOD = $config->{netskope}{NTSKP_PERIOD}; +my $NTSKP_SCORE = $config->{netskope}{NTSKP_SCORE}; +my $NTSKP_CATEGORIES = $config->{netskope}{NTSKP_CATEGORIES}; + +my $uri; +my $skip = 0; +my $response; +my $json; +my $data; +my $length; +my $domain; +my $cci; + +my $file_out = "extracted-" . strftime("%Y%m%d", localtime) . ".txt"; +print "File: $file_out\n"; +print "Tenant: $NTSKP_TENANT\n"; + +while ($skip < 500000) { + $uri = "$NTSKP_TENANT/api/v1/events?token=$NTSKP_TOKEN&type=page&timeperiod=$NTSKP_PERIOD&skip=$skip"; + $response = HTTP::Tiny->new->get($uri); + print "HTTP: $response->{status} $response->{reason}\n"; + $json = Cpanel::JSON::XS->new->utf8->decode($response->{content}); + print "API: $json->{'status'}\n"; + $data = $json->{'data'}; + + $length = (@{$data}); + if ($length == 0) { + print "All data collected\n"; + last; + } + + open my $fh_out, ">>", $file_out; + for (my $i = 0; $i < $length; $i++) { + if (!$data->[$i]->{'domain'}) { + my $url = $data->[$i]->{'url'}; + $url =~ s!^https?://(?:www\.)?!!i; + $url =~ s!/.*!!; + $url =~ s/[\?\#\:].*//; + $domain = $url; + } else { + $domain = $data->[$i]->{'domain'}; + } + if ($data->[$i]->{'cci'}) { + $cci = $data->[$i]->{'cci'}; + } else { + $cci = 'none'; + } + + my $timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime($data->[$i]->{'timestamp'})); + print $fh_out "$timestamp,$domain,$cci,$data->[$i]->{'category'},$data->[$i]->{'ccl'},$data->[$i]->{'user'}\n"; + } + close $fh_out; + $skip += 5000; + #print "Next batch $skip\n"; +} +print "Done\n"; diff --git a/ntskp-api-03.pl b/ntskp-api-03.pl new file mode 100755 index 0000000..f0bccc6 --- /dev/null +++ b/ntskp-api-03.pl @@ -0,0 +1,21 @@ +#!/usr/bin/perl -w +use 5.024; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use HTTP::Tiny; +#use Cpanel::JSON::XS; +use JSON::PP; + +my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"; +my $config = Config::Tiny->read($CONFIG_FILE, 'utf8'); +my $NTSKP_TENANT = $config->{netskope}->{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}->{NTSKP_TOKEN}; +my $NTSKP_PERIOD = $config->{netskope}->{NTSKP_PERIOD}; + +my $uri = "$NTSKP_TENANT/api/v1/events?token=$NTSKP_TOKEN&type=page&timeperiod=$NTSKP_PERIOD"; +my $response = HTTP::Tiny->new->get($uri); +#my $json = Cpanel::JSON::XS->new->indent(1)->encode($response->{content}); +my $json = JSON::PP->new->pretty(1)->encode($response->{content}); +print $json; diff --git a/ntskp-api-04.pl b/ntskp-api-04.pl new file mode 100755 index 0000000..1812694 --- /dev/null +++ b/ntskp-api-04.pl @@ -0,0 +1,21 @@ +#!/usr/bin/perl -w +use 5.024; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use HTTP::Tiny; +#use Cpanel::JSON::XS; +use JSON::PP; + +my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"; +my $config = Config::Tiny->read($CONFIG_FILE, 'utf8'); +my $NTSKP_TENANT = $config->{netskope}->{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}->{NTSKP_TOKEN}; +my $NTSKP_PERIOD = $config->{netskope}->{NTSKP_PERIOD}; + +my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=498"; +my $response = HTTP::Tiny->new->get($uri); +#my $json = Cpanel::JSON::XS->new->indent(1)->encode($response->{content}); +my $json = JSON::PP->new->pretty(1)->encode($response->{content}); +print $json; diff --git a/ntskp-api-05.pl b/ntskp-api-05.pl new file mode 100755 index 0000000..5d5d37f --- /dev/null +++ b/ntskp-api-05.pl @@ -0,0 +1,59 @@ +#!/usr/bin/perl -w +use strict; +use warnings; +use autodie; +use POSIX qw(strftime); +use Config::Tiny; +use HTTP::Tiny; +use JSON::PP; + +my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"; +my $config = Config::Tiny->read($CONFIG_FILE, 'utf8'); +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_PERIOD = $config->{netskope}{NTSKP_PERIOD}; +my $NTSKP_SCORE = $config->{netskope}{NTSKP_SCORE}; +my $NTSKP_CATEGORIES = $config->{netskope}{NTSKP_CATEGORIES}; +my $from_email = 'mischa@netskope.com'; +my $to_email = 'mischa@netskope.com'; +my $subject = "AZ Blocklist Report"; + +#print "Tenant: $NTSKP_TENANT\n"; + +my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=498"; +my $response = HTTP::Tiny->new->get($uri); +#print "HTTP: $response->{status} $response->{reason}\n"; +my $json = JSON::PP->new->utf8->decode($response->{content}); +#print "API: $json->{'status'}\n"; +my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + +my $length = (@{$data}); +if ($length == 0) { + print "No widgets found\n"; + last; +} + +open my $fh_email, "|-", "/usr/sbin/sendmail -t"; +printf $fh_email "To: %s\n", $to_email; +printf $fh_email "From: %s\n", $from_email; +printf $fh_email "Subject: %s\n\n", $subject; + +for (my $i = 0; $i < $length; $i++) { + print "$data->[$i]->{'id'} - $data->[$i]->{'name'}\n"; + print $fh_email "$data->[$i]->{'id'} - $data->[$i]->{'name'}\n"; + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$data->[$i]->{'id'}"; + $response = HTTP::Tiny->new->get($uri); + #print "HTTP: $response->{status} $response->{reason}\n"; + my $count = 0; + foreach (split(/\r\n/, $response->{content})) { + last if ($count == 30); + my @fields = split(/,/); + next if ($fields[1] =~ '"'); + print "$fields[1],"; + print $fh_email "$fields[1],"; + $count++; + } + print "\n"; + print $fh_email "\n"; +} +close $fh_email; diff --git a/ntskp-api-06.pl b/ntskp-api-06.pl new file mode 100755 index 0000000..6d57d61 --- /dev/null +++ b/ntskp-api-06.pl @@ -0,0 +1,20 @@ +#!/usr/bin/perl -w +use strict; +use warnings; +use autodie; +use POSIX qw(strftime); +use File::Temp qw/ tempfile tempdir /; +use Text::CSV; + +my $file = "widget-18465-20200611.txt"; +open my $fh, "<", $file; + +my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + +my $header = $csv->getline($fh); + +while (my $row = $csv->getline($fh)) { + print "$row->[1]\n"; +} +close $fh; + diff --git a/ntskp-phishing.txt b/ntskp-phishing.txt new file mode 100644 index 0000000..2e116c5 --- /dev/null +++ b/ntskp-phishing.txt @@ -0,0 +1,22 @@ + + + + + +
+Unfortunately, the service you are using has been targeted by a +very sophisticated password attack. To protect you, the security +team recommended that we reset all customer passwords immediately. +Effective immediately, you will be required to reset your service +password before you can login again. To reset your password please +use your regular service password reset link. +

+ +
Regards,
+

+
Your friendly neighbourhood scammer
+

+ diff --git a/ntskp-send.sh b/ntskp-send.sh new file mode 100755 index 0000000..0ca9045 --- /dev/null +++ b/ntskp-send.sh @@ -0,0 +1,12 @@ +# high5.nl +#cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer \nContent-Type: text/html")" mischa@high5.nl +#cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer \nContent-Type: text/html")" mischa@high5.nl +# ntskp.com +cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer \nContent-Type: text/html")" herman.akker23@ntskp.com +cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer \nContent-Type: text/html")" herman.akker23@ntskp.com +# Microsoft +cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer \nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com +cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer \nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com +# Died +#cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer \nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com +#cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer \nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com diff --git a/ntskp-spam.txt b/ntskp-spam.txt new file mode 100644 index 0000000..75ebdcf --- /dev/null +++ b/ntskp-spam.txt @@ -0,0 +1,17 @@ + + + + + +
+Please have a look at this CV at your earliest convenience. +

+ +
Regards,
+

+
Your friendly neighbourhood scammer
+

+ diff --git a/oss1.py b/oss1.py new file mode 100755 index 0000000..ee53a73 --- /dev/null +++ b/oss1.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +import os +import sys +import json +import requests +import configparser + +############################################### +# Look for oss.cnf file in current working directory +CONFIG_FILE = "./oss.cnf" +if not os.path.isfile(CONFIG_FILE): + logging.error(f"The config file {CONFIG_FILE} doesn't exist") + sys.exit(1) +config = configparser.RawConfigParser() +config.read(CONFIG_FILE) +NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT') +NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN') +NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD') + +############################################### + +ssl_session = requests.Session() +uri = f"{NTSKP_TENANT}/api/v1/alerts?token={NTSKP_TOKEN}&timeperiod={NTSKP_PERIOD}&groupby=application&query=access_method+eq+Client+and+action+eq+block" +try: + r = ssl_session.get(uri) + r.raise_for_status() +except Exception as e: + logging.error(f'Error: {str(e)}') + sys.exit(1) +json = r.json() +if 'data' in json: + for item in json['data']: + if 'app' in item: + print(f"{item['app']} - {item['category']}", end=', ') + print() diff --git a/oss2.py b/oss2.py new file mode 100755 index 0000000..8678275 --- /dev/null +++ b/oss2.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +import os +import sys +import json +import requests +import configparser + +############################################### +# Look for oss.cnf file in current working directory +CONFIG_FILE = "./oss.cnf" +if not os.path.isfile(CONFIG_FILE): + logging.error(f"The config file {CONFIG_FILE} doesn't exist") + sys.exit(1) +config = configparser.RawConfigParser() +config.read(CONFIG_FILE) +NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT') +NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN') +NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD') + +############################################### + +ssl_session = requests.Session() +uri = f"{NTSKP_TENANT}/api/v1/report?token={NTSKP_TOKEN}&timeperiod={NTSKP_PERIOD}&type=connection&groupby=application&query=app-cci-app-tag+eq+'Under_Review'" +try: + r = ssl_session.get(uri) + r.raise_for_status() +except Exception as e: + logging.error(f'Error: {str(e)}') + sys.exit(1) +json = r.json() +if 'data' in json: + for item in json['data']: + print(f"{item['app']}", end=', ') + print() diff --git a/oss3.py b/oss3.py new file mode 100755 index 0000000..c3d9c66 --- /dev/null +++ b/oss3.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +import os +import sys +import json +import requests +import configparser + +############################################### +# Look for oss.cnf file in current working directory +CONFIG_FILE = "./oss.cnf" +if not os.path.isfile(CONFIG_FILE): + logging.error(f"The config file {CONFIG_FILE} doesn't exist") + sys.exit(1) +config = configparser.RawConfigParser() +config.read(CONFIG_FILE) +NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT') +NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN') +NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD') + +############################################### + +ssl_session = requests.Session() +uri = f"{NTSKP_TENANT}/api/v1/report?token={NTSKP_TOKEN}&timeperiod={NTSKP_PERIOD}&type=connection&groupby=application&query=app-cci-app-tag+eq+'Pending_GRC_Review'" +try: + r = ssl_session.get(uri) + r.raise_for_status() +except Exception as e: + logging.error(f'Error: {str(e)}') + sys.exit(1) +json = r.json() +if 'data' in json: + for item in json['data']: + print(f"{item['app']}", end=', ') + print() diff --git a/tbi.pl b/tbi.pl new file mode 100755 index 0000000..2607156 --- /dev/null +++ b/tbi.pl @@ -0,0 +1,107 @@ +#!/usr/bin/env perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615 +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.024; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use MIME::Lite; + +my $LOGMODE = ""; +#my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf"); +my @CONFIG_FILES = grep { -e } ('./tbi.cnf'); +my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8'); +my $USER_COUNT = $config->{report}{USER_COUNT}; +my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN}; +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_TIMEPERIOD = $config->{netskope}{NTSKP_TIMEPERIOD}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); +my $EMAIL_CSV = ""; + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE; +} + +sub _check_return { + my ($status, $content, $uri) = @_; + if ($status =~ /^2/ && $LOGMODE) { + print "URI: $uri\nHTTP RESPONSE: $status\n"; + print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG"); + } + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n", + ); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + my $uri = "$NTSKP_TENANT/api/v1/alerts?token=$NTSKP_TOKEN&timeperiod=$NTSKP_TIMEPERIOD&type=policy"; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS); + my $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}; + printf "%-7s %-45s %-35s %s\n", "Action", "Page", "Policy", "Category"; + say "#############################################################################################################################"; + + my @seen; + for my $item (@{$data}) { + if (exists($item->{'page'})) { + next if (grep {$_ eq $item->{'site'}} @seen); + printf "%-7s %-45s %-35s %s\n", $item->{'action'}, $item->{'page'}, $item->{'policy'}, $item->{'category'}; + push @seen, $item->{'site'}; + } + } + +} + +say "Running in $LOGMODE mode..." if $LOGMODE; +netskope(); +say "Completed." if $LOGMODE; diff --git a/z.pl b/z.pl new file mode 100755 index 0000000..630f8ba --- /dev/null +++ b/z.pl @@ -0,0 +1,199 @@ +#!/usr/bin/env perl +# +# Copyright 2020, Mischa Peters , Netskope. +# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615 +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# ZScaler integration with Netskope +# +use 5.024; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; +use MIME::Lite; + +my $LOGMODE = ""; +my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf"); +my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8'); +my $USER_COUNT = $config->{report}{USER_COUNT}; +my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN}; +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; +my $PROXY = $config->{general}{PROXY}; +my $SMTP = $config->{general}{SMTP}; +my $FROM = $config->{general}{FROM}; +my $TO = $config->{general}{TO}; +my $SUBJECT = $config->{general}{SUBJECT}; +my $TEXT = $config->{general}{TEXT} . "\n\n"; +my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache"); +my $EMAIL_CSV = ""; + +### Netskope ### +sub mail_csv { + my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE; +} + +sub _check_return { + my ($status, $content, $uri) = @_; + if ($status =~ /^2/ && $LOGMODE) { + print "URI: $uri\nHTTP RESPONSE: $status\n"; + print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG"); + } + if ($status !~ /^2/) { + print "URI: $uri\nHTTP RESPONSE: $status\n$content\n"; + my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT', + Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n", + ); + $msg->attach(Type => 'text/csv', Data => $EMAIL_CSV); + $msg->send('smtp', $SMTP, Debug=>0); + say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE; + say "exit 1"; + exit 1; + } +} + +sub netskope { + ### Collect widget IDs + my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID"; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS); + my $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'}; + if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); } + my %csv_content; + + ### Collect widget data and write to CSV + for my $widget (@{$data}) { + $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}"; + $response = $request->get($uri); + print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG"); + _check_return($response->{'status'}, $response->{'content'}, $uri); + $csv_content{$widget->{'name'}} = $response->{'content'}; + } + + ### Process domains from CSV + my @blocklist; + for my $widget_name (keys %csv_content) { + my $count = 0; + my $csv = Text::CSV->new({binary => 1, auto_diag => 1}); + open my $fh_in, "<", \$csv_content{$widget_name}; + $csv->column_names($csv->getline($fh_in)); + # "Application","Domain","Category","CCI","Users" + + print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE; + $EMAIL_CSV .= "$widget_name\n"; + DOMAIN: + while (my $row = $csv->getline_hr($fh_in)) { + last DOMAIN if ($count == $MAX_DOMAIN); + if ($row->{'Users'} < $USER_COUNT) { + print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG"); + print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG"); + $EMAIL_CSV .= "$row->{'Domain'},"; + push @blocklist, $row->{'Domain'}; + $count++; + } + } + print "\n\n" if $LOGMODE; + $EMAIL_CSV .= "\n"; + } + return @blocklist; +} + +#sub updateUrlList { + #my @domains = @{$_[0]}; + #my $uri = "$NTSKP_TENANT/api/v1/updateUrlList?token=$NTSKP_TOKEN"; + #my $request = HTTP::Tiny->new('default_headers' => \%HEADERS); + #$body = JSON::PP->new->encode({name => $NTSKP_URL_CATEGORY, list => \@domains}); + #my $response = $request->post($uri, {'content' => $body}); + #_check_return($response->{'status'}, $response->{'content'}, $uri); +#} + +### Zscaler ### + +sub zscaler { + my @domains = @{$_[0]}; + + ### Authenticate + my $now = int(gettimeofday * 1000); + my $n = substr($now, -6); + my $r = sprintf "%06d", $n >> 1; + my $key; + for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); + } + for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); + } + my $uri = "$ZS_BASE_URI/authenticatedSession"; + my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); + my $jar = HTTP::CookieJar->new; + my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar); + my $response = $request->post($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Get filter list id + $uri = "$ZS_BASE_URI/urlCategories/lite"; + $response = $request->get($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + my $json = JSON::PP->new->utf8->decode($response->{'content'}); + my $id; + for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } + } + + ### Push Domains + $uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories"; + my $method = defined($id) ? "put" : "post"; + my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); + splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS; + $body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); + $response = $request->$method($uri, {'content' => $body}); + _check_return($response->{'status'}, $response->{'content'}, $uri); + + ### Delete authenticadSession + $uri = "$ZS_BASE_URI/authenticatedSession"; + $response = $request->delete($uri); + _check_return($response->{'status'}, $response->{'content'}, $uri); +} + +say "Running in $LOGMODE mode..." if $LOGMODE; +my @domains = netskope(); +zscaler(\@domains); +mail_csv(); +say "Completed." if $LOGMODE; diff --git a/zscaler-api.pl b/zscaler-api.pl new file mode 100755 index 0000000..68a5b92 --- /dev/null +++ b/zscaler-api.pl @@ -0,0 +1,87 @@ +#!/usr/bin/perl +use 5.024; +use strict; +use warnings; +use autodie; +use Config::Tiny; +use Time::HiRes qw(gettimeofday); +use POSIX qw(strftime); +use HTTP::Tiny; +use HTTP::CookieJar; +use JSON::PP; +use Text::CSV; + +my $verbose = 1; +my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"; +my $config = Config::Tiny->read($CONFIG_FILE, 'utf8'); +my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT}; +my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN}; +my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID}; +my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS}; +my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI}; +my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY}; +my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME}; +my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD}; +my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME}; +my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC}; + +say "Running..."; + +# Authenticate +my $now = int(gettimeofday * 1000); +my $n = substr($now, -6); +my $r = sprintf "%06d", $n >> 1; +my $key; +for my $i (0..length($n)-1) { + $key .= substr($ZS_API_KEY, substr($n, $i, 1), 1); +} +for my $i (0..length($r)-1) { + $key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1); +} +my $uri = "$ZS_BASE_URI/authenticatedSession"; +my $body = JSON::PP->new->space_after->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now}); +my $jar = HTTP::CookieJar->new; +my $request = HTTP::Tiny->new('default_headers' => {"Content-Type" => "application/json", "Cache-Control" => "no-cache"}, 'cookie_jar' => $jar); +my $response = $request->post($uri, {'content' => $body}); +if ($verbose) { + say "POST $uri"; + say "BODY $body"; + say "HTTP " . $response->{'status'}; + say "COOKIE " . $jar->cookie_header($ZS_BASE_URI); +} + +# Get filter list id +$uri = "$ZS_BASE_URI/urlCategories/lite"; +$response = $request->get($uri); +my $json = JSON::PP->new->utf8->decode($response->{'content'}); +my $id; +for my $item (@{$json}) { + if (exists($item->{'configuredName'})) { + if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) { + $id = $item->{'id'}; + } + } +} + +# Push Domains +my @domains = ('secomtrust.net', 'baidupcs.com', 'cloud.baidu.com'); +$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories"; +my $method = defined($id) ? "put" : "post"; +my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime); +$#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains; +$body = JSON::PP->new->space_after->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description}); +$response = $request->$method($uri, {'content' => $body}); +if ($verbose) { + say uc($method) . " $uri"; + say "BODY $body"; + say "HTTP " . $response->{'status'}; + say "RESPONSE " . $response->{'content'} if ($response->{'status'} =~ /^4/); +} + +# Delete authenticadSession +$uri = "$ZS_BASE_URI/authenticatedSession"; +$response = $request->delete($uri); +if ($verbose) { + say "DELETE $uri"; + say "HTTP " . $response->{'status'}; +}